From 4a646dd9a0d7acd2ea7c3fbd57de1e17ead7cec8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 14:53:40 -0700 Subject: [PATCH 01/43] ci(e2e): run a PR's changed e2e tests three times behind a human-approved environment Adds a required-check candidate that selects the tests/e2e test files a PR added or modified, boots a stage-mirror stack on the runner (migrations, backend, two gateway processes behind nginx, Postgres, Jaeger, TLS cluster Valkey), and runs those files three times with retries off. The run job sits behind the e2e-changed GitHub environment, so a reviewer approves each run before the OIDC token that reads the provider keys from AWS Secrets Manager exists. Supersedes #34981 --- .github/e2e-stack/down.sh | 17 ++ .github/e2e-stack/secrets_to_env.py | 28 +++ .github/e2e-stack/up.sh | 198 +++++++++++++++++++ .github/workflows/test-e2e-changed.yml | 171 ++++++++++++++++ tests/e2e/CONTRIBUTING.md | 4 + tests/e2e/gateway/stage_mirror_ci_config.yml | 63 ++++++ 6 files changed, 481 insertions(+) create mode 100755 .github/e2e-stack/down.sh create mode 100644 .github/e2e-stack/secrets_to_env.py create mode 100755 .github/e2e-stack/up.sh create mode 100644 .github/workflows/test-e2e-changed.yml create mode 100644 tests/e2e/gateway/stage_mirror_ci_config.yml diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh new file mode 100755 index 00000000000..9f72f2d6e64 --- /dev/null +++ b/.github/e2e-stack/down.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -uo pipefail + +STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" + +for pid_file in "${STACK_DIR}"/pids/*.pid; do + [[ -f "${pid_file}" ]] || continue + pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null + kill -TERM "$(cat "${pid_file}")" 2>/dev/null + rm -f "${pid_file}" +done + +for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do + docker rm -f "${container}" >/dev/null 2>&1 +done + +exit 0 diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py new file mode 100644 index 00000000000..25b22555c29 --- /dev/null +++ b/.github/e2e-stack/secrets_to_env.py @@ -0,0 +1,28 @@ +import sys +from pathlib import Path + +from pydantic import TypeAdapter + +secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) + + +def main() -> int: + env_path = Path(sys.argv[1]) + secrets = secrets_adapter.validate_json(sys.stdin.read()) + unwritable = tuple( + key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value + ) + if unwritable: + _ = sys.stderr.write(f"values contain characters unsafe for both bash and dotenv: {', '.join(unwritable)}\n") + return 1 + lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) + with env_path.open("a") as handle: + _ = handle.write("\n".join(lines) + "\n") + for value in secrets.values(): + if value: + _ = sys.stdout.write(f"::add-mask::{value}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh new file mode 100755 index 00000000000..06fce35a896 --- /dev/null +++ b/.github/e2e-stack/up.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" +CERTS_DIR="${STACK_DIR}/certs" +LOGS_DIR="${STACK_DIR}/logs" +PIDS_DIR="${STACK_DIR}/pids" + +POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" +VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1}" +JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}" +NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29-alpine}" + +LB_PORT="${E2E_LB_PORT:-4000}" +GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}" +GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}" +BACKEND_PORT="${E2E_BACKEND_PORT:-4001}" +REDIS_PORT="${E2E_REDIS_PORT:-6379}" +DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}" +DATABASE_PORT="${E2E_DATABASE_PORT:-5432}" +DATABASE_USER="${E2E_DATABASE_USER:-litellm}" +DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" +DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" +JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" +JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" + +MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" + +mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" + +log() { printf 'e2e-stack: %s\n' "$*"; } + +port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } + +wait_for() { + local label="$1" check="$2" deadline=$((SECONDS + ${3:-120})) + until eval "${check}"; do + if ((SECONDS >= deadline)); then + log "timed out waiting for ${label}" + tail -n 60 "${LOGS_DIR}"/*.log 2>/dev/null || true + exit 1 + fi + sleep 2 + done + log "${label} is up" +} + +if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then + set -a + source "${REPO_ROOT}/tests/e2e/.env" + set +a +fi + +if ! port_open "${DATABASE_PORT}"; then + docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \ + -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \ + "${POSTGRES_IMAGE}" >/dev/null +fi +wait_for "postgres" "port_open ${DATABASE_PORT}" + +if ! port_open "${JAEGER_QUERY_PORT}"; then + docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \ + "${JAEGER_IMAGE}" >/dev/null +fi +wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null" + +openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null +openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \ + -subj "/CN=litellm-e2e-ca" \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -out "${CERTS_DIR}/ca.crt" 2>/dev/null +openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null +openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null +openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \ + -CAcreateserial -days 7 -sha256 \ + -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \ + -out "${CERTS_DIR}/server.crt" 2>/dev/null +chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt + +CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')" +cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem" + +docker rm -f e2e-valkey >/dev/null 2>&1 || true +docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \ + "${VALKEY_IMAGE}" valkey-server \ + --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \ + --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \ + --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null +VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}" +wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG" +${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null +wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok" + +CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml" +CONFIG_PATH="${CONFIG_SOURCE}" +if [[ "${REDIS_PORT}" != "6379" ]]; then + CONFIG_PATH="${STACK_DIR}/litellm-config.yml" + sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}" +fi + +SERVER_ENV=( + "LITELLM_MASTER_KEY=${MASTER_KEY}" + "DATABASE_HOST=${DATABASE_HOST}" + "DATABASE_PORT=${DATABASE_PORT}" + "DATABASE_USER=${DATABASE_USER}" + "DATABASE_PASSWORD=${DATABASE_PASSWORD}" + "DATABASE_NAME=${DATABASE_NAME}" + "DISABLE_SCHEMA_UPDATE=true" + "REDIS_HOST=127.0.0.1" + "REDIS_PORT=${REDIS_PORT}" + "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]" + "CONFIG_FILE_PATH=${CONFIG_PATH}" + "STORE_MODEL_IN_DB=True" + "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" + "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" + "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" + "PYTHONPATH=${REPO_ROOT}" +) +if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then + printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" + SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json") +fi + +cd "${REPO_ROOT}" + +log "running migrations" +env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 + +start_server() { + local name="$1"; shift + env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 & + echo $! > "${PIDS_DIR}/${name}.pid" +} + +start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" +start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" +start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" + +if [[ "$(uname)" == "Linux" ]]; then + NGINX_UPSTREAM_HOST=127.0.0.1 + NGINX_DOCKER_ARGS=(--network host) +else + NGINX_UPSTREAM_HOST=host.docker.internal + NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}") +fi + +cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true +docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ + -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null + +wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 +wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 +wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300 +wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60 + +cat > "${STACK_DIR}/stack.env" <> "${GITHUB_OUTPUT}" + if [ -n "${tests}" ]; then + echo "any=true" >> "${GITHUB_OUTPUT}" + echo "selected e2e tests: ${tests}" + else + echo "any=false" >> "${GITHUB_OUTPUT}" + echo "no e2e changes; nothing to run" + fi + + run: + name: Run changed e2e tests against the stage-mirror stack + needs: detect + if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 90 + environment: e2e-changed + permissions: + contents: read + id-token: write + services: + postgres: + image: postgres:16.6 + env: + POSTGRES_USER: litellm + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U litellm" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + jaeger: + image: jaegertracing/jaeger:2.10.0 + ports: + - 4318:4318 + - 16686:16686 + steps: + - name: Validate configuration + env: + ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} + run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; } + + - name: Checkout + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen \ + --extra proxy --extra proxy-runtime --extra extra_proxy \ + --extra semantic-router --extra bedrock-realtime \ + --group ci --group proxy-dev --group e2e-dev + uv pip install "pipecat-ai[openai]==1.4.0" + + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Install Playwright chromium + run: uv run --no-sync playwright install --with-deps chromium + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} + aws-region: us-east-1 + role-session-name: litellm-e2e-changed-${{ github.run_id }} + + - name: Fetch provider credentials from AWS Secrets Manager + run: | + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \ + --query SecretString --output text \ + | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \ + --query SecretString --output text \ + | jq -R -s '{"LITELLM_LICENSE": rtrimstr("\n")}' \ + | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env + + - name: Boot the stage-mirror stack + run: bash .github/e2e-stack/up.sh + + - name: Export stack environment + run: | + master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)" + echo "::add-mask::${master_key}" + cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}" + + - name: Run the selected tests three times with retries off + env: + TESTS: ${{ needs.detect.outputs.tests }} + run: | + read -r -a test_files <<< "${TESTS}" + for pass in 1 2 3; do + echo "::group::pass ${pass} of 3" + set +e + uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider + status=$? + set -e + echo "::endgroup::" + if [ "${status}" = "5" ]; then + echo "selected files collected no runnable tests" + exit 0 + fi + if [ "${status}" != "0" ]; then + echo "::error::pass ${pass} of 3 failed with exit code ${status}" + exit "${status}" + fi + done + + - name: Show stack logs on failure + if: failure() + run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 871d6b3904c..e04781ab205 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -52,6 +52,10 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy +### The pull request check + +Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down + ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml new file mode 100644 index 00000000000..57a92fd47fb --- /dev/null +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -0,0 +1,63 @@ +general_settings: + store_prompts_in_spend_logs: true + database_connection_pool_limit: 10 + forward_client_headers_to_llm_api: false + maximum_spend_logs_retention_period: "60d" + maximum_spend_logs_cleanup_cron: "0 1 * * *" + proxy_budget_rescheduler_min_time: 15 + proxy_budget_rescheduler_max_time: 20 + +litellm_settings: + drop_params: true + default_redis_ttl: 20 + request_timeout: 600 + num_retries: 3 + json_logs: true + store_audit_logs: true + cache: true + cache_params: + type: redis + host: 127.0.0.1 + port: 6379 + redis_startup_nodes: + - host: 127.0.0.1 + port: 6379 + ssl: true + callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] + require_auth_for_metrics_endpoint: false + +router_settings: + routing_strategy: simple-shuffle + num_retries: 3 + allowed_fails: 5 + cooldown_time: 30 + +model_list: + - model_name: gpt-5.5 + litellm_params: + model: openai/gpt-5.5 + api_key: os.environ/OPENAI_API_KEY + - model_name: claude-haiku-4-5 + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: gemini-2.5-flash-vertex + litellm_params: + model: vertex_ai/gemini-2.5-flash + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: us-central1 + vertex_credentials: os.environ/VERTEXAI_CREDENTIALS + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + +mcp_servers: + devin: + url: "https://mcp.devin.ai/mcp" + auth_type: api_key + auth_value: os.environ/DEVIN_API_KEY From b348ed7f09709647d3f9bce38be3ba49741be307 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 14:56:52 -0700 Subject: [PATCH 02/43] ci(e2e): only tail stack logs when the stack actually booted --- .github/workflows/test-e2e-changed.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 6ad252e7246..a4466bba3fd 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -136,6 +136,7 @@ jobs: | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env - name: Boot the stage-mirror stack + id: boot run: bash .github/e2e-stack/up.sh - name: Export stack environment @@ -167,5 +168,5 @@ jobs: done - name: Show stack logs on failure - if: failure() + if: failure() && steps.boot.conclusion != 'skipped' run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log From 743f94f82abf9d4963c087809fab782d47761b09 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 14:59:15 -0700 Subject: [PATCH 03/43] ci(e2e): fail when nothing collects and ignore lanes with their own checks in the smoke trigger --- .github/workflows/test-e2e-changed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index a4466bba3fd..7a638801161 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -34,7 +34,7 @@ jobs: | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \ | grep -vE '^tests/e2e/(ui|claude_code|load)/' \ | sort -u | tr '\n' ' ' | sed 's/ $//')" || true - if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -v '^tests/e2e/ui/' \ + if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE '^tests/e2e/(ui|claude_code|load)/' \ | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then tests="${SMOKE_TESTS}" echo "harness or stack changed without a test file; running the smoke suite" @@ -158,8 +158,8 @@ jobs: set -e echo "::endgroup::" if [ "${status}" = "5" ]; then - echo "selected files collected no runnable tests" - exit 0 + echo "::error::the selected files collected no runnable tests, so nothing was verified" + exit 1 fi if [ "${status}" != "0" ]; then echo "::error::pass ${pass} of 3 failed with exit code ${status}" From c62643f0cf72c2c02f08c2073a621e81d906b056 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 15:39:21 -0700 Subject: [PATCH 04/43] ci(e2e): drop trailing newlines from fetched secret values before writing the env --- .github/e2e-stack/secrets_to_env.py | 2 +- .github/workflows/test-e2e-changed.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index 25b22555c29..65022931d81 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -8,7 +8,7 @@ secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) def main() -> int: env_path = Path(sys.argv[1]) - secrets = secrets_adapter.validate_json(sys.stdin.read()) + secrets = {key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()} unwritable = tuple( key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value ) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 7a638801161..b20c84af02d 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -132,7 +132,7 @@ jobs: | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \ --query SecretString --output text \ - | jq -R -s '{"LITELLM_LICENSE": rtrimstr("\n")}' \ + | jq -R -s '{"LITELLM_LICENSE": .}' \ | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env - name: Boot the stage-mirror stack From 2bce27cfa6a0277c7237649c1c322883887e781a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 15:52:42 -0700 Subject: [PATCH 05/43] docs(e2e): describe the dedicated, least-privilege, capped credentials behind the pull request check --- tests/e2e/CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index e04781ab205..b984791c738 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -56,6 +56,8 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down +Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change + ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop From 1c4e46f17e813d5eb8e5fa0c2577f7bef3a32b9a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 17:15:36 -0700 Subject: [PATCH 06/43] ci(e2e): reload the stack config every 7s so the harness propagation budget holds, and only mask credential-named values --- .github/e2e-stack/secrets_to_env.py | 7 +++++-- tests/e2e/gateway/stage_mirror_ci_config.yml | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index 65022931d81..d2d6675d690 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -1,9 +1,12 @@ +import re import sys from pathlib import Path +from typing import Final from pydantic import TypeAdapter secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) +SECRET_NAME: Final = re.compile(r"KEY|SECRET|TOKEN|PASS|CREDENTIAL|LICENSE|AUTH") def main() -> int: @@ -18,8 +21,8 @@ def main() -> int: lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) with env_path.open("a") as handle: _ = handle.write("\n".join(lines) + "\n") - for value in secrets.values(): - if value: + for key, value in secrets.items(): + if value and SECRET_NAME.search(key): _ = sys.stdout.write(f"::add-mask::{value}\n") return 0 diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 57a92fd47fb..21664c2d0a1 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -1,4 +1,5 @@ general_settings: + proxy_config_reload_interval_seconds: 7 store_prompts_in_spend_logs: true database_connection_pool_limit: 10 forward_client_headers_to_llm_api: false From 3ead9d16884c652ccbabe618f7526d74b3f4743e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 19 Jun 2026 16:43:02 -0500 Subject: [PATCH 07/43] feat(vertex): add Lyria model support --- .../llms/vertex_ai/interactions/__init__.py | 3 + ...odel_prices_and_context_window_backup.json | 81 +++++++++++++++++++ .../vertex_passthrough_logging_handler.py | 69 ++++++++++++++++ model_prices_and_context_window.json | 81 +++++++++++++++++++ ...test_vertex_passthrough_logging_handler.py | 68 ++++++++++++++++ tests/test_litellm/test_utils.py | 37 +++++++++ 6 files changed, 339 insertions(+) create mode 100644 tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py diff --git a/litellm/llms/vertex_ai/interactions/__init__.py b/litellm/llms/vertex_ai/interactions/__init__.py index e69de29bb2d..3e2309d21ef 100644 --- a/litellm/llms/vertex_ai/interactions/__init__.py +++ b/litellm/llms/vertex_ai/interactions/__init__.py @@ -0,0 +1,3 @@ +from .transformation import VertexAIInteractionsConfig + +__all__ = ["VertexAIInteractionsConfig"] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d8a8f84b032..e854e8fc9bb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45511,6 +45511,87 @@ "output_cost_per_token": 4e-07, "supports_tool_choice": true }, + "vertex_ai/lyria-002": { + "litellm_provider": "vertex_ai", + "max_audio_length_hours": 0.009111111111111111, + "max_audio_per_prompt": 4, + "mode": "chat", + "output_cost_per_second": 0.002, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "vertex_ai/lyria-3-clip-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_image": 0.04, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1beta/interactions" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_image_input": true, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": true, + "supports_web_search": false + }, + "vertex_ai/lyria-3-pro-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_image": 0.08, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1beta/interactions" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_image_input": true, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": true, + "supports_web_search": false + }, "vertex_ai/meta/llama-3.1-405b-instruct-maas": { "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-llama_models", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 49ec18013b5..a9f68f8380b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -46,6 +46,7 @@ else: # Define EndpointType locally to avoid import issues EndpointType = Any +_LYRIA_SECONDS_PER_AUDIO_PREDICTION = 30 class VertexPassthroughLoggingHandler: @@ -270,6 +271,16 @@ class VertexPassthroughLoggingHandler: _json_response: Final[dict[str, object]] = httpx_response.json() litellm_prediction_response: ModelResponse | EmbeddingResponse | ImageResponse = ModelResponse() + if VertexPassthroughLoggingHandler._is_lyria_predict_response( + model=model, + json_response=_json_response, + ): + return VertexPassthroughLoggingHandler._handle_lyria_predict_response( + json_response=_json_response, + logging_obj=logging_obj, + model=model, + kwargs=kwargs, + ) if vertex_image_generation_class.is_image_generation_response(_json_response): litellm_prediction_response = vertex_image_generation_class.process_image_generation_response( _json_response, @@ -323,6 +334,64 @@ class VertexPassthroughLoggingHandler: "kwargs": kwargs, } + @staticmethod + def _handle_lyria_predict_response( + json_response: dict, + logging_obj: LiteLLMLoggingObj, + model: str, + kwargs: dict, + ) -> PassThroughEndpointLoggingTypedDict: + prediction_count: Final = ( + VertexPassthroughLoggingHandler._get_lyria_audio_prediction_count( + json_response=json_response + ) + ) + model_info: Final = litellm.model_cost.get(f"vertex_ai/{model}", {}) + response_cost: Final = ( + model_info.get("output_cost_per_second", 0.0) + * _LYRIA_SECONDS_PER_AUDIO_PREDICTION + * prediction_count + ) + + logging_obj.model = model + logging_obj.model_call_details["model"] = model + logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" + logging_obj.custom_llm_provider = "vertex_ai" + logging_obj.model_call_details["response_cost"] = response_cost + + kwargs["response_cost"] = response_cost + kwargs["model"] = model + kwargs["custom_llm_provider"] = "vertex_ai" + + standard_pass_through_response_object: Final[StandardPassThroughResponseObject] = { + "response": json_response, + } + return { + "result": standard_pass_through_response_object, + "kwargs": kwargs, + } + + @staticmethod + def _is_lyria_predict_response(model: str, json_response: dict) -> bool: + return ( + model == "lyria-002" + and VertexPassthroughLoggingHandler._get_lyria_audio_prediction_count( + json_response=json_response + ) + > 0 + ) + + @staticmethod + def _get_lyria_audio_prediction_count(json_response: dict) -> int: + predictions: Final = json_response.get("predictions") + if not isinstance(predictions, list): + return 0 + return sum( + 1 + for prediction in predictions + if isinstance(prediction, dict) and prediction.get("audioContent") + ) + @staticmethod def _extract_embed_content_input(request_body: dict | None, batch: bool) -> str: """Extract raw input text from an :embedContent or :batchEmbedContents request body for token counting.""" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d8a8f84b032..e854e8fc9bb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45511,6 +45511,87 @@ "output_cost_per_token": 4e-07, "supports_tool_choice": true }, + "vertex_ai/lyria-002": { + "litellm_provider": "vertex_ai", + "max_audio_length_hours": 0.009111111111111111, + "max_audio_per_prompt": 4, + "mode": "chat", + "output_cost_per_second": 0.002, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "vertex_ai/lyria-3-clip-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_image": 0.04, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1beta/interactions" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_image_input": true, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": true, + "supports_web_search": false + }, + "vertex_ai/lyria-3-pro-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_image": 0.08, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1beta/interactions" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_image_input": true, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": true, + "supports_web_search": false + }, "vertex_ai/meta/llama-3.1-405b-instruct-maas": { "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-llama_models", diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py new file mode 100644 index 00000000000..7af67cc0795 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -0,0 +1,68 @@ +from datetime import datetime +from unittest.mock import MagicMock + +import httpx +import litellm +import pytest + +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( + VertexPassthroughLoggingHandler, +) + + +def test_lyria_predict_response_preserves_audio_response_and_logs_cost( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + {"output_cost_per_second": 0.002}, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "audioContent": "clip-1", + "mimeType": "audio/wav", + }, + { + "audioContent": "clip-2", + "mimeType": "audio/wav", + }, + ] + }, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["result"] == { + "response": { + "predictions": [ + { + "audioContent": "clip-1", + "mimeType": "audio/wav", + }, + { + "audioContent": "clip-2", + "mimeType": "audio/wav", + }, + ] + } + } + assert result["kwargs"]["model"] == "lyria-002" + assert result["kwargs"]["custom_llm_provider"] == "vertex_ai" + assert result["kwargs"]["response_cost"] == pytest.approx(0.12) + assert logging_obj.model == "lyria-002" + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.12) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 200cfd02197..d8255e9ff1b 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1067,6 +1067,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/images/variations", "/v1/images/edits", "/v1/batch", + "/v1beta/interactions", "/v1/audio/transcriptions", "/v1/audio/speech", "/v1/ocr", @@ -2833,6 +2834,42 @@ def test_gemini_lyria_3_preview_models_in_cost_map(): assert clip["output_cost_per_image"] == 0.04 +def test_vertex_ai_lyria_models_in_cost_map(): + import json + from pathlib import Path + + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + lyria_2 = model_cost.get("vertex_ai/lyria-002") + clip = model_cost.get("vertex_ai/lyria-3-clip-preview") + pro = model_cost.get("vertex_ai/lyria-3-pro-preview") + + assert lyria_2 is not None + assert clip is not None + assert pro is not None + assert lyria_2["litellm_provider"] == "vertex_ai" + assert clip["litellm_provider"] == "vertex_ai" + assert pro["litellm_provider"] == "vertex_ai" + assert lyria_2["output_cost_per_second"] == 0.002 + assert lyria_2["supported_modalities"] == ["text"] + assert lyria_2["supported_output_modalities"] == ["audio"] + assert lyria_2["supports_audio_output"] is True + assert clip["output_cost_per_image"] == 0.04 + assert pro["output_cost_per_image"] == 0.08 + assert clip["supported_endpoints"] == ["/v1beta/interactions"] + assert pro["supported_endpoints"] == ["/v1beta/interactions"] + assert clip["supported_modalities"] == ["text", "image"] + assert pro["supported_modalities"] == ["text", "image"] + assert clip["supported_regions"] == ["global"] + assert pro["supported_regions"] == ["global"] + assert clip["supports_audio_output"] is True + assert pro["supports_audio_output"] is True + assert clip["supports_image_input"] is True + assert pro["supports_image_input"] is True + + def test_model_info_for_fireworks_short_form_models(): """ Test that fireworks_ai short-form model entries (fireworks_ai/) From 514e9a1ee62e52480343d084b8f87b54c67f5a41 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 19 Jun 2026 17:44:54 -0500 Subject: [PATCH 08/43] fix(vertex): address lyria review feedback --- ...odel_prices_and_context_window_backup.json | 1 + .../vertex_passthrough_logging_handler.py | 44 ++++++++++-------- model_prices_and_context_window.json | 1 + ...test_vertex_passthrough_logging_handler.py | 46 ++++++++++++++++++- tests/test_litellm/test_utils.py | 2 + 5 files changed, 75 insertions(+), 19 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e854e8fc9bb..1a1d92805e2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45512,6 +45512,7 @@ "supports_tool_choice": true }, "vertex_ai/lyria-002": { + "audio_seconds_per_prediction": 30, "litellm_provider": "vertex_ai", "max_audio_length_hours": 0.009111111111111111, "max_audio_per_prompt": 4, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index a9f68f8380b..6b2d7763fa0 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -44,9 +44,7 @@ else: PassThroughEndpointLogging = Any LiteLLMBatch = Any -# Define EndpointType locally to avoid import issues EndpointType = Any -_LYRIA_SECONDS_PER_AUDIO_PREDICTION = 30 class VertexPassthroughLoggingHandler: @@ -271,11 +269,11 @@ class VertexPassthroughLoggingHandler: _json_response: Final[dict[str, object]] = httpx_response.json() litellm_prediction_response: ModelResponse | EmbeddingResponse | ImageResponse = ModelResponse() - if VertexPassthroughLoggingHandler._is_lyria_predict_response( + if VertexPassthroughLoggingHandler._is_audio_predict_response( model=model, json_response=_json_response, ): - return VertexPassthroughLoggingHandler._handle_lyria_predict_response( + return VertexPassthroughLoggingHandler._handle_audio_predict_response( json_response=_json_response, logging_obj=logging_obj, model=model, @@ -335,23 +333,19 @@ class VertexPassthroughLoggingHandler: } @staticmethod - def _handle_lyria_predict_response( + def _handle_audio_predict_response( json_response: dict, logging_obj: LiteLLMLoggingObj, model: str, kwargs: dict, ) -> PassThroughEndpointLoggingTypedDict: - prediction_count: Final = ( - VertexPassthroughLoggingHandler._get_lyria_audio_prediction_count( - json_response=json_response - ) + prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count( + json_response=json_response ) - model_info: Final = litellm.model_cost.get(f"vertex_ai/{model}", {}) response_cost: Final = ( - model_info.get("output_cost_per_second", 0.0) - * _LYRIA_SECONDS_PER_AUDIO_PREDICTION - * prediction_count - ) + VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) + or 0.0 + ) * prediction_count logging_obj.model = model logging_obj.model_call_details["model"] = model @@ -372,17 +366,31 @@ class VertexPassthroughLoggingHandler: } @staticmethod - def _is_lyria_predict_response(model: str, json_response: dict) -> bool: + def _is_audio_predict_response(model: str, json_response: dict) -> bool: return ( - model == "lyria-002" - and VertexPassthroughLoggingHandler._get_lyria_audio_prediction_count( + VertexPassthroughLoggingHandler._get_audio_prediction_count( json_response=json_response ) > 0 + and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost( + model=model + ) + is not None ) @staticmethod - def _get_lyria_audio_prediction_count(json_response: dict) -> int: + def _get_audio_prediction_unit_cost(model: str) -> float | None: + model_info: Final = litellm.model_cost.get(f"vertex_ai/{model}", {}) + output_cost_per_second: Final = model_info.get("output_cost_per_second") + audio_seconds_per_prediction: Final = model_info.get("audio_seconds_per_prediction") + if not isinstance(output_cost_per_second, (int, float)) or not isinstance( + audio_seconds_per_prediction, (int, float) + ): + return None + return float(output_cost_per_second * audio_seconds_per_prediction) + + @staticmethod + def _get_audio_prediction_count(json_response: dict) -> int: predictions: Final = json_response.get("predictions") if not isinstance(predictions, list): return 0 diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e854e8fc9bb..1a1d92805e2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45512,6 +45512,7 @@ "supports_tool_choice": true }, "vertex_ai/lyria-002": { + "audio_seconds_per_prediction": 30, "litellm_provider": "vertex_ai", "max_audio_length_hours": 0.009111111111111111, "max_audio_per_prompt": 4, diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 7af67cc0795..9f65fa09b1b 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -16,7 +16,10 @@ def test_lyria_predict_response_preserves_audio_response_and_logs_cost( monkeypatch.setitem( litellm.model_cost, "vertex_ai/lyria-002", - {"output_cost_per_second": 0.002}, + { + "audio_seconds_per_prediction": 30, + "output_cost_per_second": 0.002, + }, ) logging_obj = MagicMock() logging_obj.model_call_details = {} @@ -66,3 +69,44 @@ def test_lyria_predict_response_preserves_audio_response_and_logs_cost( assert result["kwargs"]["response_cost"] == pytest.approx(0.12) assert logging_obj.model == "lyria-002" assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.12) + + +def test_audio_predict_response_uses_model_map_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/music-audio-preview", + { + "audio_seconds_per_prediction": 12, + "output_cost_per_second": 0.5, + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "audioContent": "clip", + "mimeType": "audio/wav", + } + ] + }, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/music-audio-preview:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["kwargs"]["model"] == "music-audio-preview" + assert result["kwargs"]["response_cost"] == pytest.approx(6.0) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(6.0) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d8255e9ff1b..6ae7c27b758 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -949,6 +949,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "max_tokens": {"type": "number"}, "metadata": {"type": "object"}, "provider_specific_entry": {"type": "object"}, + "audio_seconds_per_prediction": {"type": "number"}, "mode": { "type": "string", "enum": [ @@ -2852,6 +2853,7 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert lyria_2["litellm_provider"] == "vertex_ai" assert clip["litellm_provider"] == "vertex_ai" assert pro["litellm_provider"] == "vertex_ai" + assert lyria_2["audio_seconds_per_prediction"] == 30 assert lyria_2["output_cost_per_second"] == 0.002 assert lyria_2["supported_modalities"] == ["text"] assert lyria_2["supported_output_modalities"] == ["audio"] From e00fe023a973860345f56278233398a3f8a5b4ff Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 15 Jul 2026 19:25:19 -0500 Subject: [PATCH 09/43] test(models): validate Lyria audio metadata --- tests/test_litellm/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6ae7c27b758..7bec649099a 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -944,6 +944,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "code_interpreter_cost_per_session": {"type": "number"}, "inference_geo": {"type": "string"}, "litellm_provider": {"type": "string"}, + "max_audio_length_hours": {"type": "number"}, + "max_audio_per_prompt": {"type": "number"}, "max_input_tokens": {"type": "number"}, "max_output_tokens": {"type": "number"}, "max_tokens": {"type": "number"}, From b96844dd0c23a084108191df7ff37423a40f862f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 15 Jul 2026 19:56:53 -0500 Subject: [PATCH 10/43] feat(vertex): expose Lyria through audio speech --- litellm/cost_calculator.py | 12 +- .../text_to_speech/transformation.py | 160 ++++++++++ litellm/main.py | 6 +- ...odel_prices_and_context_window_backup.json | 15 +- .../vertex_passthrough_logging_handler.py | 12 +- litellm/proxy/proxy_server.py | 11 +- litellm/types/utils.py | 4 + litellm/utils.py | 6 + model_prices_and_context_window.json | 15 +- ...test_vertex_passthrough_logging_handler.py | 33 ++ .../text_to_speech/test_transformation.py | 298 +++++++++++++++++- tests/test_litellm/test_cost_calculator.py | 18 ++ tests/test_litellm/test_utils.py | 15 +- 13 files changed, 565 insertions(+), 40 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b83e9b395a8..2bac5e234bc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -496,9 +496,19 @@ def cost_per_token( # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models if call_type == "speech" or call_type == "aspeech": speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) - cost_metric: Final = select_cost_metric_for_model(speech_model_info) prompt_cost: float = 0.0 completion_cost: float = 0.0 + if not speech_model_info.get("input_cost_per_character") and not speech_model_info.get( + "input_cost_per_token" + ): + output_cost_per_generation: Final = speech_model_info.get("output_cost_per_image") + output_cost_per_second: Final = speech_model_info.get("output_cost_per_second") + audio_seconds_per_prediction: Final = speech_model_info.get("audio_seconds_per_prediction") + if output_cost_per_generation is not None: + return prompt_cost, float(output_cost_per_generation) + if output_cost_per_second is not None and audio_seconds_per_prediction is not None: + return prompt_cost, float(output_cost_per_second) * float(audio_seconds_per_prediction) + cost_metric: Final = select_cost_metric_for_model(speech_model_info) if cost_metric == "cost_per_character": if prompt_characters is None: raise ValueError( diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 332f892ae6b..ad39aa35f8c 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, Final, Union import httpx +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( speech_media_type_from_audio_bytes, ) @@ -471,3 +473,161 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): # Initialize the HttpxBinaryResponseContent instance return HttpxBinaryResponseContent(response) + + +class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): + LYRIA_MODELS = { + "lyria-002", + "lyria-3-clip-preview", + "lyria-3-pro-preview", + } + + @classmethod + def is_lyria_model(cls, model: str) -> bool: + return model.removeprefix("vertex_ai/") in cls.LYRIA_MODELS + + def get_supported_openai_params(self, model: str) -> list: + return ["response_format"] + + def map_openai_params( + self, + model: str, + optional_params: dict, + voice: str | dict | None = None, + drop_params: bool = False, + kwargs: dict = {}, + ) -> tuple[str | None, dict]: + mapped_params = dict(optional_params) + base_model = model.removeprefix("vertex_ai/") + unsupported_params = [param for param in ("speed", "instructions") if mapped_params.get(param) is not None] + if unsupported_params: + if drop_params or litellm.drop_params: + for param in unsupported_params: + mapped_params.pop(param, None) + else: + raise UnsupportedParamsError( + status_code=400, + message=( + f"Vertex AI {base_model} does not support the OpenAI parameters: " + f"{', '.join(unsupported_params)}. To drop unsupported openai params " + "from the call, set `litellm.drop_params = True`" + ), + ) + response_format = mapped_params.get("response_format") + supported_formats = ( + {"wav"} if base_model == "lyria-002" else {"mp3", "wav"} if base_model == "lyria-3-pro-preview" else {"mp3"} + ) + if response_format is not None and response_format not in supported_formats: + if drop_params or litellm.drop_params: + mapped_params.pop("response_format", None) + else: + raise UnsupportedParamsError( + status_code=400, + message=( + f"Vertex AI {base_model} does not support response_format={response_format!r}. " + f"Supported values: {', '.join(sorted(supported_formats))}. " + "To drop unsupported openai params from the call, set `litellm.drop_params = True`" + ), + ) + return voice if isinstance(voice, str) else None, mapped_params + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, + ) -> str: + base_model = model.removeprefix("vertex_ai/") + project = self.safe_get_vertex_ai_project(litellm_params) + if project is None: + _, project = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=None, + custom_llm_provider="vertex_ai", + ) + if base_model.startswith("lyria-3-"): + from litellm.llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig, + ) + + return VertexAIInteractionsConfig().get_complete_url( + api_base=api_base, + model=base_model, + litellm_params={**litellm_params, "vertex_project": project}, + ) + location = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() + base_url = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") + return f"{base_url}/v1/projects/{project}/locations/{location}/publishers/google/models/{base_model}:predict" + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> TextToSpeechRequestData: + access_token, project = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=self.safe_get_vertex_ai_project(litellm_params), + custom_llm_provider="vertex_ai", + ) + headers.update( + { + "Authorization": f"Bearer {access_token}", + "x-goog-user-project": project, + "Content-Type": "application/json", + } + ) + base_model = model.removeprefix("vertex_ai/") + if base_model == "lyria-002": + request_body = { + "instances": [{"prompt": input}], + "parameters": {"sample_count": 1}, + } + else: + request_body = {"model": base_model, "input": input} + if optional_params.get("response_format") == "wav": + request_body["response_format"] = { + "type": "audio", + "mime_type": "audio/wav", + } + return TextToSpeechRequestData(dict_body=request_body, headers=headers) + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + from litellm.types.llms.openai import HttpxBinaryResponseContent + + response_json = raw_response.json() + base_model = model.removeprefix("vertex_ai/") + audio_data: str | None = None + mime_type: str | None = None + if base_model == "lyria-002": + predictions = response_json.get("predictions") or [] + if predictions: + audio_data = predictions[0].get("audioContent") or predictions[0].get("bytesBase64Encoded") + mime_type = predictions[0].get("mimeType") + else: + for step in response_json.get("steps") or response_json.get("outputs") or []: + content_items = step.get("content") or [] if step.get("type") == "model_output" else [step] + for content in content_items: + if content.get("type") == "audio" and content.get("data"): + audio_data = content["data"] + mime_type = content.get("mime_type") + if audio_data is None: + raise ValueError(f"No generated audio found in Vertex AI {base_model} response") + mime_type = mime_type or ("audio/wav" if base_model == "lyria-002" else "audio/mpeg") + response = HttpxBinaryResponseContent( + httpx.Response( + status_code=raw_response.status_code, + content=base64.b64decode(audio_data), + headers={"content-type": mime_type}, + ) + ) + response._hidden_params = {"audio_mime_type": mime_type} + return response diff --git a/litellm/main.py b/litellm/main.py index 0128e4defe5..55db92d44b3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8235,6 +8235,7 @@ def speech( ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) @@ -8259,7 +8260,10 @@ def speech( # Vertex AI Text-to-Speech (Google Cloud TTS) if text_to_speech_provider_config is None: - text_to_speech_provider_config = VertexAITextToSpeechConfig() + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): + text_to_speech_provider_config = VertexAILyriaTextToSpeechConfig() + else: + text_to_speech_provider_config = VertexAITextToSpeechConfig() # Cast to specific Vertex AI config type to access dispatch method vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1a1d92805e2..2955a0ffa4e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45516,9 +45516,12 @@ "litellm_provider": "vertex_ai", "max_audio_length_hours": 0.009111111111111111, "max_audio_per_prompt": 4, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_second": 0.002, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1/audio/speech" + ], "supported_modalities": [ "text" ], @@ -45533,12 +45536,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", @@ -45566,12 +45570,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 6b2d7763fa0..ab3b24f470f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -368,14 +368,8 @@ class VertexPassthroughLoggingHandler: @staticmethod def _is_audio_predict_response(model: str, json_response: dict) -> bool: return ( - VertexPassthroughLoggingHandler._get_audio_prediction_count( - json_response=json_response - ) - > 0 - and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost( - model=model - ) - is not None + VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0 + and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) is not None ) @staticmethod @@ -397,7 +391,7 @@ class VertexPassthroughLoggingHandler: return sum( 1 for prediction in predictions - if isinstance(prediction, dict) and prediction.get("audioContent") + if isinstance(prediction, dict) and (prediction.get("audioContent") or prediction.get("bytesBase64Encoded")) ) @staticmethod diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 27132c90e05..f11cbc92224 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11176,9 +11176,14 @@ async def audio_speech( upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - media_type: Final = resolve_speech_media_type( - upstream_content_type=upstream_content_type, - response_format=requested_format if isinstance(requested_format, str) else None, + hidden_audio_mime_type: Final = hidden_params.get("audio_mime_type") + media_type: Final = ( + hidden_audio_mime_type + if isinstance(hidden_audio_mime_type, str) + else resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=requested_format if isinstance(requested_format, str) else None, + ) ) return StreamingResponse( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ee6f09e05dc..d2a09639362 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -310,6 +310,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_video_per_second: float | None # only for vertex ai models output_cost_per_audio_per_second: float | None # only for vertex ai models output_cost_per_second: float | None # for OpenAI Speech models + audio_seconds_per_prediction: float | None + max_audio_length_hours: float | None + max_audio_per_prompt: int | None output_cost_per_second_1080p: ( float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) @@ -333,6 +336,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "image_generation", "chat", "audio_transcription", + "audio_speech", "responses", "ocr", "realtime", diff --git a/litellm/utils.py b/litellm/utils.py index ba456fc353b..7c8974906ed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5880,6 +5880,9 @@ def _get_model_info_helper( "output_cost_per_token_above_512k_tokens", None ), output_cost_per_second=_model_info.get("output_cost_per_second", None), + audio_seconds_per_prediction=_model_info.get("audio_seconds_per_prediction", None), + max_audio_length_hours=_model_info.get("max_audio_length_hours", None), + max_audio_per_prompt=_model_info.get("max_audio_per_prompt", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), @@ -9415,9 +9418,12 @@ class ProviderConfigManager: # mapping would drop response_format before the bridge sees it (LIT-6501) return None from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): + return VertexAILyriaTextToSpeechConfig() return VertexAITextToSpeechConfig() elif litellm.LlmProviders.MINIMAX == provider: from litellm.llms.minimax.text_to_speech.transformation import ( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1a1d92805e2..2955a0ffa4e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45516,9 +45516,12 @@ "litellm_provider": "vertex_ai", "max_audio_length_hours": 0.009111111111111111, "max_audio_per_prompt": 4, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_second": 0.002, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1/audio/speech" + ], "supported_modalities": [ "text" ], @@ -45533,12 +45536,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", @@ -45566,12 +45570,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 9f65fa09b1b..ccce19f1634 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -110,3 +110,36 @@ def test_audio_predict_response_uses_model_map_metadata( assert result["kwargs"]["model"] == "music-audio-preview" assert result["kwargs"]["response_cost"] == pytest.approx(6.0) assert logging_obj.model_call_details["response_cost"] == pytest.approx(6.0) + + +def test_audio_predict_response_supports_bytes_base64_encoded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + { + "audio_seconds_per_prediction": 30, + "output_cost_per_second": 0.002, + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={"predictions": [{"bytesBase64Encoded": "clip"}]}, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["kwargs"]["response_cost"] == pytest.approx(0.06) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index fba337b5f2c..910bc977a79 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -4,11 +4,13 @@ from unittest.mock import MagicMock, Mock, patch import httpx import pytest - import litellm from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager class TestVertexAITextToSpeechConfig: @@ -41,9 +43,7 @@ class TestVertexAITextToSpeechConfig: @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") - def test_transform_text_to_speech_request_body( - self, mock_get_token, mock_ensure_token - ): + def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token): """Test that transform_text_to_speech_request generates correct request body""" # Mock authentication mock_ensure_token.return_value = ("mock-token", "test-project") @@ -104,9 +104,7 @@ class TestVertexAITextToSpeechConfig: config = VertexAITextToSpeechConfig() # Test with a Chirp3 HD voice - voice_str, voice_dict = config._map_voice_to_vertex_format( - "en-US-Chirp3-HD-Charon" - ) + voice_str, voice_dict = config._map_voice_to_vertex_format("en-US-Chirp3-HD-Charon") assert voice_str == "en-US-Chirp3-HD-Charon" assert voice_dict is not None @@ -169,6 +167,284 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm +class TestVertexAILyriaTextToSpeechConfig: + @pytest.mark.parametrize( + "model", + ["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"], + ) + def test_provider_config_manager_selects_lyria_config(self, model): + config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAILyriaTextToSpeechConfig) + + def test_get_complete_url_for_lyria_2(self): + config = VertexAILyriaTextToSpeechConfig() + + url = config.get_complete_url( + model="lyria-002", + api_base=None, + litellm_params={ + "vertex_project": "music-project", + "vertex_location": "europe-west4", + }, + ) + + assert url == ( + "https://europe-west4-aiplatform.googleapis.com/v1/projects/music-project/" + "locations/europe-west4/publishers/google/models/lyria-002:predict" + ) + + def test_get_complete_url_for_lyria_3(self): + config = VertexAILyriaTextToSpeechConfig() + + url = config.get_complete_url( + model="lyria-3-pro-preview", + api_base=None, + litellm_params={"vertex_project": "music-project"}, + ) + + assert url == ("https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions") + + @pytest.mark.parametrize( + ("model", "response_format", "expected_body"), + [ + ( + "lyria-002", + "wav", + { + "instances": [{"prompt": "A bright synth track"}], + "parameters": {"sample_count": 1}, + }, + ), + ( + "lyria-3-clip-preview", + "mp3", + { + "model": "lyria-3-clip-preview", + "input": "A bright synth track", + }, + ), + ( + "lyria-3-pro-preview", + "wav", + { + "model": "lyria-3-pro-preview", + "input": "A bright synth track", + "response_format": { + "type": "audio", + "mime_type": "audio/wav", + }, + }, + ), + ], + ) + @patch.object(VertexAILyriaTextToSpeechConfig, "_ensure_access_token") + def test_transform_request( + self, + mock_ensure_token, + model, + response_format, + expected_body, + ): + mock_ensure_token.return_value = ("mock-token", "music-project") + config = VertexAILyriaTextToSpeechConfig() + + request = config.transform_text_to_speech_request( + model=model, + input="A bright synth track", + voice="alloy", + optional_params={"response_format": response_format}, + litellm_params={"vertex_project": "music-project"}, + headers={}, + ) + + assert request["dict_body"] == expected_body + assert request["headers"]["Authorization"] == "Bearer mock-token" + assert request["headers"]["x-goog-user-project"] == "music-project" + + @pytest.mark.parametrize( + ("model", "response_json", "expected_audio", "expected_mime_type"), + [ + ( + "lyria-002", + { + "predictions": [ + { + "bytesBase64Encoded": "bHlyaWEtMi1hdWRpbw==", + } + ] + }, + b"lyria-2-audio", + "audio/wav", + ), + ( + "lyria-3-pro-preview", + { + "steps": [ + { + "type": "model_output", + "content": [ + {"type": "text", "text": "Generated lyrics"}, + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + }, + ], + } + ] + }, + b"lyria-3-audio", + "audio/mpeg", + ), + ( + "lyria-3-clip-preview", + { + "outputs": [ + {"type": "text", "text": "Generated lyrics"}, + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + }, + ] + }, + b"lyria-3-audio", + "audio/mpeg", + ), + ], + ) + def test_transform_response( + self, + model, + response_json, + expected_audio, + expected_mime_type, + ): + config = VertexAILyriaTextToSpeechConfig() + raw_response = httpx.Response(200, json=response_json) + + response = config.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert response.content == expected_audio + assert response._hidden_params["audio_mime_type"] == expected_mime_type + + @pytest.mark.parametrize( + ("model", "response_format"), + [ + ("lyria-002", "mp3"), + ("lyria-3-clip-preview", "wav"), + ("lyria-3-pro-preview", "opus"), + ], + ) + def test_rejects_unsupported_response_format(self, model, response_format): + config = VertexAILyriaTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError): + config.map_openai_params( + model=model, + optional_params={"response_format": response_format}, + ) + + @pytest.mark.parametrize("param", ["speed", "instructions"]) + def test_rejects_unsupported_openai_params(self, param): + config = VertexAILyriaTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError): + config.map_openai_params( + model="lyria-3-pro-preview", + optional_params={param: "unsupported"}, + ) + + @pytest.mark.parametrize( + ("model", "response_format", "response_json", "expected_url", "expected_body"), + [ + ( + "lyria-002", + "wav", + { + "predictions": [ + { + "audioContent": "bHlyaWEtMi1hdWRpbw==", + "mimeType": "audio/wav", + } + ] + }, + "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/us-central1/publishers/google/models/lyria-002:predict", + { + "instances": [{"prompt": "A bright synth track"}], + "parameters": {"sample_count": 1}, + }, + ), + ( + "lyria-3-pro-preview", + "mp3", + { + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + } + ], + } + ] + }, + "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions", + { + "model": "lyria-3-pro-preview", + "input": "A bright synth track", + }, + ), + ], + ) + def test_litellm_speech_dispatches_to_lyria_api( + self, + model, + response_format, + response_json, + expected_url, + expected_body, + ): + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = response_json + with ( + patch.object( + VertexAILyriaTextToSpeechConfig, + "_ensure_access_token", + return_value=("mock-token", "music-project"), + ), + patch( + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post", + return_value=mock_response, + ) as mock_post, + ): + response = litellm.speech( + model=f"vertex_ai/{model}", + input="A bright synth track", + voice="alloy", + response_format=response_format, + vertex_project="music-project", + vertex_location="us-central1", + ) + + assert response.content in {b"lyria-2-audio", b"lyria-3-audio"} + mock_post.assert_called_once() + assert mock_post.call_args.kwargs["url"] == expected_url + assert mock_post.call_args.kwargs["json"] == expected_body + + @patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") @@ -182,9 +458,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ # Mock HTTP response mock_response = Mock(spec=httpx.Response) - mock_response.content = ( - b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World" - ) + mock_response.content = b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World" mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="} @@ -203,9 +477,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ call_kwargs = mock_post.call_args.kwargs # Verify the URL is the Google Cloud TTS API - assert ( - call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" - ) + assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" # Verify request body structure assert "json" in call_kwargs diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7c2174018e8..ffc71c76d03 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -146,6 +146,24 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 +@pytest.mark.parametrize( + ("model", "expected_cost"), + [ + ("vertex_ai/lyria-002", 0.06), + ("vertex_ai/lyria-3-clip-preview", 0.04), + ("vertex_ai/lyria-3-pro-preview", 0.08), + ], +) +def test_vertex_lyria_speech_cost(model, expected_cost, _local_model_cost_map): + cost = completion_cost( + model=model, + prompt="A bright synth track", + call_type="speech", + ) + + assert cost == pytest.approx(expected_cost) + + def test_baseten_model_api_pricing_entries(_local_model_cost_map): expected_pricing = { diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 7bec649099a..1b0b27032ce 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2855,15 +2855,25 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert lyria_2["litellm_provider"] == "vertex_ai" assert clip["litellm_provider"] == "vertex_ai" assert pro["litellm_provider"] == "vertex_ai" + assert lyria_2["mode"] == "audio_speech" + assert clip["mode"] == "audio_speech" + assert pro["mode"] == "audio_speech" assert lyria_2["audio_seconds_per_prediction"] == 30 assert lyria_2["output_cost_per_second"] == 0.002 assert lyria_2["supported_modalities"] == ["text"] assert lyria_2["supported_output_modalities"] == ["audio"] assert lyria_2["supports_audio_output"] is True + assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"] assert clip["output_cost_per_image"] == 0.04 assert pro["output_cost_per_image"] == 0.08 - assert clip["supported_endpoints"] == ["/v1beta/interactions"] - assert pro["supported_endpoints"] == ["/v1beta/interactions"] + assert clip["supported_endpoints"] == [ + "/v1beta/interactions", + "/v1/audio/speech", + ] + assert pro["supported_endpoints"] == [ + "/v1beta/interactions", + "/v1/audio/speech", + ] assert clip["supported_modalities"] == ["text", "image"] assert pro["supported_modalities"] == ["text", "image"] assert clip["supported_regions"] == ["global"] @@ -2873,7 +2883,6 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert clip["supports_image_input"] is True assert pro["supports_image_input"] is True - def test_model_info_for_fireworks_short_form_models(): """ Test that fireworks_ai short-form model entries (fireworks_ai/) From f18cb0cdb48ce0338a30af940d557842eee4fb19 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 15 Jul 2026 20:28:00 -0500 Subject: [PATCH 11/43] fix(vertex): make Lyria routing and billing data-driven --- litellm/llms/vertex_ai/common_utils.py | 35 +++++++++++ .../text_to_speech/transformation.py | 36 ++++++----- ...odel_prices_and_context_window_backup.json | 19 +++++- litellm/types/utils.py | 2 + litellm/utils.py | 2 + model_prices_and_context_window.json | 19 +++++- .../text_to_speech/test_transformation.py | 62 +++++++++++++++++++ tests/test_litellm/test_utils.py | 17 +++++ 8 files changed, 172 insertions(+), 20 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index a36c920dda0..8a4c1e68623 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1,9 +1,12 @@ import re from copy import deepcopy from enum import Enum +from functools import lru_cache from typing import Any, Final, Literal, cast, get_type_hints import httpx +from pydantic import TypeAdapter, ValidationError +from typing_extensions import NotRequired, TypedDict import litellm from litellm._logging import verbose_logger @@ -21,6 +24,38 @@ from litellm.types.utils import TokenCountResponse from litellm.utils import supports_response_schema, supports_system_messages +class VertexAILyriaModelInfo(TypedDict): + vertex_ai_audio_api: Literal["lyria_predict", "lyria_interactions"] + supported_audio_formats: tuple[Literal["mp3", "wav"], ...] + output_cost_per_image: NotRequired[float] + + +_VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER = TypeAdapter(VertexAILyriaModelInfo) + + +def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyriaModelInfo | None: + if raw_model_info is None: + return None + try: + return _VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER.validate_python(raw_model_info) + except ValidationError: + return None + + +@lru_cache(maxsize=32) +def _get_bundled_vertex_ai_lyria_model_info(model_key: str) -> VertexAILyriaModelInfo | None: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + bundled_model_info = GetModelCostMap.load_local_model_cost_map().get(model_key) + return _validate_vertex_ai_lyria_model_info(bundled_model_info) + + +def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: + model_key = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + runtime_model_info = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) + return runtime_model_info or _get_bundled_vertex_ai_lyria_model_info(model_key) + + class VertexAIError(BaseLLMException): def __init__( self, diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index ad39aa35f8c..d56f151a8c2 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -21,6 +21,10 @@ from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, ) +from litellm.llms.vertex_ai.common_utils import ( + VertexAILyriaModelInfo, + get_vertex_ai_lyria_model_info, +) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from litellm.types.llms.vertex_ai_text_to_speech import ( @@ -476,15 +480,16 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): - LYRIA_MODELS = { - "lyria-002", - "lyria-3-clip-preview", - "lyria-3-pro-preview", - } - @classmethod def is_lyria_model(cls, model: str) -> bool: - return model.removeprefix("vertex_ai/") in cls.LYRIA_MODELS + return get_vertex_ai_lyria_model_info(model=model) is not None + + @staticmethod + def _get_model_info(model: str) -> VertexAILyriaModelInfo: + model_info = get_vertex_ai_lyria_model_info(model=model) + if model_info is None: + raise ValueError(f"Vertex AI model {model!r} does not declare a Lyria audio API") + return model_info def get_supported_openai_params(self, model: str) -> list: return ["response_format"] @@ -499,6 +504,7 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): ) -> tuple[str | None, dict]: mapped_params = dict(optional_params) base_model = model.removeprefix("vertex_ai/") + model_info = self._get_model_info(model=model) unsupported_params = [param for param in ("speed", "instructions") if mapped_params.get(param) is not None] if unsupported_params: if drop_params or litellm.drop_params: @@ -514,9 +520,7 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): ), ) response_format = mapped_params.get("response_format") - supported_formats = ( - {"wav"} if base_model == "lyria-002" else {"mp3", "wav"} if base_model == "lyria-3-pro-preview" else {"mp3"} - ) + supported_formats = frozenset(model_info["supported_audio_formats"]) if response_format is not None and response_format not in supported_formats: if drop_params or litellm.drop_params: mapped_params.pop("response_format", None) @@ -538,6 +542,7 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): litellm_params: dict, ) -> str: base_model = model.removeprefix("vertex_ai/") + model_info = self._get_model_info(model=model) project = self.safe_get_vertex_ai_project(litellm_params) if project is None: _, project = self._ensure_access_token( @@ -545,7 +550,7 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): project_id=None, custom_llm_provider="vertex_ai", ) - if base_model.startswith("lyria-3-"): + if model_info["vertex_ai_audio_api"] == "lyria_interactions": from litellm.llms.vertex_ai.interactions.transformation import ( VertexAIInteractionsConfig, ) @@ -581,7 +586,8 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): } ) base_model = model.removeprefix("vertex_ai/") - if base_model == "lyria-002": + model_info = self._get_model_info(model=model) + if model_info["vertex_ai_audio_api"] == "lyria_predict": request_body = { "instances": [{"prompt": input}], "parameters": {"sample_count": 1}, @@ -605,9 +611,10 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): response_json = raw_response.json() base_model = model.removeprefix("vertex_ai/") + model_info = self._get_model_info(model=model) audio_data: str | None = None mime_type: str | None = None - if base_model == "lyria-002": + if model_info["vertex_ai_audio_api"] == "lyria_predict": predictions = response_json.get("predictions") or [] if predictions: audio_data = predictions[0].get("audioContent") or predictions[0].get("bytesBase64Encoded") @@ -621,7 +628,8 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): mime_type = content.get("mime_type") if audio_data is None: raise ValueError(f"No generated audio found in Vertex AI {base_model} response") - mime_type = mime_type or ("audio/wav" if base_model == "lyria-002" else "audio/mpeg") + default_format = model_info["supported_audio_formats"][0] + mime_type = mime_type or {"mp3": "audio/mpeg", "wav": "audio/wav"}[default_format] response = HttpxBinaryResponseContent( httpx.Response( status_code=raw_response.status_code, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2955a0ffa4e..9986d5056d1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45519,6 +45519,9 @@ "mode": "audio_speech", "output_cost_per_second": 0.002, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "wav" + ], "supported_endpoints": [ "/v1/audio/speech" ], @@ -45528,7 +45531,8 @@ "supported_output_modalities": [ "audio" ], - "supports_audio_output": true + "supports_audio_output": true, + "vertex_ai_audio_api": "lyria_predict" }, "vertex_ai/lyria-3-clip-preview": { "input_cost_per_token": 0, @@ -45540,6 +45544,9 @@ "output_cost_per_image": 0.04, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3" + ], "supported_endpoints": [ "/v1beta/interactions", "/v1/audio/speech" @@ -45562,7 +45569,8 @@ "supports_response_schema": false, "supports_system_messages": false, "supports_vision": true, - "supports_web_search": false + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" }, "vertex_ai/lyria-3-pro-preview": { "input_cost_per_token": 0, @@ -45574,6 +45582,10 @@ "output_cost_per_image": 0.08, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3", + "wav" + ], "supported_endpoints": [ "/v1beta/interactions", "/v1/audio/speech" @@ -45596,7 +45608,8 @@ "supports_response_schema": false, "supports_system_messages": false, "supports_vision": true, - "supports_web_search": false + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" }, "vertex_ai/meta/llama-3.1-405b-instruct-maas": { "input_cost_per_token": 5e-06, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d2a09639362..bb0485be7c1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -168,6 +168,8 @@ class ProviderSpecificModelInfo(TypedDict, total=False): default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None] supports_output_config: bool | None supports_image_size: bool | None + supported_audio_formats: list[Literal["mp3", "wav"]] | None + vertex_ai_audio_api: Literal["lyria_predict", "lyria_interactions"] | None bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None bedrock_converse_supports_strict_tools: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index 7c8974906ed..d6a68b3ce6e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5940,6 +5940,8 @@ def _get_model_info_helper( provider_specific_entry=_model_info.get("provider_specific_entry", None), uses_embed_content=_model_info.get("uses_embed_content", None), supports_image_size=_model_info.get("supports_image_size", None), + supported_audio_formats=_model_info.get("supported_audio_formats", None), + vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None), ) for cost_key, cost_value in _model_info.items(): if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2955a0ffa4e..9986d5056d1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45519,6 +45519,9 @@ "mode": "audio_speech", "output_cost_per_second": 0.002, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "wav" + ], "supported_endpoints": [ "/v1/audio/speech" ], @@ -45528,7 +45531,8 @@ "supported_output_modalities": [ "audio" ], - "supports_audio_output": true + "supports_audio_output": true, + "vertex_ai_audio_api": "lyria_predict" }, "vertex_ai/lyria-3-clip-preview": { "input_cost_per_token": 0, @@ -45540,6 +45544,9 @@ "output_cost_per_image": 0.04, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3" + ], "supported_endpoints": [ "/v1beta/interactions", "/v1/audio/speech" @@ -45562,7 +45569,8 @@ "supports_response_schema": false, "supports_system_messages": false, "supports_vision": true, - "supports_web_search": false + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" }, "vertex_ai/lyria-3-pro-preview": { "input_cost_per_token": 0, @@ -45574,6 +45582,10 @@ "output_cost_per_image": 0.08, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3", + "wav" + ], "supported_endpoints": [ "/v1beta/interactions", "/v1/audio/speech" @@ -45596,7 +45608,8 @@ "supports_response_schema": false, "supports_system_messages": false, "supports_vision": true, - "supports_web_search": false + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" }, "vertex_ai/meta/llama-3.1-405b-instruct-maas": { "input_cost_per_token": 5e-06, diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 910bc977a79..a1bb203e67e 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -180,6 +180,68 @@ class TestVertexAILyriaTextToSpeechConfig: assert isinstance(config, VertexAILyriaTextToSpeechConfig) + @pytest.mark.parametrize( + ("model", "vertex_ai_audio_api", "supported_audio_formats", "expected_url"), + [ + ( + "future-lyria-predict", + "lyria_predict", + ["wav"], + "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/" + "us-central1/publishers/google/models/future-lyria-predict:predict", + ), + ( + "future-music-interactions", + "lyria_interactions", + ["mp3", "wav"], + "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions", + ), + ], + ) + def test_dispatches_from_model_metadata( + self, + monkeypatch, + model, + vertex_ai_audio_api, + supported_audio_formats, + expected_url, + ): + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model}", + { + "vertex_ai_audio_api": vertex_ai_audio_api, + "supported_audio_formats": supported_audio_formats, + }, + ) + + config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAILyriaTextToSpeechConfig) + assert ( + config.get_complete_url( + model=model, + api_base=None, + litellm_params={ + "vertex_project": "music-project", + "vertex_location": "us-central1", + }, + ) + == expected_url + ) + + def test_vertex_chirp_does_not_select_lyria_config(self): + config = ProviderConfigManager.get_provider_text_to_speech_config( + model="chirp", + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAITextToSpeechConfig) + assert not isinstance(config, VertexAILyriaTextToSpeechConfig) + def test_get_complete_url_for_lyria_2(self): config = VertexAILyriaTextToSpeechConfig() diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 1b0b27032ce..a179a82fcb2 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1048,6 +1048,17 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, "supports_speed": {"type": "boolean"}, + "supported_audio_formats": { + "type": "array", + "items": { + "type": "string", + "enum": ["mp3", "wav"], + }, + }, + "vertex_ai_audio_api": { + "type": "string", + "enum": ["lyria_predict", "lyria_interactions"], + }, "bedrock_output_config_effort_ceiling": { "type": "string", "enum": ["low", "medium", "high", "max", "xhigh"], @@ -2863,9 +2874,15 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert lyria_2["supported_modalities"] == ["text"] assert lyria_2["supported_output_modalities"] == ["audio"] assert lyria_2["supports_audio_output"] is True + assert lyria_2["supported_audio_formats"] == ["wav"] + assert lyria_2["vertex_ai_audio_api"] == "lyria_predict" assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"] assert clip["output_cost_per_image"] == 0.04 assert pro["output_cost_per_image"] == 0.08 + assert clip["supported_audio_formats"] == ["mp3"] + assert pro["supported_audio_formats"] == ["mp3", "wav"] + assert clip["vertex_ai_audio_api"] == "lyria_interactions" + assert pro["vertex_ai_audio_api"] == "lyria_interactions" assert clip["supported_endpoints"] == [ "/v1beta/interactions", "/v1/audio/speech", From e9e2fbb4385e412273ce03c6e0038d013316ce8a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 12 Aug 2026 12:31:44 -0500 Subject: [PATCH 12/43] style(vertex-ai): modernize Lyria tests --- .../llms/vertex_ai/test_vertex_passthrough_logging_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index ccce19f1634..1e8d569d3d1 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -2,9 +2,9 @@ from datetime import datetime from unittest.mock import MagicMock import httpx -import litellm import pytest +import litellm from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) From bd5123564c66d1eb68c253ae6d2405c1e383104c Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 12 Aug 2026 12:56:06 -0500 Subject: [PATCH 13/43] fix(vertex-ai): classify Lyria model metadata --- ci_cd/generate_model_prices_schema.py | 21 ++++++++++++ .../text_to_speech/transformation.py | 2 +- .../vertex_passthrough_logging_handler.py | 3 +- model_prices_and_context_window.schema.json | 33 +++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 57cc742d5c4..ee0dad25c81 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -58,6 +58,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = { } ARRAY_KEYS: dict[str, JsonSchema] = { + "supported_audio_formats": { + "type": "array", + "description": "Audio container formats the model can return.", + "items": {"type": "string", "enum": ["mp3", "wav"]}, + }, "supported_endpoints": { "type": "array", "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", @@ -116,6 +121,10 @@ ARRAY_KEYS: dict[str, JsonSchema] = { } INTEGER_KEYS: dict[str, JsonSchema] = { + "max_audio_per_prompt": { + **NONNEG_INTEGER, + "description": "Maximum number of audio outputs accepted or generated per prompt.", + }, "max_tokens": { **NONNEG_INTEGER, "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.", @@ -141,6 +150,14 @@ INTEGER_KEYS: dict[str, JsonSchema] = { } NUMBER_KEYS: dict[str, JsonSchema] = { + "audio_seconds_per_prediction": { + **NONNEG_NUMBER, + "description": "Audio duration, in seconds, produced by one prediction.", + }, + "max_audio_length_hours": { + **NONNEG_NUMBER, + "description": "Maximum generated audio duration, expressed in hours.", + }, "regional_processing_uplift_multiplier_eu": { "type": "number", "minimum": 1, @@ -231,6 +248,10 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: }, "comment": STRING, "audio_transcription_config": STRING, + "vertex_ai_audio_api": { + "type": "string", + "enum": ["lyria_predict", "lyria_interactions"], + }, } diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index d56f151a8c2..373f6c28f9e 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -500,7 +500,7 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): optional_params: dict, voice: str | dict | None = None, drop_params: bool = False, - kwargs: dict = {}, + kwargs: dict | None = None, ) -> tuple[str | None, dict]: mapped_params = dict(optional_params) base_model = model.removeprefix("vertex_ai/") diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index ab3b24f470f..4d348b51055 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -343,8 +343,7 @@ class VertexPassthroughLoggingHandler: json_response=json_response ) response_cost: Final = ( - VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) - or 0.0 + VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) or 0.0 ) * prediction_count logging_obj.model = model diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 9e370e5406a..58ca91f3977 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -53,6 +53,11 @@ "type": "number", "minimum": 0 }, + "audio_seconds_per_prediction": { + "type": "number", + "minimum": 0, + "description": "Audio duration, in seconds, produced by one prediction." + }, "audio_transcription_config": { "type": "string" }, @@ -363,6 +368,16 @@ "type": "string", "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers." }, + "max_audio_length_hours": { + "type": "number", + "minimum": 0, + "description": "Maximum generated audio duration, expressed in hours." + }, + "max_audio_per_prompt": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of audio outputs accepted or generated per prompt." + }, "max_input_tokens": { "type": "integer", "minimum": 0, @@ -603,6 +618,17 @@ "type": "string", "description": "URL of the provider pricing/model page this entry was taken from." }, + "supported_audio_formats": { + "type": "array", + "description": "Audio container formats the model can return.", + "items": { + "type": "string", + "enum": [ + "mp3", + "wav" + ] + } + }, "supported_endpoints": { "type": "array", "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", @@ -826,6 +852,13 @@ "uses_embed_content": { "type": "boolean" }, + "vertex_ai_audio_api": { + "type": "string", + "enum": [ + "lyria_predict", + "lyria_interactions" + ] + }, "web_search_billing_unit": { "type": "string", "description": "Whether web search is billed per query or per prompt.", From 6ec53f284617cc52adfb78f5c8e0b03b5a3c125e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 12 Aug 2026 13:33:33 -0500 Subject: [PATCH 14/43] style(vertex-ai): satisfy Lyria quality gates --- litellm/cost_calculator.py | 10 +- litellm/llms/vertex_ai/common_utils.py | 16 +- .../llms/vertex_ai/interactions/__init__.py | 2 +- .../text_to_speech/transformation.py | 147 +++++++++++------- litellm/main.py | 8 +- .../vertex_passthrough_logging_handler.py | 49 ++++-- litellm/types/utils.py | 10 +- 7 files changed, 153 insertions(+), 89 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 2bac5e234bc..5dcbb1d5f37 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -498,16 +498,14 @@ def cost_per_token( speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) prompt_cost: float = 0.0 completion_cost: float = 0.0 - if not speech_model_info.get("input_cost_per_character") and not speech_model_info.get( - "input_cost_per_token" - ): + if not speech_model_info.get("input_cost_per_character") and not speech_model_info.get("input_cost_per_token"): output_cost_per_generation: Final = speech_model_info.get("output_cost_per_image") - output_cost_per_second: Final = speech_model_info.get("output_cost_per_second") + speech_output_cost_per_second: Final = speech_model_info.get("output_cost_per_second") audio_seconds_per_prediction: Final = speech_model_info.get("audio_seconds_per_prediction") if output_cost_per_generation is not None: return prompt_cost, float(output_cost_per_generation) - if output_cost_per_second is not None and audio_seconds_per_prediction is not None: - return prompt_cost, float(output_cost_per_second) * float(audio_seconds_per_prediction) + if speech_output_cost_per_second is not None and audio_seconds_per_prediction is not None: + return prompt_cost, float(speech_output_cost_per_second) * float(audio_seconds_per_prediction) cost_metric: Final = select_cost_metric_for_model(speech_model_info) if cost_metric == "cost_per_character": if prompt_characters is None: diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 8a4c1e68623..8885d19c1c0 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -6,7 +6,7 @@ from typing import Any, Final, Literal, cast, get_type_hints import httpx from pydantic import TypeAdapter, ValidationError -from typing_extensions import NotRequired, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -25,12 +25,12 @@ from litellm.utils import supports_response_schema, supports_system_messages class VertexAILyriaModelInfo(TypedDict): - vertex_ai_audio_api: Literal["lyria_predict", "lyria_interactions"] - supported_audio_formats: tuple[Literal["mp3", "wav"], ...] - output_cost_per_image: NotRequired[float] + vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"]] + supported_audio_formats: ReadOnly[tuple[Literal["mp3", "wav"], ...]] + output_cost_per_image: NotRequired[ReadOnly[float]] -_VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER = TypeAdapter(VertexAILyriaModelInfo) +_VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER: Final = TypeAdapter(VertexAILyriaModelInfo) def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyriaModelInfo | None: @@ -46,13 +46,13 @@ def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyri def _get_bundled_vertex_ai_lyria_model_info(model_key: str) -> VertexAILyriaModelInfo | None: from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - bundled_model_info = GetModelCostMap.load_local_model_cost_map().get(model_key) + bundled_model_info: Final = GetModelCostMap.load_local_model_cost_map().get(model_key) return _validate_vertex_ai_lyria_model_info(bundled_model_info) def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: - model_key = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" - runtime_model_info = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) + model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) return runtime_model_info or _get_bundled_vertex_ai_lyria_model_info(model_key) diff --git a/litellm/llms/vertex_ai/interactions/__init__.py b/litellm/llms/vertex_ai/interactions/__init__.py index 3e2309d21ef..f6f86f65e87 100644 --- a/litellm/llms/vertex_ai/interactions/__init__.py +++ b/litellm/llms/vertex_ai/interactions/__init__.py @@ -1,3 +1,3 @@ from .transformation import VertexAIInteractionsConfig -__all__ = ["VertexAIInteractionsConfig"] +__all__ = ("VertexAIInteractionsConfig",) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 373f6c28f9e..81f1cbd5157 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -8,7 +8,7 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s import base64 from collections.abc import Coroutine from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Union +from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union import httpx @@ -40,6 +40,10 @@ else: LiteLLMLoggingObj = Any HttpxBinaryResponseContent = Any +_LyriaVoice: TypeAlias = ( + str | dict | None +) # mutable-ok: inherited interface supports structured provider voice dictionaries + class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): """ @@ -486,26 +490,34 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): @staticmethod def _get_model_info(model: str) -> VertexAILyriaModelInfo: - model_info = get_vertex_ai_lyria_model_info(model=model) + model_info: Final = get_vertex_ai_lyria_model_info(model=model) if model_info is None: raise ValueError(f"Vertex AI model {model!r} does not declare a Lyria audio API") return model_info - def get_supported_openai_params(self, model: str) -> list: - return ["response_format"] + def get_supported_openai_params( + self, model: str + ) -> list: # mutable-ok: inherited provider interface returns a concrete parameter list + return [ # mutable-ok: inherited provider interface requires a concrete parameter list + "response_format" + ] def map_openai_params( self, model: str, - optional_params: dict, - voice: str | dict | None = None, + optional_params: dict, # mutable-ok: inherited provider interface accepts a concrete parameter dictionary + voice: _LyriaVoice = None, drop_params: bool = False, - kwargs: dict | None = None, - ) -> tuple[str | None, dict]: - mapped_params = dict(optional_params) - base_model = model.removeprefix("vertex_ai/") - model_info = self._get_model_info(model=model) - unsupported_params = [param for param in ("speed", "instructions") if mapped_params.get(param) is not None] + kwargs: dict | None = None, # mutable-ok: inherited provider interface accepts a concrete keyword dictionary + ) -> tuple[str | None, dict]: # mutable-ok: inherited provider interface returns concrete mapped parameters + mapped_params: Final = dict( # mutable-ok: mapping drops unsupported parameters before provider dispatch + optional_params + ) + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + unsupported_params: Final = tuple( + param for param in ("speed", "instructions") if mapped_params.get(param) is not None + ) if unsupported_params: if drop_params or litellm.drop_params: for param in unsupported_params: @@ -519,8 +531,8 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): "from the call, set `litellm.drop_params = True`" ), ) - response_format = mapped_params.get("response_format") - supported_formats = frozenset(model_info["supported_audio_formats"]) + response_format: Final = mapped_params.get("response_format") + supported_formats: Final = frozenset(model_info["supported_audio_formats"]) if response_format is not None and response_format not in supported_formats: if drop_params or litellm.drop_params: mapped_params.pop("response_format", None) @@ -539,17 +551,20 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): self, model: str, api_base: str | None, - litellm_params: dict, + litellm_params: dict, # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters ) -> str: - base_model = model.removeprefix("vertex_ai/") - model_info = self._get_model_info(model=model) - project = self.safe_get_vertex_ai_project(litellm_params) - if project is None: - _, project = self._ensure_access_token( + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + configured_project: Final = self.safe_get_vertex_ai_project(litellm_params) + project: Final = ( + self._ensure_access_token( credentials=self.safe_get_vertex_ai_credentials(litellm_params), project_id=None, custom_llm_provider="vertex_ai", - ) + )[1] + if configured_project is None + else configured_project + ) if model_info["vertex_ai_audio_api"] == "lyria_interactions": from litellm.llms.vertex_ai.interactions.transformation import ( VertexAIInteractionsConfig, @@ -558,10 +573,13 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): return VertexAIInteractionsConfig().get_complete_url( api_base=api_base, model=base_model, - litellm_params={**litellm_params, "vertex_project": project}, + litellm_params={ # mutable-ok: interactions dispatch expects a concrete parameter dictionary + **litellm_params, + "vertex_project": project, + }, ) - location = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() - base_url = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") + location: Final = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() + base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") return f"{base_url}/v1/projects/{project}/locations/{location}/publishers/google/models/{base_model}:predict" def transform_text_to_speech_request( @@ -569,9 +587,9 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): model: str, input: str, voice: str | None, - optional_params: dict, - litellm_params: dict, - headers: dict, + optional_params: dict, # mutable-ok: inherited provider interface accepts concrete mapped parameters + litellm_params: dict, # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters + headers: dict, # mutable-ok: inherited provider interface accepts and updates concrete HTTP headers ) -> TextToSpeechRequestData: access_token, project = self._ensure_access_token( credentials=self.safe_get_vertex_ai_credentials(litellm_params), @@ -579,23 +597,32 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): custom_llm_provider="vertex_ai", ) headers.update( - { + { # mutable-ok: HTTP dispatch requires a concrete header dictionary "Authorization": f"Bearer {access_token}", "x-goog-user-project": project, "Content-Type": "application/json", } ) - base_model = model.removeprefix("vertex_ai/") - model_info = self._get_model_info(model=model) + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) if model_info["vertex_ai_audio_api"] == "lyria_predict": - request_body = { - "instances": [{"prompt": input}], - "parameters": {"sample_count": 1}, + request_body = { # mutable-ok: predict dispatch requires a concrete provider request dictionary; rebind-ok: exactly one provider API shape initializes the request + "instances": [ # mutable-ok: predict dispatch requires a concrete instances list + {"prompt": input} # mutable-ok: predict dispatch requires a concrete instance dictionary + ], + "parameters": { # mutable-ok: predict dispatch requires a concrete parameters dictionary + "sample_count": 1 + }, } else: - request_body = {"model": base_model, "input": input} + request_body = { # mutable-ok: interactions dispatch requires a concrete provider request dictionary; rebind-ok: exactly one provider API shape initializes the request + "model": base_model, + "input": input, + } if optional_params.get("response_format") == "wav": - request_body["response_format"] = { + request_body[ + "response_format" + ] = { # mutable-ok: interactions dispatch requires a nested response-format dictionary "type": "audio", "mime_type": "audio/wav", } @@ -609,33 +636,49 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): ) -> "HttpxBinaryResponseContent": from litellm.types.llms.openai import HttpxBinaryResponseContent - response_json = raw_response.json() - base_model = model.removeprefix("vertex_ai/") - model_info = self._get_model_info(model=model) - audio_data: str | None = None - mime_type: str | None = None + response_json: Final = raw_response.json() + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + audio_data: str | None = None # rebind-ok: response parsing discovers audio data in provider-specific shapes + mime_type: str | None = None # rebind-ok: response parsing discovers the MIME type beside the audio payload if model_info["vertex_ai_audio_api"] == "lyria_predict": - predictions = response_json.get("predictions") or [] + predictions: Final = response_json.get("predictions") or () if predictions: - audio_data = predictions[0].get("audioContent") or predictions[0].get("bytesBase64Encoded") - mime_type = predictions[0].get("mimeType") + audio_data = predictions[0].get("audioContent") or predictions[0].get( + "bytesBase64Encoded" + ) # rebind-ok: predict response supplies the generated audio value + mime_type = predictions[0].get("mimeType") # rebind-ok: predict response supplies its audio MIME type else: - for step in response_json.get("steps") or response_json.get("outputs") or []: - content_items = step.get("content") or [] if step.get("type") == "model_output" else [step] + for step in response_json.get("steps") or response_json.get("outputs") or (): + content_items = step.get("content") or () if step.get("type") == "model_output" else (step,) for content in content_items: if content.get("type") == "audio" and content.get("data"): - audio_data = content["data"] - mime_type = content.get("mime_type") + audio_data = content[ + "data" + ] # rebind-ok: interactions response supplies the generated audio value + mime_type = content.get( + "mime_type" + ) # rebind-ok: interactions response supplies its audio MIME type if audio_data is None: raise ValueError(f"No generated audio found in Vertex AI {base_model} response") - default_format = model_info["supported_audio_formats"][0] - mime_type = mime_type or {"mp3": "audio/mpeg", "wav": "audio/wav"}[default_format] - response = HttpxBinaryResponseContent( + default_format: Final = model_info["supported_audio_formats"][0] + mime_type = ( + mime_type + or { # mutable-ok: short-lived lookup selects the default response MIME type; rebind-ok: absent provider MIME type falls back to model metadata + "mp3": "audio/mpeg", + "wav": "audio/wav", + }[default_format] + ) + response: Final = HttpxBinaryResponseContent( httpx.Response( status_code=raw_response.status_code, content=base64.b64decode(audio_data), - headers={"content-type": mime_type}, + headers={ # mutable-ok: httpx requires a concrete response header dictionary + "content-type": mime_type + }, ) ) - response._hidden_params = {"audio_mime_type": mime_type} + response._hidden_params = { # mutable-ok: response metadata is a concrete dictionary + "audio_mime_type": mime_type + } return response diff --git a/litellm/main.py b/litellm/main.py index 55db92d44b3..040af897256 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8261,9 +8261,13 @@ def speech( # Vertex AI Text-to-Speech (Google Cloud TTS) if text_to_speech_provider_config is None: if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): - text_to_speech_provider_config = VertexAILyriaTextToSpeechConfig() + text_to_speech_provider_config = ( + VertexAILyriaTextToSpeechConfig() + ) # rebind-ok: model metadata selects the Lyria provider implementation else: - text_to_speech_provider_config = VertexAITextToSpeechConfig() + text_to_speech_provider_config = ( + VertexAITextToSpeechConfig() + ) # rebind-ok: non-Lyria Vertex models use the standard TTS implementation # Cast to specific Vertex AI config type to access dispatch method vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 4d348b51055..53df964e541 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -334,10 +334,10 @@ class VertexPassthroughLoggingHandler: @staticmethod def _handle_audio_predict_response( - json_response: dict, + json_response: dict, # mutable-ok: passthrough logging receives the decoded provider response dictionary logging_obj: LiteLLMLoggingObj, model: str, - kwargs: dict, + kwargs: dict, # mutable-ok: passthrough logging enriches the shared callback metadata dictionary ) -> PassThroughEndpointLoggingTypedDict: prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count( json_response=json_response @@ -346,26 +346,41 @@ class VertexPassthroughLoggingHandler: VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) or 0.0 ) * prediction_count - logging_obj.model = model - logging_obj.model_call_details["model"] = model - logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" - logging_obj.custom_llm_provider = "vertex_ai" - logging_obj.model_call_details["response_cost"] = response_cost + logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Vertex model + logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata + "model" + ] = model + logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata + "custom_llm_provider" + ] = "vertex_ai" + logging_obj.custom_llm_provider = ( # rebind-ok: attribution records the resolved provider + "vertex_ai" + ) + logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata + "response_cost" + ] = response_cost - kwargs["response_cost"] = response_cost - kwargs["model"] = model - kwargs["custom_llm_provider"] = "vertex_ai" + kwargs[ # rebind-ok: callback metadata is enriched for downstream hooks + "response_cost" + ] = response_cost + kwargs["model"] = model # rebind-ok: callback metadata records the resolved model + kwargs["custom_llm_provider"] = "vertex_ai" # rebind-ok: callback metadata records the resolved provider - standard_pass_through_response_object: Final[StandardPassThroughResponseObject] = { + standard_pass_through_response_object: Final[ + StandardPassThroughResponseObject + ] = { # mutable-ok: callback contract requires a concrete response dictionary "response": json_response, } - return { + return { # mutable-ok: passthrough logging contract requires a concrete result dictionary "result": standard_pass_through_response_object, "kwargs": kwargs, } @staticmethod - def _is_audio_predict_response(model: str, json_response: dict) -> bool: + def _is_audio_predict_response( + model: str, + json_response: dict, # mutable-ok: predicate inspects the decoded provider response dictionary without mutation + ) -> bool: return ( VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0 and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) is not None @@ -373,7 +388,9 @@ class VertexPassthroughLoggingHandler: @staticmethod def _get_audio_prediction_unit_cost(model: str) -> float | None: - model_info: Final = litellm.model_cost.get(f"vertex_ai/{model}", {}) + model_info: Final = litellm.model_cost.get(f"vertex_ai/{model}") + if model_info is None: + return None output_cost_per_second: Final = model_info.get("output_cost_per_second") audio_seconds_per_prediction: Final = model_info.get("audio_seconds_per_prediction") if not isinstance(output_cost_per_second, (int, float)) or not isinstance( @@ -383,7 +400,9 @@ class VertexPassthroughLoggingHandler: return float(output_cost_per_second * audio_seconds_per_prediction) @staticmethod - def _get_audio_prediction_count(json_response: dict) -> int: + def _get_audio_prediction_count( + json_response: dict, # mutable-ok: counter inspects the decoded provider response dictionary without mutation + ) -> int: predictions: Final = json_response.get("predictions") if not isinstance(predictions, list): return 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bb0485be7c1..6c645e70bee 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -168,8 +168,8 @@ class ProviderSpecificModelInfo(TypedDict, total=False): default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None] supports_output_config: bool | None supports_image_size: bool | None - supported_audio_formats: list[Literal["mp3", "wav"]] | None - vertex_ai_audio_api: Literal["lyria_predict", "lyria_interactions"] | None + supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None] + vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None] bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None bedrock_converse_supports_strict_tools: bool | None @@ -312,9 +312,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_video_per_second: float | None # only for vertex ai models output_cost_per_audio_per_second: float | None # only for vertex ai models output_cost_per_second: float | None # for OpenAI Speech models - audio_seconds_per_prediction: float | None - max_audio_length_hours: float | None - max_audio_per_prompt: int | None + audio_seconds_per_prediction: ReadOnly[float | None] + max_audio_length_hours: ReadOnly[float | None] + max_audio_per_prompt: ReadOnly[int | None] output_cost_per_second_1080p: ( float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) From abb655daa8851f15ce534a378761c268c811d942 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 30 Aug 2026 09:58:28 -0500 Subject: [PATCH 15/43] fix(vertex-ai): encode Lyria predict URL path segments Percent-encode project, location, and model as single path segments. Lyria 3 speech builds the interactions URL through staging's minter with the already resolved project --- .../text_to_speech/transformation.py | 20 ++++++++++-- .../text_to_speech/test_transformation.py | 32 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 81f1cbd5157..17c4f0ed0c0 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -17,6 +17,7 @@ from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( speech_media_type_from_audio_bytes, ) +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, @@ -570,17 +571,32 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): VertexAIInteractionsConfig, ) - return VertexAIInteractionsConfig().get_complete_url( + resolved_project: Final = project + + def mint_access_token( + _credentials: VERTEX_CREDENTIALS_TYPES | None, + project_id: str | None, + ) -> tuple[str, str]: + return "", project_id or resolved_project + + return VertexAIInteractionsConfig(mint_access_token=mint_access_token).get_complete_url( api_base=api_base, model=base_model, litellm_params={ # mutable-ok: interactions dispatch expects a concrete parameter dictionary **litellm_params, "vertex_project": project, + "vertex_location": "global", }, ) location: Final = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") - return f"{base_url}/v1/projects/{project}/locations/{location}/publishers/google/models/{base_model}:predict" + encoded_project: Final = encode_url_path_segment(project, field_name="project") + encoded_location: Final = encode_url_path_segment(location, field_name="location") + encoded_model: Final = encode_url_path_segment(base_model, field_name="model") + return ( + f"{base_url}/v1/projects/{encoded_project}/locations/{encoded_location}" + f"/publishers/google/models/{encoded_model}:predict" + ) def transform_text_to_speech_request( self, diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index a1bb203e67e..3fc5278d3dd 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,4 +1,5 @@ import base64 +from typing import Final from unittest.mock import MagicMock, Mock, patch import httpx @@ -259,6 +260,37 @@ class TestVertexAILyriaTextToSpeechConfig: "locations/europe-west4/publishers/google/models/lyria-002:predict" ) + def test_get_complete_url_encodes_injected_predict_path_segments(self, monkeypatch: pytest.MonkeyPatch) -> None: + injected: Final = ( + "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" + ) + encoded: Final = ( + "victim-project%2Flocations%2Fus-central1%2Fpublishers%2Fgoogle" + "%2Fmodels%2Fother-model%3Apredict%3Fignored%3D" + ) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{injected}", + { + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + }, + ) + + url: Final = VertexAILyriaTextToSpeechConfig().get_complete_url( + model=injected, + api_base="https://us-central1-aiplatform.googleapis.com", + litellm_params={ + "vertex_project": injected, + "vertex_location": injected, + }, + ) + + assert url == ( + "https://us-central1-aiplatform.googleapis.com" + f"/v1/projects/{encoded}/locations/{encoded}/publishers/google/models/{encoded}:predict" + ) + def test_get_complete_url_for_lyria_3(self): config = VertexAILyriaTextToSpeechConfig() From e18df8f09e7aef60ddb76979ec7a4ddeae5f5aeb Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 30 Aug 2026 09:58:28 -0500 Subject: [PATCH 16/43] fix(vertex-ai): fall back to bundled Lyria costs Prefer runtime model_cost when both numeric fields are present, then bundled Lyria metadata so stale maps still bill 0.002 * 30 --- litellm/llms/vertex_ai/common_utils.py | 2 + .../vertex_passthrough_logging_handler.py | 20 ++++++++-- ...test_vertex_passthrough_logging_handler.py | 39 +++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 8885d19c1c0..b649d8dac0e 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -28,6 +28,8 @@ class VertexAILyriaModelInfo(TypedDict): vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"]] supported_audio_formats: ReadOnly[tuple[Literal["mp3", "wav"], ...]] output_cost_per_image: NotRequired[ReadOnly[float]] + output_cost_per_second: NotRequired[ReadOnly[float]] + audio_seconds_per_prediction: NotRequired[ReadOnly[float]] _VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER: Final = TypeAdapter(VertexAILyriaModelInfo) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 53df964e541..f7625d71168 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -1,5 +1,6 @@ import asyncio import re +from collections.abc import Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -10,7 +11,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.vertex_ai.common_utils import get_vertex_location_from_url +from litellm.llms.vertex_ai.common_utils import ( + get_vertex_ai_lyria_model_info, + get_vertex_location_from_url, +) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator as VertexModelResponseIterator, ) @@ -388,8 +392,18 @@ class VertexPassthroughLoggingHandler: @staticmethod def _get_audio_prediction_unit_cost(model: str) -> float | None: - model_info: Final = litellm.model_cost.get(f"vertex_ai/{model}") - if model_info is None: + runtime_unit_cost: Final = VertexPassthroughLoggingHandler._audio_prediction_unit_cost_from_model_info( + model_info=litellm.model_cost.get(f"vertex_ai/{model}") + ) + if runtime_unit_cost is not None: + return runtime_unit_cost + return VertexPassthroughLoggingHandler._audio_prediction_unit_cost_from_model_info( + model_info=get_vertex_ai_lyria_model_info(model=model) + ) + + @staticmethod + def _audio_prediction_unit_cost_from_model_info(model_info: object) -> float | None: + if not isinstance(model_info, Mapping): return None output_cost_per_second: Final = model_info.get("output_cost_per_second") audio_seconds_per_prediction: Final = model_info.get("audio_seconds_per_prediction") diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 1e8d569d3d1..f3d28d58c22 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Final from unittest.mock import MagicMock import httpx @@ -143,3 +144,41 @@ def test_audio_predict_response_supports_bytes_base64_encoded( assert result["kwargs"]["response_cost"] == pytest.approx(0.06) assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) + + +def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_map_omits_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + stale_runtime_model_cost: Final = { + key: value for key, value in litellm.model_cost.items() if key != "vertex_ai/lyria-002" + } + monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "audioContent": "clip", + "mimeType": "audio/wav", + } + ] + }, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert "vertex_ai/lyria-002" not in litellm.model_cost + assert result["kwargs"]["model"] == "lyria-002" + assert result["kwargs"]["response_cost"] == pytest.approx(0.06) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) From c784e604acc7791e882391a512b4836366fef0e1 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 30 Aug 2026 10:15:50 -0500 Subject: [PATCH 17/43] test(vertex-ai): drop Lyria auth patches from transform tests Subclass the Lyria transformer to stub token minting, and mark the remaining litellm.speech patches so TQ008 stays within budget. --- .../vertex_ai/text_to_speech/test_transformation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 3fc5278d3dd..457c3f76dbb 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -335,16 +335,17 @@ class TestVertexAILyriaTextToSpeechConfig: ), ], ) - @patch.object(VertexAILyriaTextToSpeechConfig, "_ensure_access_token") def test_transform_request( self, - mock_ensure_token, model, response_format, expected_body, ): - mock_ensure_token.return_value = ("mock-token", "music-project") - config = VertexAILyriaTextToSpeechConfig() + class _LyriaConfig(VertexAILyriaTextToSpeechConfig): + def _ensure_access_token(self, *args: object, **kwargs: object) -> tuple[str, str]: + return "mock-token", "music-project" + + config = _LyriaConfig() request = config.transform_text_to_speech_request( model=model, @@ -514,12 +515,12 @@ class TestVertexAILyriaTextToSpeechConfig: mock_response.status_code = 200 mock_response.json.return_value = response_json with ( - patch.object( + patch.object( # test-quality-ok: litellm.speech has no seam for Vertex token minting VertexAILyriaTextToSpeechConfig, "_ensure_access_token", return_value=("mock-token", "music-project"), ), - patch( + patch( # test-quality-ok: litellm.speech has no seam for the HTTP handler "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post", return_value=mock_response, ) as mock_post, From cab3801f23aff8c5ac73f0f38d212ab74c9392b6 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 30 Aug 2026 12:00:42 -0500 Subject: [PATCH 18/43] fix(vertex-ai): simplify Lyria provider typing --- .../llms/vertex_ai/interactions/__init__.py | 3 -- .../text_to_speech/transformation.py | 30 ++++++++++--------- litellm/types/llms/openai.py | 3 ++ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/litellm/llms/vertex_ai/interactions/__init__.py b/litellm/llms/vertex_ai/interactions/__init__.py index f6f86f65e87..e69de29bb2d 100644 --- a/litellm/llms/vertex_ai/interactions/__init__.py +++ b/litellm/llms/vertex_ai/interactions/__init__.py @@ -1,3 +0,0 @@ -from .transformation import VertexAIInteractionsConfig - -__all__ = ("VertexAIInteractionsConfig",) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 17c4f0ed0c0..2047be2ea37 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -621,8 +621,8 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): ) base_model: Final = model.removeprefix("vertex_ai/") model_info: Final = self._get_model_info(model=model) - if model_info["vertex_ai_audio_api"] == "lyria_predict": - request_body = { # mutable-ok: predict dispatch requires a concrete provider request dictionary; rebind-ok: exactly one provider API shape initializes the request + request_body: Final[dict[str, object]] = ( # mutable-ok: HTTP dispatch requires a concrete provider payload + { # mutable-ok: predict dispatch requires a concrete provider request dictionary "instances": [ # mutable-ok: predict dispatch requires a concrete instances list {"prompt": input} # mutable-ok: predict dispatch requires a concrete instance dictionary ], @@ -630,18 +630,22 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): "sample_count": 1 }, } - else: - request_body = { # mutable-ok: interactions dispatch requires a concrete provider request dictionary; rebind-ok: exactly one provider API shape initializes the request + if model_info["vertex_ai_audio_api"] == "lyria_predict" + else { # mutable-ok: interactions dispatch requires a concrete provider request dictionary "model": base_model, "input": input, + **( + { # mutable-ok: interactions dispatch requires a nested response-format dictionary + "response_format": { # mutable-ok: interactions response format is a concrete provider payload + "type": "audio", + "mime_type": "audio/wav", + } + } + if optional_params.get("response_format") == "wav" + else {} # mutable-ok: no response override is merged for non-WAV output + ), } - if optional_params.get("response_format") == "wav": - request_body[ - "response_format" - ] = { # mutable-ok: interactions dispatch requires a nested response-format dictionary - "type": "audio", - "mime_type": "audio/wav", - } + ) return TextToSpeechRequestData(dict_body=request_body, headers=headers) def transform_text_to_speech_response( @@ -694,7 +698,5 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): }, ) ) - response._hidden_params = { # mutable-ok: response metadata is a concrete dictionary - "audio_mime_type": mime_type - } + response.set_audio_mime_type(mime_type) return response diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 32d88da0085..0b13191d977 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -119,6 +119,9 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): return self._hidden_params["response_cost"] = response_cost + def set_audio_mime_type(self, audio_mime_type: str) -> None: + self._hidden_params["audio_mime_type"] = audio_mime_type + class NotGiven: """ From 1af9c229fa452c1a226bcc22f34f74cfffd54c54 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 17:29:56 -0700 Subject: [PATCH 19/43] ci(e2e): leave the managed-files opt-in file to its own lane instead of failing on an empty collection --- .github/workflows/test-e2e-changed.yml | 5 +++-- tests/e2e/CONTRIBUTING.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index b20c84af02d..c1a74f007ce 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -27,14 +27,15 @@ jobs: REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} SMOKE_TESTS: tests/e2e/access_control + OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$' run: | files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \ --jq '.[] | select(.status != "removed") | .filename')" tests="$(printf '%s\n' "${files}" \ | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \ - | grep -vE '^tests/e2e/(ui|claude_code|load)/' \ + | grep -vE "${OWN_LANE}" \ | sort -u | tr '\n' ' ' | sed 's/ $//')" || true - if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE '^tests/e2e/(ui|claude_code|load)/' \ + if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE "${OWN_LANE}" \ | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then tests="${SMOKE_TESTS}" echo "harness or stack changed without a test file; running the smoke suite" diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b984791c738..81fcdf84e74 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,7 +54,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### The pull request check -Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down +Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change From a3ee6b25667d7640214465b58364d04554c02f20 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 17:50:35 -0700 Subject: [PATCH 20/43] ci(e2e): fail a pass whose every collected test was skipped --- .github/e2e-stack/assert_tests_ran.py | 21 +++++++++++++++++++++ .github/workflows/test-e2e-changed.yml | 4 +++- tests/e2e/CONTRIBUTING.md | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 .github/e2e-stack/assert_tests_ran.py diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py new file mode 100644 index 00000000000..092a9db7256 --- /dev/null +++ b/.github/e2e-stack/assert_tests_ran.py @@ -0,0 +1,21 @@ +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Final + + +def main() -> int: + report: Final = ET.parse(Path(sys.argv[1])).getroot() + suites: Final = tuple(report.iter("testsuite")) + collected: Final = sum(int(suite.get("tests", "0")) for suite in suites) + skipped: Final = sum(int(suite.get("skipped", "0")) for suite in suites) + executed: Final = collected - skipped + _ = sys.stdout.write(f"executed {executed} of {collected} collected tests ({skipped} skipped)\n") + if executed > 0: + return 0 + _ = sys.stdout.write("::error::every selected test was skipped, so nothing was verified\n") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index c1a74f007ce..e08660d0412 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -152,9 +152,10 @@ jobs: run: | read -r -a test_files <<< "${TESTS}" for pass in 1 2 3; do + report="${RUNNER_TEMP}/e2e-pass-${pass}.xml" echo "::group::pass ${pass} of 3" set +e - uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider + uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider --junitxml="${report}" status=$? set -e echo "::endgroup::" @@ -166,6 +167,7 @@ jobs: echo "::error::pass ${pass} of 3 failed with exit code ${status}" exit "${status}" fi + uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" done - name: Show stack logs on failure diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 81fcdf84e74..ef6fbb5405d 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,7 +54,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### The pull request check -Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down +Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change From a9bef8d370ebd553071729b32a9f27f729f1d17e Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 18:01:19 -0700 Subject: [PATCH 21/43] docs(e2e): drop the spend-cap claim from the credentials paragraph --- tests/e2e/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index ef6fbb5405d..1e88f6505cf 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -56,7 +56,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down -Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change +Credentials: everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. The cloud credentials in it are dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role, and the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project. The provider API keys carry no lane-specific spend cap, since a cap that trips mid-month would fail every run until it resets, so the reviewer's approval of the `e2e-changed` environment is the control on how a PR's tests use them. Rotating any value is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change ### Record and replay From 200e2901d661d9bcea83d6e69c3839ea71b07e2c Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:33:10 +0530 Subject: [PATCH 22/43] fix(anthropic_responses): preserve Responses refusal blocks in Anthropic translation When OpenAI Responses returns a refusal content block, Anthropic /v1/messages erased the refusal text into an empty content array and emitted stop_reason 'end_turn'. Translate refusal blocks to Anthropic text blocks, map stop_reason to 'refusal', and add 'refusal' to AnthropicFinishReason. Fixes #39721 --- .../responses_adapters/streaming_iterator.py | 22 +++++- .../responses_adapters/transformation.py | 35 +++++++-- litellm/types/llms/anthropic.py | 2 +- ...t_responses_adapters_streaming_iterator.py | 34 +++++++++ .../test_responses_adapters_transformation.py | 76 +++++++++++++------ 5 files changed, 138 insertions(+), 31 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index a97ce18d179..5f6c5bad190 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -111,7 +111,7 @@ class AnthropicResponsesStreamWrapper: item_type: Final = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) - if item_type == "message": + if item_type in ("message", "refusal"): self._open_block(item_id, {"type": "text", "text": ""}) elif item_type == "function_call": call_id: Final = ( @@ -132,7 +132,7 @@ class AnthropicResponsesStreamWrapper: return # ---- text delta ---- - if event_type == "response.output_text.delta": + if event_type in ("response.output_text.delta", "response.refusal.delta"): item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index @@ -238,6 +238,24 @@ class AnthropicResponsesStreamWrapper: if out_type == "function_call": stop_reason = "tool_use" break + elif out_type == "refusal": + stop_reason = "refusal" + break + elif out_type == "message": + content_parts = getattr(out_item, "content", ()) or ( + out_item.get("content") or () if isinstance(out_item, dict) else () + ) + for part in content_parts: + part_type = getattr(part, "type", None) or ( + part.get("type") if isinstance(part, dict) else None + ) + if ( + part_type == "refusal" + or hasattr(part, "refusal") + or (isinstance(part, dict) and "refusal" in part) + ): + stop_reason = "refusal" + break self._chunk_queue.append( { diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 0eb0e38a46e..6cabd35117d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -631,10 +631,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(item, ResponseOutputMessage): for part in item.content: - if getattr(part, "type", None) == "output_text": + part_type = getattr(part, "type", None) + if part_type == "output_text": content.append( AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump() ) + elif part_type == "refusal" or hasattr(part, "refusal"): + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "refusal", "") or "" + ).model_dump() + ) + stop_reason = "refusal" elif isinstance(item, ResponseFunctionToolCall): try: @@ -654,11 +662,22 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(item, dict): item_type = item.get("type") if item_type == "message": - for part in item.get("content", []): - if isinstance(part, dict) and part.get("type") == "output_text": - content.append( - AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump() - ) + for part in item.get("content", ()): + if isinstance(part, dict): + part_type = part.get("type") + if part_type == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif part_type == "refusal" or "refusal" in part: + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("refusal", "") or "" + ).model_dump() + ) + stop_reason = "refusal" elif item_type == "reasoning": content.extend( self._thinking_blocks_from_reasoning_item( @@ -679,6 +698,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) stop_reason = "tool_use" + elif item_type == "refusal": + refusal_text = item.get("refusal") or item.get("text", "") or "" + content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) + stop_reason = "refusal" # status -> stop_reason override if response.status == "incomplete": diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b3462203c4b..f1107c87c73 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -658,7 +658,7 @@ class AnthropicOutputTokensDetails(BaseModel): thinking_tokens: int | None = None -AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] +AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] class AnthropicResponse(BaseModel): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index aebbed88c70..77ecfb73ff6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -308,3 +308,37 @@ class TestResponseCompletedUsage: "cache_creation_input_tokens": 10, "cache_read_input_tokens": 4004, } + + +class TestRefusalStreamEvents: + def test_refusal_delta_emits_text_delta(self): + chunks = _process_all( + [ + {"type": "response.created"}, + {"type": "response.refusal.delta", "item_id": "ref_1", "delta": "I cannot fulfill this."}, + ] + ) + assert any( + c.get("type") == "content_block_delta" and c.get("delta", {}).get("text") == "I cannot fulfill this." + for c in chunks + ) + + def test_response_completed_with_refusal_sets_stop_reason_refusal(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Policy violation"}]}], + usage=None, + ) + chunks = _process_all([{"type": "response.completed", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + + def test_response_completed_with_standalone_refusal_item_sets_stop_reason_refusal(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "refusal", "refusal": "Standalone refusal"}], + usage=None, + ) + chunks = _process_all([{"type": "response.completed", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 5ecf604f096..f48c31ea5ed 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -147,9 +147,7 @@ class TestOutputConfigStructuredOutput: def test_output_config_format_explicit_strict_true_is_preserved(self): """Nested output_config.format with explicit strict=True is preserved.""" - req = _make_request( - output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}} - ) + req = _make_request(output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}}) kwargs = _ADAPTER.translate_request(req) assert kwargs["text"]["format"]["strict"] is True @@ -1207,6 +1205,18 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg +def _make_refusal_message(refusal_text: str) -> MagicMock: + from openai.types.responses import ResponseOutputMessage + + part = MagicMock() + part.type = "refusal" + part.refusal = refusal_text + + msg = MagicMock(spec=ResponseOutputMessage) + msg.content = [part] + return msg + + def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: """Build a mock ResponseFunctionToolCall.""" from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] @@ -1279,6 +1289,38 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["stop_reason"] == "end_turn" + def test_refusal_part_becomes_text_block_and_sets_stop_reason_refusal(self): + response = _make_mock_response(output=[_make_refusal_message("I cannot fulfill this request.")]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "I cannot fulfill this request." + assert result["stop_reason"] == "refusal" + + def test_dict_refusal_part_in_message_becomes_text_block(self): + output_item = { + "type": "message", + "content": [{"type": "refusal", "refusal": "Refused by policy"}], + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Refused by policy" + assert result["stop_reason"] == "refusal" + + def test_dict_refusal_item_becomes_text_block(self): + output_item = { + "type": "refusal", + "refusal": "Standalone refusal", + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Standalone refusal" + assert result["stop_reason"] == "refusal" + def test_incomplete_status_sets_max_tokens(self): """status='incomplete' overrides stop_reason to 'max_tokens'.""" response = _make_mock_response( @@ -1337,9 +1379,7 @@ class TestTranslateResponse: ] ) result: Any = _ADAPTER.translate_response(response) - assert result["content"] == [ - {"type": "thinking", "thinking": "Weighing the options.", "signature": None} - ] + assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" @@ -1481,9 +1521,7 @@ class TestToolResultImages: }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}], }, ] @@ -1630,9 +1668,7 @@ class TestToolResultDocuments: }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}], }, ] @@ -1665,9 +1701,7 @@ class TestToolResultDocuments: def test_document_title_becomes_filename(self): output = self._tool_output(self._translate([self._base64_document(title="quarterly-report.pdf")])) - assert output == [ - {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI} - ] + assert output == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}] def test_url_document_becomes_file_url_part(self): output = self._tool_output( @@ -1776,9 +1810,7 @@ class TestUserContentDocuments: def test_document_title_becomes_filename(self): content = self._user_content(self._translate([self._base64_document(title="quarterly-report.pdf")])) - assert content == [ - {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI} - ] + assert content == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}] def test_url_document_becomes_file_url_part(self): content = self._user_content( @@ -1808,9 +1840,7 @@ class TestUserContentDocuments: assert content == [{"type": "input_text", "text": "still here"}] def test_document_breakpoint_rides_on_the_file_part(self): - content = self._user_content( - self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)]) - ) + content = self._user_content(self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)])) assert content == [ { "type": "input_file", @@ -1857,7 +1887,9 @@ class TestPromptCacheBreakpointToResponses: ] def test_system_without_breakpoint_still_becomes_instructions(self): - request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}]) + request = _make_request( + system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}] + ) kwargs = _ADAPTER.translate_request(request) assert kwargs["instructions"] == "Be concise.\nBe helpful." assert kwargs["input"] == [ From 0b34abe8fe3904453ff4796cfdb8b8981dfaf7e8 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:02:07 +0530 Subject: [PATCH 23/43] fix(anthropic): harden refusal translation --- .../adapters/streaming_iterator.py | 39 +++++- .../adapters/transformation.py | 37 +++++- .../responses_adapters/streaming_iterator.py | 114 +++++++++++------- .../responses_adapters/transformation.py | 58 +++++++-- litellm/types/llms/anthropic.py | 7 ++ .../anthropic_messages/anthropic_response.py | 11 +- ...al_pass_through_adapters_transformation.py | 45 +++++++ .../test_streaming_iterator_first_delta.py | 87 +++++++++++++ ...t_responses_adapters_streaming_iterator.py | 60 +++++++-- .../test_responses_adapters_transformation.py | 41 ++++--- 10 files changed, 397 insertions(+), 102 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 78ff83cafbf..41db4f12143 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -4,13 +4,14 @@ import copy import json import traceback from collections import deque -from collections.abc import AsyncIterator, Iterator, Sequence +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import ( TYPE_CHECKING, Any, Final, Literal, Protocol, + cast, get_args, ) @@ -305,6 +306,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage + self._refusal_text_parts: list[str] = [] self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -572,6 +574,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): current_content_block_index=self.current_content_block_index, applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None), ) + processed_chunk = self._with_refusal_stop_details(processed_chunk) # Check if this is a usage chunk and we have a held stop_reason chunk if will_merge_into_held: @@ -806,6 +809,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): current_content_block_index=self.current_content_block_index, applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None), ) + processed_chunk = self._with_refusal_stop_details(processed_chunk) # Check if this is a usage chunk and we have a held stop_reason chunk if will_merge_into_held: @@ -993,6 +997,31 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): def _increment_content_block_index(self): self.current_content_block_index += 1 + def _with_refusal_stop_details( + self, + processed_chunk: ContentBlockDelta | MessageBlockDelta, + ) -> ContentBlockDelta | MessageBlockDelta: + if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: + return processed_chunk + delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) + if delta.get("stop_reason") == "max_tokens": + return processed_chunk + return cast( + ContentBlockDelta | MessageBlockDelta, + { + **processed_chunk, + "delta": { + **delta, + "stop_reason": "refusal", + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "".join(self._refusal_text_parts), + }, + }, + }, + ) + @staticmethod def _delta_has_content(processed_chunk: dict[str, Any]) -> bool: """Return True if a translated chunk carries a non-empty @@ -1044,6 +1073,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return False if getattr(delta, "content", None): return False + if getattr(delta, "refusal", None): + return False if getattr(delta, "reasoning_content", None): return False # thinking_blocks whose entries are all empty AND unsigned must not @@ -1069,8 +1100,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): """ from .transformation import LiteLLMAnthropicMessagesAdapter - # Example logic - customize based on your needs: - # If chunk indicates a tool call + refusal_text: Final = LiteLLMAnthropicMessagesAdapter._refusal_text(chunk.choices[0].delta) + if refusal_text is not None: + self._refusal_text_parts.append(refusal_text) + if chunk.choices[0].finish_reason is not None: return False diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 573a461e89e..5655f59a4cc 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -307,6 +307,17 @@ class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass + @staticmethod + def _refusal_text(message_or_delta: object) -> str | None: + refusal: Final = getattr(message_or_delta, "refusal", None) + if isinstance(refusal, str): + return refusal + provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None) + if isinstance(provider_specific_fields, Mapping): + provider_refusal: Final = provider_specific_fields.get("refusal") + return provider_refusal if isinstance(provider_refusal, str) else None + return None + ### FOR [BETA] `/v1/messages` endpoint support def _extract_signature_from_tool_call(self, tool_call: object) -> str | None: @@ -1324,6 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter: new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) + if (refusal_text := self._refusal_text(choice.message)) is not None: + new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) # Handle tool calls (in parallel to text content) if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: for tool_call in choice.message.tool_calls: @@ -1482,14 +1495,23 @@ class LiteLLMAnthropicMessagesAdapter: choices=response.choices, tool_name_mapping=tool_name_mapping, ) + refusal_text: Final = next( + (text for choice in response.choices if (text := self._refusal_text(choice.message)) is not None), + None, + ) if polyfill_result is not None and polyfill_result.compaction_block is not None: anthropic_content.insert(0, polyfill_result.compaction_block) ## extract finish reason - anthropic_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( + translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( openai_finish_reason=response.choices[0].finish_reason ) + anthropic_finish_reason: Final = ( + "refusal" + if refusal_text is not None and translated_finish_reason != "max_tokens" + else translated_finish_reason + ) # extract usage usage: Final[Usage] = getattr(response, "usage") anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) @@ -1511,6 +1533,15 @@ class LiteLLMAnthropicMessagesAdapter: usage=anthropic_usage, content=anthropic_content, stop_reason=anthropic_finish_reason, + stop_details=( + { + "type": "refusal", + "category": None, + "explanation": refusal_text, + } + if anthropic_finish_reason == "refusal" + else None + ), ) applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None @@ -1551,7 +1582,9 @@ class LiteLLMAnthropicMessagesAdapter: "signature": thought_sig, } return "tool_use", cast("ContentBlockContentBlockDict", tool_block) - elif choice.delta.content is not None and len(choice.delta.content) > 0: + elif (choice.delta.content is not None and len(choice.delta.content) > 0) or self._refusal_text( + choice.delta + ) is not None: return "text", TextBlock(type="text", text="") elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): thinking_blocks = choice.delta.thinking_blocks or [] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 5f6c5bad190..1b0ab5923d8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -1,5 +1,6 @@ # What is this? ## Translates OpenAI call to Anthropic `/v1/messages` format +import asyncio import json import traceback from collections import deque @@ -49,6 +50,8 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque = deque() + self._refusal_text_parts: list[str] = [] + self._sync_responses_iterator: Any = None def _make_message_start(self) -> dict[str, Any]: return { @@ -111,7 +114,7 @@ class AnthropicResponsesStreamWrapper: item_type: Final = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) - if item_type in ("message", "refusal"): + if item_type == "message": self._open_block(item_id, {"type": "text", "text": ""}) elif item_type == "function_call": call_id: Final = ( @@ -131,8 +134,14 @@ class AnthropicResponsesStreamWrapper: ) return + if event_type == "response.refusal.delta": + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + if isinstance(delta, str): + self._refusal_text_parts.append(delta) + return + # ---- text delta ---- - if event_type in ("response.output_text.delta", "response.refusal.delta"): + if event_type == "response.output_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index @@ -215,52 +224,53 @@ class AnthropicResponsesStreamWrapper: response_obj: Final = getattr(event, "response", None) or ( event.get("response") if isinstance(event, dict) else None ) - stop_reason = "end_turn" - anthropic_usage: AnthropicUsage = AnthropicUsage(input_tokens=0, output_tokens=0) - - if response_obj is not None: - status: Final = getattr(response_obj, "status", None) - if status == "incomplete": - stop_reason = "max_tokens" - anthropic_usage = ( - LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( - getattr(response_obj, "usage", None) - ) + output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () + refusal_text: Final = LiteLLMAnthropicToResponsesAPIAdapter._refusal_text_from_output(output) or ( + "".join(self._refusal_text_parts) or None + ) + status: Final = getattr(response_obj, "status", None) if response_obj is not None else None + has_tool_call: Final = any( + getattr(item, "type", None) == "function_call" + or (isinstance(item, dict) and item.get("type") == "function_call") + for item in output + ) + stop_reason: Final = ( + "max_tokens" + if status == "incomplete" + else "refusal" + if refusal_text is not None + else "tool_use" + if has_tool_call + else "end_turn" + ) + anthropic_usage: Final[AnthropicUsage] = ( + LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( + getattr(response_obj, "usage", None) ) + if response_obj is not None + else AnthropicUsage(input_tokens=0, output_tokens=0) + ) - # Check if tool_use was in the output to override stop_reason - if response_obj is not None: - output: Final = getattr(response_obj, "output", []) or [] - for out_item in output: - out_type = getattr(out_item, "type", None) or ( - out_item.get("type") if isinstance(out_item, dict) else None - ) - if out_type == "function_call": - stop_reason = "tool_use" - break - elif out_type == "refusal": - stop_reason = "refusal" - break - elif out_type == "message": - content_parts = getattr(out_item, "content", ()) or ( - out_item.get("content") or () if isinstance(out_item, dict) else () - ) - for part in content_parts: - part_type = getattr(part, "type", None) or ( - part.get("type") if isinstance(part, dict) else None - ) - if ( - part_type == "refusal" - or hasattr(part, "refusal") - or (isinstance(part, dict) and "refusal" in part) - ): - stop_reason = "refusal" - break + message_delta_payload: Final = { + "stop_reason": stop_reason, + "stop_sequence": None, + **( + { + "stop_details": { + "type": "refusal", + "category": None, + "explanation": refusal_text, + } + } + if stop_reason == "refusal" + else {} + ), + } self._chunk_queue.append( { "type": "message_delta", - "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "delta": message_delta_payload, "usage": dict(anthropic_usage), } ) @@ -284,10 +294,22 @@ class AnthropicResponsesStreamWrapper: # Consume the upstream stream try: - async for event in self.responses_stream: - self._process_event(event) - if self._chunk_queue: - return self._chunk_queue.popleft() + if hasattr(self.responses_stream, "__aiter__"): + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + else: + if self._sync_responses_iterator is None: + self._sync_responses_iterator = iter(self.responses_stream) + missing: Final = object() + while True: + event = await asyncio.to_thread(next, self._sync_responses_iterator, missing) + if event is missing: + break + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() except StopAsyncIteration: pass except Exception as e: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 6cabd35117d..92f3e08f24d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,7 +6,7 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from itertools import groupby from typing import Any, Final, cast @@ -69,6 +69,38 @@ class LiteLLMAnthropicToResponsesAPIAdapter: chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage) + @staticmethod + def _refusal_text_from_output(output: Iterable[object]) -> str | None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + def refusal_text_from_item(item: object) -> str | None: + if isinstance(item, ResponseOutputMessage): + return next( + (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal)), + None, + ) + if not isinstance(item, Mapping): + return None + item_mapping: Final = cast(Mapping[str, object], item) + raw_parts: Final = item_mapping.get("content") + if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): + return None + return next( + ( + refusal + for part in cast(Sequence[object], raw_parts) + if isinstance(part, Mapping) + and cast(Mapping[str, object], part).get("type") == "refusal" + and isinstance((refusal := cast(Mapping[str, object], part).get("refusal")), str) + ), + None, + ) + + return next( + (text for item in output if (text := refusal_text_from_item(item)) is not None), + None, + ) + # ------------------------------------------------------------------ # # Request translation: Anthropic -> Responses API # # ------------------------------------------------------------------ # @@ -624,6 +656,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" + refusal_text: Final = self._refusal_text_from_output(cast(Iterable[object], response.output)) for item in response.output: if isinstance(item, ResponseReasoningItem): @@ -636,13 +669,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content.append( AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump() ) - elif part_type == "refusal" or hasattr(part, "refusal"): + elif part_type == "refusal": content.append( AnthropicResponseContentBlockText( type="text", text=getattr(part, "refusal", "") or "" ).model_dump() ) - stop_reason = "refusal" elif isinstance(item, ResponseFunctionToolCall): try: @@ -671,13 +703,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: type="text", text=part.get("text", "") ).model_dump() ) - elif part_type == "refusal" or "refusal" in part: + elif part_type == "refusal": content.append( AnthropicResponseContentBlockText( type="text", text=part.get("refusal", "") or "" ).model_dump() ) - stop_reason = "refusal" elif item_type == "reasoning": content.extend( self._thinking_blocks_from_reasoning_item( @@ -698,14 +729,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) stop_reason = "tool_use" - elif item_type == "refusal": - refusal_text = item.get("refusal") or item.get("text", "") or "" - content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) - stop_reason = "refusal" - - # status -> stop_reason override if response.status == "incomplete": stop_reason = "max_tokens" + elif refusal_text is not None: + stop_reason = "refusal" anthropic_usage: Final = self.translate_responses_api_usage_to_anthropic_usage(response.usage) @@ -718,4 +745,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: usage=anthropic_usage, content=content, stop_reason=stop_reason, + stop_details=( + { + "type": "refusal", + "category": None, + "explanation": refusal_text, + } + if stop_reason == "refusal" + else None + ), ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f1107c87c73..cdfc5227e06 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -520,8 +520,15 @@ ContentBlockContentBlockDict = ToolUseBlock | TextBlock | ChatCompletionThinking ContentBlockStart = ContentBlockStartToolUse | ContentBlockStartText +class AnthropicStopDetails(TypedDict, total=False): + type: ReadOnly[Literal["refusal"]] + category: ReadOnly[str | None] + explanation: ReadOnly[str | None] + + class MessageDelta(TypedDict, total=False): stop_reason: str | None + stop_details: AnthropicStopDetails class ServerToolUsage(TypedDict, total=False): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 4fe1dafc73b..038a23a3ca2 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -5,6 +5,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, + AnthropicStopDetails, ContextManagementResponse, ServerToolUsage, ) @@ -78,16 +79,6 @@ class AnthropicUsage(TypedDict, total=False): server_tool_use: NotRequired[ReadOnly[ServerToolUsage]] -class AnthropicStopDetails(TypedDict, total=False): - """ - Safeguard verdict accompanying a `stop_reason: "refusal"` response: - https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback - """ - - category: ReadOnly[str | None] - explanation: ReadOnly[str | None] - - class AnthropicMessagesResponse(TypedDict, total=False): """ Anthropic Messages API Response: https://docs.anthropic.com/en/api/messages diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2d74c00071b..fba5532d1e5 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -40,6 +40,51 @@ from litellm.types.utils import ( ) +def test_translate_chat_refusal_to_anthropic_response(): + response = ModelResponse( + id="chatcmpl-refusal", + model="openai-model", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message(content=None, role="assistant", refusal="I cannot fulfill this request."), + ) + ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["content"] == [{"type": "text", "text": "I cannot fulfill this request."}] + assert result["stop_reason"] == "refusal" + assert result.get("stop_details") == { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + } + + +def test_translate_chat_length_takes_precedence_over_refusal(): + response = ModelResponse( + id="chatcmpl-partial-refusal", + model="openai-model", + choices=[ + Choices( + index=0, + finish_reason="length", + message=Message(content=None, role="assistant", refusal="Partial refusal"), + ) + ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["stop_reason"] == "max_tokens" + assert result.get("stop_details") is None + + def test_translate_streaming_openai_chunk_to_anthropic_content_block(): choices = [ StreamingChoices( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 17d42f55ae0..e7381986ec2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -108,6 +108,93 @@ def _text_deltas(events: List[dict]) -> List[str]: ] +def test_streaming_chat_refusal_emits_only_refusal_stop_details(): + chunks = [ + _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == [] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"] == { + "stop_reason": "refusal", + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + }, + } + + +@pytest.mark.asyncio +async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): + chunks = [ + _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == [] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): + chunks = [ + _make_chunk( + Delta(content=None, refusal="I cannot fulfill this request."), + finish_reason="stop", + ) + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.asyncio +async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async(): + chunks = [ + _make_chunk( + Delta(content=None, refusal="I cannot fulfill this request."), + finish_reason="stop", + ) + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +async def test_streaming_chat_length_takes_precedence_over_refusal(async_mode: bool): + chunks = [ + _make_chunk(Delta(content=None, refusal="Partial refusal")), + _make_chunk(Delta(content=None), finish_reason="length"), + ] + stream = _AsyncStream(chunks) if async_mode else iter(chunks) + wrapper = AnthropicStreamWrapper(completion_stream=stream, model="openai-model") + + events = await _drain_async(wrapper) if async_mode else _drain_sync(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "max_tokens" + assert "stop_details" not in message_delta["delta"] + + def _input_json_deltas(events: List[dict]) -> List[str]: return [ e["delta"]["partial_json"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 77ecfb73ff6..d8f16dde3e7 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -34,6 +34,14 @@ def _drain_async(events: list) -> list: return asyncio.run(_run()) +def _drain_sync_upstream(events: list) -> list: + async def _run() -> list: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter(events), model="m") + return [chunk async for chunk in wrapper] + + return asyncio.run(_run()) + + class TestMessageStartEmittedExactlyOnce: """The ``__anext__`` fallback emits ``message_start`` before consuming the stream, so ``_process_event`` must not emit a second one when @@ -55,6 +63,15 @@ class TestMessageStartEmittedExactlyOnce: chunks = _drain_async([{"type": "response.created"}]) assert chunks[0]["type"] == "message_start" + def test_sync_upstream_iterator_is_consumed(self): + chunks = _drain_sync_upstream( + [ + {"type": "response.created"}, + {"type": "response.output_text.delta", "item_id": "m1", "delta": "hi"}, + ] + ) + assert any(chunk.get("delta", {}).get("text") == "hi" for chunk in chunks) + class TestProcessEventResponseCreatedGuard: """``_process_event`` must emit ``message_start`` exactly once even if @@ -311,17 +328,37 @@ class TestResponseCompletedUsage: class TestRefusalStreamEvents: - def test_refusal_delta_emits_text_delta(self): + def test_refusal_event_sequence_emits_only_stop_details(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}], + usage=None, + ) chunks = _process_all( [ {"type": "response.created"}, - {"type": "response.refusal.delta", "item_id": "ref_1", "delta": "I cannot fulfill this."}, + {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.refusal.delta", "item_id": "msg_1", "delta": "I cannot fulfill this."}, + {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.completed", "response": response}, ] ) - assert any( - c.get("type") == "content_block_delta" and c.get("delta", {}).get("text") == "I cannot fulfill this." - for c in chunks - ) + assert [chunk["type"] for chunk in chunks] == [ + "message_start", + "content_block_start", + "content_block_stop", + "message_delta", + "message_stop", + ] + assert chunks[3]["delta"] == { + "stop_reason": "refusal", + "stop_sequence": None, + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this.", + }, + } def test_response_completed_with_refusal_sets_stop_reason_refusal(self): response = SimpleNamespace( @@ -333,12 +370,13 @@ class TestRefusalStreamEvents: message_delta = next(c for c in chunks if c["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" - def test_response_completed_with_standalone_refusal_item_sets_stop_reason_refusal(self): + def test_incomplete_status_takes_precedence_over_refusal(self): response = SimpleNamespace( - status="completed", - output=[{"type": "refusal", "refusal": "Standalone refusal"}], + status="incomplete", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Partial refusal"}]}], usage=None, ) - chunks = _process_all([{"type": "response.completed", "response": response}]) + chunks = _process_all([{"type": "response.incomplete", "response": response}]) message_delta = next(c for c in chunks if c["type"] == "message_delta") - assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_reason"] == "max_tokens" + assert "stop_details" not in message_delta["delta"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index f48c31ea5ed..8205e993d26 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1205,16 +1205,16 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg -def _make_refusal_message(refusal_text: str) -> MagicMock: - from openai.types.responses import ResponseOutputMessage +def _make_refusal_message(refusal_text: str): + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal - part = MagicMock() - part.type = "refusal" - part.refusal = refusal_text - - msg = MagicMock(spec=ResponseOutputMessage) - msg.content = [part] - return msg + return ResponseOutputMessage( + id="msg_refusal", + content=[ResponseOutputRefusal(type="refusal", refusal=refusal_text)], + role="assistant", + status="completed", + type="message", + ) def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: @@ -1296,6 +1296,11 @@ class TestTranslateResponse: assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "I cannot fulfill this request." assert result["stop_reason"] == "refusal" + assert result.get("stop_details") == { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + } def test_dict_refusal_part_in_message_becomes_text_block(self): output_item = { @@ -1308,18 +1313,16 @@ class TestTranslateResponse: assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "Refused by policy" assert result["stop_reason"] == "refusal" + assert result.get("stop_details", {}).get("explanation") == "Refused by policy" - def test_dict_refusal_item_becomes_text_block(self): - output_item = { - "type": "refusal", - "refusal": "Standalone refusal", - } - response = _make_mock_response(output=[output_item]) + def test_incomplete_status_takes_precedence_over_refusal(self): + response = _make_mock_response( + output=[_make_refusal_message("Partial refusal")], + status="incomplete", + ) result: Any = _ADAPTER.translate_response(response) - assert len(result["content"]) == 1 - assert result["content"][0]["type"] == "text" - assert result["content"][0]["text"] == "Standalone refusal" - assert result["stop_reason"] == "refusal" + assert result["stop_reason"] == "max_tokens" + assert result.get("stop_details") is None def test_incomplete_status_sets_max_tokens(self): """status='incomplete' overrides stop_reason to 'max_tokens'.""" From 7399b3844dfc45ab0c9ddbb116b26235ce4f7df0 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:33:06 +0530 Subject: [PATCH 24/43] fix(anthropic): satisfy lint budget gates for refusal translation --- .../adapters/streaming_iterator.py | 14 +++++----- .../adapters/transformation.py | 2 +- .../responses_adapters/streaming_iterator.py | 10 +++---- .../responses_adapters/transformation.py | 28 ++++++++++--------- litellm/types/llms/anthropic.py | 2 +- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 41db4f12143..66fcad474d5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -11,7 +11,7 @@ from typing import ( Final, Literal, Protocol, - cast, + cast, # noqa: TID251 # rebuilt message_delta dict spans the ContentBlockDelta/MessageBlockDelta union get_args, ) @@ -306,7 +306,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage - self._refusal_text_parts: list[str] = [] + self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -1003,17 +1003,17 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): ) -> ContentBlockDelta | MessageBlockDelta: if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: return processed_chunk - delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) + delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": return processed_chunk - return cast( + return cast( # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch ContentBlockDelta | MessageBlockDelta, - { + { # mutable-ok: fresh translation payload; never mutated after construction **processed_chunk, - "delta": { + "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": { + "stop_details": { # mutable-ok: fresh stop_details payload; never mutated after construction "type": "refusal", "category": None, "explanation": "".join(self._refusal_text_parts), diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 5655f59a4cc..2d950f007a5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1534,7 +1534,7 @@ class LiteLLMAnthropicMessagesAdapter: content=anthropic_content, stop_reason=anthropic_finish_reason, stop_details=( - { + { # mutable-ok: fresh refusal stop_details payload built per response "type": "refusal", "category": None, "explanation": refusal_text, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 1b0ab5923d8..8e5ce77e48f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -50,7 +50,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque = deque() - self._refusal_text_parts: list[str] = [] + self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks self._sync_responses_iterator: Any = None def _make_message_start(self) -> dict[str, Any]: @@ -251,19 +251,19 @@ class AnthropicResponsesStreamWrapper: else AnthropicUsage(input_tokens=0, output_tokens=0) ) - message_delta_payload: Final = { + message_delta_payload: Final = { # mutable-ok: fresh message_delta payload built per chunk "stop_reason": stop_reason, "stop_sequence": None, **( - { - "stop_details": { + { # mutable-ok: fresh refusal stop_details payload built per chunk + "stop_details": { # mutable-ok: fresh refusal stop_details payload built per chunk "type": "refusal", "category": None, "explanation": refusal_text, } } if stop_reason == "refusal" - else {} + else {} # mutable-ok: empty spread placeholder for non-refusal stop ), } diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 92f3e08f24d..318065148b8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -81,20 +81,20 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) if not isinstance(item, Mapping): return None - item_mapping: Final = cast(Mapping[str, object], item) + item_mapping: Final = cast(Mapping[str, object], item) # cast-ok: keys re-checked before use raw_parts: Final = item_mapping.get("content") if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): return None - return next( - ( - refusal - for part in cast(Sequence[object], raw_parts) - if isinstance(part, Mapping) - and cast(Mapping[str, object], part).get("type") == "refusal" - and isinstance((refusal := cast(Mapping[str, object], part).get("refusal")), str) - ), - None, - ) + for part in cast(Sequence[object], raw_parts): # cast-ok: members re-validated below + if not isinstance(part, Mapping): + continue + part_mapping = cast(Mapping[str, object], part) # cast-ok: keys re-checked before use + if part_mapping.get("type") != "refusal": + continue + refusal = part_mapping.get("refusal") + if isinstance(refusal, str): + return refusal + return None return next( (text for item in output if (text := refusal_text_from_item(item)) is not None), @@ -656,7 +656,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" - refusal_text: Final = self._refusal_text_from_output(cast(Iterable[object], response.output)) + refusal_text: Final = self._refusal_text_from_output( + cast(Iterable[object], response.output) # cast-ok: output items re-validated per item + ) for item in response.output: if isinstance(item, ResponseReasoningItem): @@ -746,7 +748,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content=content, stop_reason=stop_reason, stop_details=( - { + { # mutable-ok: fresh refusal stop_details payload built per response "type": "refusal", "category": None, "explanation": refusal_text, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index cdfc5227e06..365d59a179b 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -528,7 +528,7 @@ class AnthropicStopDetails(TypedDict, total=False): class MessageDelta(TypedDict, total=False): stop_reason: str | None - stop_details: AnthropicStopDetails + stop_details: ReadOnly[AnthropicStopDetails] class ServerToolUsage(TypedDict, total=False): From 0f59b6fb7a996debcb350f6bf10a18e5ba276a62 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 12:03:42 -0700 Subject: [PATCH 25/43] ci(e2e): refine changed-test selection and runner lifecycle --- .github/e2e-stack/assert_tests_ran.py | 29 +++++-- .github/e2e-stack/secrets_to_env.py | 44 ++++++---- .github/e2e-stack/up.sh | 8 +- .github/workflows/test-e2e-changed.yml | 80 ++++++++++++++++--- .../test_e2e_changed_gate.py | 55 +++++++++++++ tests/e2e/CONTRIBUTING.md | 12 ++- tests/e2e/gateway/stage_mirror_ci_config.yml | 4 - 7 files changed, 186 insertions(+), 46 deletions(-) create mode 100644 tests/code_coverage_tests/test_e2e_changed_gate.py diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py index 092a9db7256..7fd3f7c7c33 100644 --- a/.github/e2e-stack/assert_tests_ran.py +++ b/.github/e2e-stack/assert_tests_ran.py @@ -5,15 +5,28 @@ from typing import Final def main() -> int: - report: Final = ET.parse(Path(sys.argv[1])).getroot() - suites: Final = tuple(report.iter("testsuite")) - collected: Final = sum(int(suite.get("tests", "0")) for suite in suites) - skipped: Final = sum(int(suite.get("skipped", "0")) for suite in suites) - executed: Final = collected - skipped - _ = sys.stdout.write(f"executed {executed} of {collected} collected tests ({skipped} skipped)\n") - if executed > 0: + selected: Final = tuple(sys.argv[2:]) + try: + report: Final = ET.parse(Path(sys.argv[1])).getroot() + except (ET.ParseError, OSError): + _ = sys.stdout.write("::error::could not read the test execution report\n") + return 1 + cases: Final = tuple(report.iter("testcase")) + passed: Final = frozenset( + case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error")) + ) + missing: Final = tuple(path for path in selected if path not in passed) + for path in selected: + collected: Final = sum(case.get("file") == path for case in cases) + skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases) + _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n") + if ( + selected + and not missing + and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error")) + ): return 0 - _ = sys.stdout.write("::error::every selected test was skipped, so nothing was verified\n") + _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n") return 1 diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index d2d6675d690..99e717b271d 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -1,29 +1,41 @@ +import os import re import sys from pathlib import Path from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError -secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) -SECRET_NAME: Final = re.compile(r"KEY|SECRET|TOKEN|PASS|CREDENTIAL|LICENSE|AUTH") +secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) +ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") def main() -> int: - env_path = Path(sys.argv[1]) - secrets = {key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()} - unwritable = tuple( - key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value - ) - if unwritable: - _ = sys.stderr.write(f"values contain characters unsafe for both bash and dotenv: {', '.join(unwritable)}\n") + env_path: Final = Path(sys.argv[1]) + try: + secrets: Final = { + key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items() + } + except (ValidationError, UnicodeError): + _ = sys.stderr.write("expected a JSON object containing string environment values\n") + return 1 + if any( + ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") for key, value in secrets.items() + ): + _ = sys.stderr.write("environment names or values cannot be represented in both bash and dotenv\n") + return 1 + for value in secrets.values(): + if value: + _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n") + sys.stdout.flush() + lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) + try: + with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle: + os.fchmod(handle.fileno(), 0o600) + _ = handle.write("\n".join(lines) + "\n") + except OSError: + _ = sys.stderr.write("could not write the environment file\n") return 1 - lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) - with env_path.open("a") as handle: - _ = handle.write("\n".join(lines) + "\n") - for key, value in secrets.items(): - if value and SECRET_NAME.search(key): - _ = sys.stdout.write(f"::add-mask::{value}\n") return 0 diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 06fce35a896..e891add341d 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +umask 077 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" @@ -8,9 +9,9 @@ LOGS_DIR="${STACK_DIR}/logs" PIDS_DIR="${STACK_DIR}/pids" POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" -VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1}" +VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}" JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}" -NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29-alpine}" +NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}" LB_PORT="${E2E_LB_PORT:-4000}" GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}" @@ -28,6 +29,8 @@ JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" +chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" +chmod 755 "${CERTS_DIR}" log() { printf 'e2e-stack: %s\n' "$*"; } @@ -38,7 +41,6 @@ wait_for() { until eval "${check}"; do if ((SECONDS >= deadline)); then log "timed out waiting for ${label}" - tail -n 60 "${LOGS_DIR}"/*.log 2>/dev/null || true exit 1 fi sleep 2 diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index e08660d0412..b802e9173f5 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -26,27 +26,26 @@ jobs: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} - SMOKE_TESTS: tests/e2e/access_control + HEAD_SHA: ${{ github.event.pull_request.head.sha }} OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$' run: | + gh api "repos/${REPO}/pulls/${PR_NUMBER}" \ + --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \ + | grep -Fxq "${HEAD_SHA}" files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \ --jq '.[] | select(.status != "removed") | .filename')" + gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}" tests="$(printf '%s\n' "${files}" \ | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \ | grep -vE "${OWN_LANE}" \ | sort -u | tr '\n' ' ' | sed 's/ $//')" || true - if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE "${OWN_LANE}" \ - | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then - tests="${SMOKE_TESTS}" - echo "harness or stack changed without a test file; running the smoke suite" - fi echo "tests=${tests}" >> "${GITHUB_OUTPUT}" if [ -n "${tests}" ]; then echo "any=true" >> "${GITHUB_OUTPUT}" echo "selected e2e tests: ${tests}" else echo "any=false" >> "${GITHUB_OUTPUT}" - echo "no e2e changes; nothing to run" + echo "no changed e2e test files supported by this stack; nothing to run" fi run: @@ -88,6 +87,7 @@ jobs: uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false + ref: ${{ github.sha }} - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -120,14 +120,24 @@ jobs: run: uv run --no-sync playwright install --with-deps chromium - name: Configure AWS credentials + id: aws uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} aws-region: us-east-1 role-session-name: litellm-e2e-changed-${{ github.run_id }} + role-duration-seconds: 900 + output-env-credentials: false + output-credentials: true - name: Fetch provider credentials from AWS Secrets Manager + env: + AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} + AWS_DEFAULT_REGION: us-east-1 run: | + umask 077 aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \ --query SecretString --output text \ | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env @@ -138,7 +148,12 @@ jobs: - name: Boot the stage-mirror stack id: boot - run: bash .github/e2e-stack/up.sh + run: | + umask 077 + if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then + echo "::error::stage-mirror stack failed to boot; raw logs are not published" + exit 1 + fi - name: Export stack environment run: | @@ -149,13 +164,16 @@ jobs: - name: Run the selected tests three times with retries off env: TESTS: ${{ needs.detect.outputs.tests }} + E2E_FIXTURE_MODE: live run: | + umask 077 read -r -a test_files <<< "${TESTS}" for pass in 1 2 3; do report="${RUNNER_TEMP}/e2e-pass-${pass}.xml" echo "::group::pass ${pass} of 3" set +e - uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider --junitxml="${report}" + uv run --no-sync pytest "${test_files[@]}" --rootdir=. --reruns 0 -v -p no:cacheprovider \ + -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1 status=$? set -e echo "::endgroup::" @@ -163,13 +181,49 @@ jobs: echo "::error::the selected files collected no runnable tests, so nothing was verified" exit 1 fi + if ! uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"; then + echo "::error::pass ${pass} of 3 did not verify every selected file" + exit 1 + fi if [ "${status}" != "0" ]; then echo "::error::pass ${pass} of 3 failed with exit code ${status}" exit "${status}" fi - uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" + echo "pass ${pass} of 3 passed" done - - name: Show stack logs on failure - if: failure() && steps.boot.conclusion != 'skipped' - run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log + - name: Stop the stack + if: always() && steps.boot.outcome != 'skipped' + run: bash .github/e2e-stack/down.sh + + - name: Remove credentials and raw output + if: always() + run: | + rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml + rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" + + gate: + name: e2e-changed-tests + needs: [detect, run] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require three successful passes when tests changed + env: + DETECT_RESULT: ${{ needs.detect.result }} + ANY_TESTS: ${{ needs.detect.outputs.any }} + RUN_RESULT: ${{ needs.run.result }} + run: | + if [ "${DETECT_RESULT}" != "success" ]; then + echo "::error::changed-test detection did not succeed" + exit 1 + fi + if [ "${ANY_TESTS}" = "false" ]; then + echo "no changed e2e test files supported by this stack; nothing to run" + exit 0 + fi + if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then + echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch" + exit 1 + fi diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py new file mode 100644 index 00000000000..7a82a8ebb60 --- /dev/null +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -0,0 +1,55 @@ +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Final + +import pytest + +GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py" +SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") + + +@pytest.mark.parametrize( + ("second_outcome", "expected_status"), + (("passed", 0), ("skipped", 1), ("failure", 1), ("error", 1), ("deselected", 1)), +) +def test_each_changed_file_must_run(tmp_path: Path, second_outcome: str, expected_status: int) -> None: + suite: Final = ET.Element("testsuite") + _ = ET.SubElement(suite, "testcase", file=SELECTED[0]) + if second_outcome != "deselected": + second: Final = ET.SubElement(suite, "testcase", file=SELECTED[1]) + if second_outcome != "passed": + _ = ET.SubElement(second, second_outcome) + report: Final = tmp_path / "report.xml" + ET.ElementTree(suite).write(report) + + result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True) + + assert result.returncode == expected_status, result.stdout + + +@pytest.mark.parametrize("outcome", ("failure", "error")) +def test_passing_case_does_not_hide_a_failure_in_the_same_file(tmp_path: Path, outcome: str) -> None: + suite: Final = ET.Element("testsuite") + _ = ET.SubElement(suite, "testcase", file=SELECTED[0]) + failed: Final = ET.SubElement(suite, "testcase", file=SELECTED[0]) + _ = ET.SubElement(failed, outcome) + report: Final = tmp_path / "report.xml" + ET.ElementTree(suite).write(report) + + result: Final = subprocess.run( + [sys.executable, str(GATE), str(report), SELECTED[0]], capture_output=True, text=True + ) + + assert result.returncode == 1 + + +@pytest.mark.parametrize("contents", ("", "')) +def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None: + report: Final = tmp_path / "report.xml" + _ = report.write_text(contents) + + result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True) + + assert result.returncode == 1 diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 1e88f6505cf..d51e729fd1c 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,9 +54,17 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### The pull request check -Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times with pytest retries off. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack -Credentials: everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. The cloud credentials in it are dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role, and the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project. The provider API keys carry no lane-specific spend cap, since a cap that trips mid-month would fail every run until it resets, so the reviewer's approval of the `e2e-changed` environment is the control on how a PR's tests use them. Rotating any value is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change +Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run without retrying. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch + +Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code + +Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. Keep provider credentials dedicated to this lane with only the permissions those tests need + +Fetched values are masked before use, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs + +To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use ### Record and replay diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 21664c2d0a1..6b3756b5f9f 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -52,10 +52,6 @@ model_list: litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY - - model_name: gemini-2.5-flash - litellm_params: - model: gemini/gemini-2.5-flash - api_key: os.environ/GEMINI_API_KEY mcp_servers: devin: From 5051e6d44acb627d912cba11b4afdc91ae5ff8c6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 5 Sep 2026 12:08:11 -0700 Subject: [PATCH 26/43] ci(e2e): include execution gate checks in code quality --- .github/workflows/test-code-quality.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index c112bf2bb22..a0e0ce6665a 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -74,6 +74,9 @@ jobs: - name: check_workflow_startup_safety run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py + - name: test_e2e_changed_gate + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py + - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py From c55248d113800f32283532cb811a2509b27ba94a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:07:29 -0700 Subject: [PATCH 27/43] fix(router): treat a routing entry with no latency samples as zero latency Latency-based routing averaged a deployment's cached samples with total / len(samples) and raised ZeroDivisionError once an entry held none, which the proxy answered as a 500 for every later request on that model group. Cost-based routing writes the same {model_group}_map entry with minute counters only, so a group used by both strategies hit this on every latency-routed request. A deployment with no samples now counts as 0 latency, the same as one the router has never seen Resolves LIT-7053 --- litellm/router_strategy/lowest_latency.py | 23 +++++++------- .../router_strategy/test_lowest_latency.py | 30 +++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index a1b67eaeaf9..4a1fb4f325a 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -1,6 +1,7 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) import random +from collections.abc import Sequence from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, Final @@ -25,6 +26,12 @@ class RoutingArgs(LiteLLMPydanticObjectBase): max_latency_list_size: int = 10 +def _average_latency(samples: Sequence[float | int]) -> float: + if not samples: + return 0.0 + return sum(sample for sample in samples if isinstance(sample, float)) / len(samples) + + class LowestLatencyLoggingHandler(CustomLogger): test_flag: bool = False logged_success: int = 0 @@ -431,23 +438,13 @@ class LowestLatencyLoggingHandler(CustomLogger): item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) # get average latency or average ttft (depending on streaming/non-streaming) - total: float = 0.0 use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 ) - if use_ttft: - for _call_latency in item_ttft_latency: - if isinstance(_call_latency, float): - total += _call_latency - item_latency = total / len(item_ttft_latency) - else: - for _call_latency in item_latency: - if isinstance(_call_latency, float): - total += _call_latency - item_latency = total / len(item_latency) + average_latency = _average_latency(item_ttft_latency if use_ttft else item_latency) # -------------- # # Debugging Logic @@ -456,7 +453,7 @@ class LowestLatencyLoggingHandler(CustomLogger): # this helps a user to debug why the router picked a specfic deployment # _deployment_api_base = _deployment.get("litellm_params", {}).get("api_base", "") if _deployment_api_base is not None: - _latency_per_deployment[_deployment_api_base] = item_latency + _latency_per_deployment[_deployment_api_base] = average_latency # -------------- # # End of Debugging Logic # -------------- # @@ -466,7 +463,7 @@ class LowestLatencyLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, item_latency)) + potential_deployments.append((_deployment, average_latency)) if len(potential_deployments) == 0: return None diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 6701f4a7aa2..251b760a8d8 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -163,3 +163,33 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): assert latencies and latencies[-1] == pytest.approx(2.0) assert not isinstance(latencies[-1], timedelta) json.dumps({"latency": latencies}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cached_entry", + [{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}], + ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"], +) +async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry): + """A cached entry with no latency samples (cost-based routing shares the group's map key and writes + minute buckets only) must count as 0 latency, like an unseen deployment, instead of dividing by zero.""" + cache = DualCache() + handler = LowestLatencyLoggingHandler(router_cache=cache) + cache.set_cache( + key="gemini-embedding-001_map", + value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}}, + ) + healthy_deployments = [ + {"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}}, + {"model_info": {"id": "slower"}, "litellm_params": {}}, + ] + + picked = await handler.async_get_available_deployments( + model_group="gemini-embedding-001", + healthy_deployments=healthy_deployments, + request_kwargs={"stream": False, "metadata": {}}, + ) + + assert picked is not None + assert picked["model_info"]["id"] == DEPLOYMENT_ID From 65d8bbb8ac4610c9722ecae03ffa16b352c47256 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:10:40 -0700 Subject: [PATCH 28/43] fix(e2e): wait for every gateway before using a new model and keep the network rerun The changed-tests workflow overrode the suite's `--reruns 1` with `--reruns 0`, so a transport blip failed a pass that pytest.ini already scopes to network errors and 5xx responses. Pass 2 of run 33692484803 also went red 15s after a model write with "no healthy deployments": the barrier only polled /v1/models through nginx, which proves one gateway converged, and the next request rolled the other. The stack now exports LITELLM_PROXY_REPLICA_URLS, the barrier polls every replica with the full budget before settling, and up.sh refuses to boot without DD_API_KEY, since the gateway config enables the datadog callback on every run --- .github/e2e-stack/up.sh | 7 ++ .github/workflows/test-e2e-changed.yml | 4 +- tests/e2e/CONTRIBUTING.md | 6 +- tests/e2e/claude_code/_env.py | 1 + tests/e2e/claude_code/conftest.py | 1 + tests/e2e/e2e_config.py | 18 +++- tests/e2e/proxy_client.py | 117 +++++++++++++++++++------ tests/e2e/test_proxy_client.py | 87 ++++++++++++++++++ 8 files changed, 206 insertions(+), 35 deletions(-) create mode 100644 tests/e2e/test_proxy_client.py diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index e891add341d..2f2e6c6f9a8 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -54,6 +54,12 @@ if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then set +a fi +if [[ -z "${DD_API_KEY:-}" ]]; then + log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env" + exit 1 +fi +export DD_SITE="${DD_SITE:-datadoghq.com}" + if ! port_open "${DATABASE_PORT}"; then docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \ -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \ @@ -189,6 +195,7 @@ wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness cat > "${STACK_DIR}/stack.env" <> "${GITHUB_ENV}" - - name: Run the selected tests three times with retries off + - name: Run the selected tests three times env: TESTS: ${{ needs.detect.outputs.tests }} E2E_FIXTURE_MODE: live @@ -172,7 +172,7 @@ jobs: report="${RUNNER_TEMP}/e2e-pass-${pass}.xml" echo "::group::pass ${pass} of 3" set +e - uv run --no-sync pytest "${test_files[@]}" --rootdir=. --reruns 0 -v -p no:cacheprovider \ + uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \ -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1 status=$? set -e diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index d51e729fd1c..7bfd02023dd 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,13 +54,13 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times with pytest retries off. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack -Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run without retrying. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch +Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code -Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. Keep provider credentials dedicated to this lane with only the permissions those tests need +Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need Fetched values are masked before use, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs diff --git a/tests/e2e/claude_code/_env.py b/tests/e2e/claude_code/_env.py index ed2da1fe1e2..889d8f848dd 100644 --- a/tests/e2e/claude_code/_env.py +++ b/tests/e2e/claude_code/_env.py @@ -99,5 +99,6 @@ def require_proxy_client( base_url=cfg.base_url, master_key=cfg.api_key, control_plane_base_url=cfg.base_url, + replica_urls=(cfg.base_url,), ) return ProxyClientConfig(client=client, api_key=cfg.api_key) diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py index aaad7667936..6e3dce0377e 100644 --- a/tests/e2e/claude_code/conftest.py +++ b/tests/e2e/claude_code/conftest.py @@ -594,6 +594,7 @@ def _build_control_plane_client(proxy_config: ProxyConfig): base_url=proxy_config.base_url, master_key=proxy_config.api_key, control_plane_base_url=proxy_config.base_url, + replica_urls=(proxy_config.base_url,), ) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 21c5a338dc3..70d1180cb38 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -10,6 +10,7 @@ import os import time import uuid from pathlib import Path +from typing import Final from dotenv import load_dotenv @@ -32,6 +33,14 @@ CONTROL_PLANE_BASE_URL = os.environ.get( "LITELLM_CONTROL_PLANE_URL", PROXY_BASE_URL ).rstrip("/") + +def parse_replica_urls(raw: str, fallback: str) -> tuple[str, ...]: + urls: Final = tuple(url.strip().rstrip("/") for url in raw.split(",") if url.strip()) + return urls or (fallback,) + + +PROXY_REPLICA_URLS: Final = parse_replica_urls(os.environ.get("LITELLM_PROXY_REPLICA_URLS", ""), PROXY_BASE_URL) + UI_USERNAME = os.environ.get("E2E_UI_USERNAME", "admin") UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY) @@ -86,10 +95,11 @@ SLOW_PROVIDER_TIMEOUT_SECONDS = float(os.environ.get("E2E_SLOW_PROVIDER_TIMEOUT" # (`proxy_config_reload_interval_seconds`, 30s by default and 7s on the e2e stack) # plus margin. # -# The barriers below wait this out instead of returning on first sight, because a -# single successful read only proves ONE replica converged: every request opens a -# fresh connection, so a load-balanced Service routes each one independently and -# the next call re-rolls. See ProxyClient._await_model_servable. +# The barriers below wait this out on top of polling /v1/models on every replica in +# PROXY_REPLICA_URLS: that poll proves each addressed gateway converged, but not the +# workers behind it, and behind a load balancer (PROXY_REPLICA_URLS unset) a +# successful read only proves ONE replica converged, because every request opens a +# fresh connection and the next call re-rolls. See ProxyClient._await_model_servable. PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15")) EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes") diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 2d382a610e1..c5b34a534b7 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -10,12 +10,15 @@ from __future__ import annotations import time import warnings -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import datetime +from types import MappingProxyType +from typing import Final from e2e_http import ( AnthropicHeaders, + AuthHeaders, NoBody, ProbeResult, Result, @@ -71,6 +74,7 @@ from e2e_config import ( POLL_INTERVAL, POLL_TIMEOUT, PROXY_BASE_URL, + PROXY_REPLICA_URLS, REQUEST_TIMEOUT, SLOW_PROVIDER_TIMEOUT_SECONDS, settle_propagation, @@ -79,10 +83,11 @@ from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] -# After /model/new, poll data-plane /v1/models until the model is listed (or fail). -# Bound by MODEL_SERVABLE_TIMEOUT so a stuck reload does not burn the spend -# poll_timeout (120s). Return on first listing; settle_propagation owns the separate -# wait that lets every worker and replica reload before the caller uses the model. +# After /model/new, poll /v1/models on every replica in PROXY_REPLICA_URLS until each +# lists the model (or fail). Bound by MODEL_SERVABLE_TIMEOUT per replica so a stuck +# reload does not burn the spend poll_timeout (120s). Return on first listing; +# settle_propagation owns the separate wait that lets the workers behind each replica +# reload before the caller uses the model. MODEL_SERVABLE_TIMEOUT = 40.0 MODEL_SERVABLE_DB_SYNC_SECONDS = 0.0 MODEL_SERVABLE_INTERVAL = 2.0 @@ -108,6 +113,16 @@ class NotServable: ServableOutcome = Servable | NotServable +type ModelsPoller = Callable[[float], Result[ModelsListResponse]] + + +@dataclass(frozen=True, slots=True) +class NotServableOn: + """`NotServable` labeled with the replica whose /v1/models never listed the model.""" + + replica: str + last_result: Result[ModelsListResponse] | None + def await_servable( list_models: Callable[[float], Result[ModelsListResponse]], @@ -171,9 +186,41 @@ def await_servable( sleep(wait) +def await_servable_everywhere( + pollers: Mapping[str, ModelsPoller], + *, + model_name: str, + timeout: float, + interval: float, + request_timeout: float, + db_sync_seconds: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> Servable | NotServableOn: + """`await_servable` against every replica in turn, each with the full budget, so + the model is only servable once every replica has listed it.""" + for replica, list_models in pollers.items(): + match await_servable( + list_models, + model_name=model_name, + timeout=timeout, + interval=interval, + request_timeout=request_timeout, + db_sync_seconds=db_sync_seconds, + now=now, + sleep=sleep, + ): + case NotServable(last_result=last_result): + return NotServableOn(replica=replica, last_result=last_result) + case Servable(): + continue + return Servable() + + def servable_timeout_message( *, model_name: str, + replica: str, timeout: float, db_sync_seconds: float, last_result: Result[ModelsListResponse] | None, @@ -184,8 +231,8 @@ def servable_timeout_message( else "" ) return ( - f"model {model_name!r} was created but never became servable on the data " - f"plane within {timeout}s of first listing (plus {db_sync_seconds}s continuous " + f"model {model_name!r} was created but never became servable on {replica} " + f"within {timeout}s of first listing (plus {db_sync_seconds}s continuous " f"DB sync) after /model/new (control/data-plane propagation or " f"STORE_MODEL_IN_DB reload issue){last_error}" ) @@ -194,6 +241,7 @@ def servable_timeout_message( @dataclass(frozen=True, slots=True) class ProxyClient: transport: Transport + replicas: Mapping[str, Transport] poll_timeout: float = 120.0 poll_interval: float = 5.0 model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT @@ -310,12 +358,13 @@ class ProxyClient: model name passed". We poll the data-plane /v1/models until the model appears, then settle for the remainder of the propagation budget. - Both steps are needed, and the second is the one that matters at >1 replica. - The poll proves *a* replica is serving the model; it cannot prove they all - are, because every request opens a fresh connection and a load-balanced - Service routes each one independently -- so the caller's next request - re-rolls and can land on a replica that has not reloaded yet. Waiting out - PROPAGATION_TIMEOUT is what makes the model safe to use anywhere.""" + Both steps are needed. The poll asks every replica in PROXY_REPLICA_URLS + directly, so behind the stack's load balancer it proves each gateway serves + the model rather than whichever one the balancer routed the poll to. It still + cannot see the workers behind a gateway, nor any replica when only the + balancer address is configured (every request opens a fresh connection, so + the caller's next request re-rolls), so waiting out PROPAGATION_TIMEOUT is + what makes the model safe to use anywhere.""" model_id = unwrap( self.transport.post( "/model/new", @@ -330,16 +379,10 @@ class ProxyClient: return model_id def _await_model_servable(self, model_name: str, listed_for: str | None = None) -> None: - """Block until the data plane lists `model_name`, or fail at model_servable_timeout.""" - headers = self.transport.master if listed_for is None else self.transport.bearer(listed_for) - outcome = await_servable( - lambda poll_timeout: self.transport.get( - "/v1/models", - headers=headers, - params=NoBody(), - response_type=ModelsListResponse, - timeout=poll_timeout, - ), + """Block until every replica lists `model_name`, or fail at model_servable_timeout.""" + headers: Final = self.transport.master if listed_for is None else self.transport.bearer(listed_for) + outcome: Final = await_servable_everywhere( + {url: self._models_poller(transport, headers) for url, transport in self.replicas.items()}, model_name=model_name, timeout=self.model_servable_timeout, interval=self.model_servable_interval, @@ -351,16 +394,27 @@ class ProxyClient: match outcome: case Servable(): return - case NotServable(last_result=last_result): + case NotServableOn(replica=replica, last_result=last_result): raise AssertionError( servable_timeout_message( model_name=model_name, + replica=replica, timeout=self.model_servable_timeout, db_sync_seconds=self.model_servable_db_sync_seconds, last_result=last_result, ) ) + @staticmethod + def _models_poller(transport: Transport, headers: AuthHeaders) -> ModelsPoller: + return lambda poll_timeout: transport.get( + "/v1/models", + headers=headers, + params=NoBody(), + response_type=ModelsListResponse, + timeout=poll_timeout, + ) + def update_model(self, model_id: str, litellm_params: LiteLLMParamsBody) -> None: """Merge `litellm_params` over the deployment `model_id`'s stored params via POST /model/update. The proxy overlays only the non-null fields and clears @@ -547,16 +601,20 @@ def build_proxy_client( base_url: str = PROXY_BASE_URL, master_key: str = MASTER_KEY, control_plane_base_url: str = CONTROL_PLANE_BASE_URL, + replica_urls: tuple[str, ...] = PROXY_REPLICA_URLS, ) -> ProxyClient: """The ProxyClient every suite's client is built from: a SplitTransport that routes LLM calls to the data plane (PROXY_BASE_URL) and management/admin calls to the control plane (CONTROL_PLANE_BASE_URL), with the shared poll budget. The two base URLs are the same for a monolithic proxy, so routing is then a no-op. + ``replica_urls`` (PROXY_REPLICA_URLS) names every data-plane replica the model + barrier polls directly; it is the data-plane URL itself unless the stack + exports each gateway's own address. The endpoints are injectable for callers that resolve the proxy some other way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must - pass all three together, since a caller that overrides only the data plane - would leave management calls pointed at the env default. + pass all four together, since a caller that overrides only the data plane + would leave management calls and the replica poll pointed at the env defaults. Test-to-proxy traffic always goes over the wire, in every E2E_FIXTURE_MODE: record and replay scope to the proxy's provider-bound calls via the @@ -573,8 +631,15 @@ def build_proxy_client( request_timeout=REQUEST_TIMEOUT, ), ) + replicas: Final = MappingProxyType( + { + url: HttpTransport(base_url=url, master_key=master_key, request_timeout=REQUEST_TIMEOUT) + for url in replica_urls + } + ) return ProxyClient( transport=split, + replicas=replicas, poll_timeout=POLL_TIMEOUT, poll_interval=POLL_INTERVAL, ) diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py new file mode 100644 index 00000000000..a508c97b9fb --- /dev/null +++ b/tests/e2e/test_proxy_client.py @@ -0,0 +1,87 @@ +"""Harness coverage for the model barrier that gates on every replica. + +No proxy needed and no ``e2e`` marker: this pins that a model registered through +the control plane only counts as servable once every configured replica lists it +on /v1/models, which is what keeps a two-gateway stack from handing a test a +model that one gateway has not reloaded yet. The fakes are plain pollers and an +injected clock, so nothing here monkeypatches anything. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from itertools import chain, repeat +from typing import Final + +import pytest + +from e2e_config import parse_replica_urls +from e2e_http import Success +from models import ModelListEntry, ModelsListResponse +from proxy_client import ModelsPoller, NotServableOn, Servable, await_servable_everywhere + +MODEL: Final = "gpt-under-test" +TIMEOUT: Final = 10.0 +INTERVAL: Final = 2.0 + + +@dataclass +class FakeClock: + elapsed: float = 0.0 + + def now(self) -> float: + return self.elapsed + + def sleep(self, seconds: float) -> None: + self.elapsed += seconds + + +def _listing(*model_ids: str) -> Success[ModelsListResponse]: + entries: Final = tuple(ModelListEntry(id=model_id) for model_id in model_ids) + return Success(status_code=200, data=ModelsListResponse(data=entries)) + + +def _poller(results: Iterable[Success[ModelsListResponse]]) -> ModelsPoller: + it: Final = iter(results) + return lambda _timeout: next(it) + + +def _await(pollers: Mapping[str, ModelsPoller]) -> Servable | NotServableOn: + clock: Final = FakeClock() + return await_servable_everywhere( + pollers, + model_name=MODEL, + timeout=TIMEOUT, + interval=INTERVAL, + request_timeout=5.0, + db_sync_seconds=0.0, + now=clock.now, + sleep=clock.sleep, + ) + + +class TestAwaitServableEverywhere: + @pytest.mark.parametrize("missing", ["gateway-1", "gateway-2"]) + def test_fails_on_the_replica_that_never_lists_the_model(self, missing: str) -> None: + pollers: Final = { + "gateway-1": _poller(repeat(_listing(MODEL))), + "gateway-2": _poller(repeat(_listing(MODEL))), + } | {missing: _poller(repeat(_listing()))} + assert _await(pollers) == NotServableOn(replica=missing, last_result=_listing()) + + def test_passes_once_every_replica_lists_the_model(self) -> None: + pollers: Final = { + "gateway-1": _poller(repeat(_listing(MODEL))), + "gateway-2": _poller(chain(repeat(_listing(), 2), repeat(_listing(MODEL)))), + } + assert _await(pollers) == Servable() + + +class TestParseReplicaUrls: + def test_splits_and_trims_the_gateway_addresses(self) -> None: + raw: Final = " http://127.0.0.1:4010/, http://127.0.0.1:4011 " + assert parse_replica_urls(raw, "http://lb") == ("http://127.0.0.1:4010", "http://127.0.0.1:4011") + + def test_falls_back_to_the_data_plane_address_when_unset(self) -> None: + assert parse_replica_urls("", "http://lb") == ("http://lb",) From be9a2ea7c97f945dc961c357b80f8b44f30c7746 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:20:20 -0700 Subject: [PATCH 29/43] fix(router): average every latency sample and drop the cost handler's dead division _average_latency skipped integer samples in the sum while counting them in the denominator, which contradicted its own Sequence[float | int] signature; it now averages every sample. Both success loggers in cost-based routing computed response_ms / completion_tokens and threw the result away, so a chat response with zero completion tokens raised ZeroDivisionError inside the handler. The proxy swallows and logs it, but the handler then skips that request's tpm and rpm update, so cost-based routing undercounts the deployment's usage. The QA run for the latency fix hit it on real gpt-5.5 traffic through /v1/chat/completions and /v1/messages --- litellm/router_strategy/lowest_cost.py | 11 +--- litellm/router_strategy/lowest_latency.py | 4 +- .../router_strategy/test_lowest_cost.py | 59 +++++++++++++++++++ .../router_strategy/test_lowest_latency.py | 2 - 4 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_cost.py diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..6eb4d86d280 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,6 +1,6 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) -from datetime import datetime, timedelta +from datetime import datetime from typing import Final import litellm @@ -52,16 +52,12 @@ class LowestCostLoggingHandler(CustomLogger): precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" cost_key: Final = f"{model_group}_map" - response_ms: Final[timedelta] = end_time - start_time - total_tokens = 0 if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) if _usage is not None and isinstance(_usage, litellm.Usage): - completion_tokens: Final = _usage.completion_tokens total_tokens = _usage.total_tokens - float(response_ms.total_seconds() / completion_tokens) # ------------ # Update usage @@ -131,18 +127,13 @@ class LowestCostLoggingHandler(CustomLogger): current_minute: Final = datetime.now().strftime("%M") precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" - response_ms: Final[timedelta] = end_time - start_time - total_tokens = 0 if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) if _usage is not None and isinstance(_usage, litellm.Usage): - completion_tokens: Final = _usage.completion_tokens total_tokens = _usage.total_tokens - float(response_ms.total_seconds() / completion_tokens) - # ------------ # Update usage # ------------ diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 4a1fb4f325a..bb6877a032b 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -26,10 +26,10 @@ class RoutingArgs(LiteLLMPydanticObjectBase): max_latency_list_size: int = 10 -def _average_latency(samples: Sequence[float | int]) -> float: +def _average_latency(samples: Sequence[float]) -> float: if not samples: return 0.0 - return sum(sample for sample in samples if isinstance(sample, float)) / len(samples) + return sum(samples) / len(samples) class LowestLatencyLoggingHandler(CustomLogger): diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..108053dddd9 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,59 @@ +from datetime import datetime + +import pytest + +import litellm +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + +DEPLOYMENT_ID = "9876" +KWARGS = { + "litellm_params": { + "metadata": {"model_group": "gpt-5.5-pool"}, + "model_info": {"id": DEPLOYMENT_ID}, + } +} + + +def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse: + return litellm.ModelResponse( + model="gpt-5.5", + choices=[{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}], + usage=litellm.Usage(prompt_tokens=12, completion_tokens=0, total_tokens=12), + ) + + +def _recorded_minute_counters(cache: DualCache) -> dict[str, int]: + cached = cache.get_cache(key="gpt-5.5-pool_map") or {} + minute_buckets = cached.get(DEPLOYMENT_ID, {}) + assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}" + return next(iter(minute_buckets.values())) + + +def test_log_success_event_counts_a_response_with_no_completion_tokens(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + + handler.log_success_event( + kwargs=KWARGS, + response_obj=_chat_response_with_no_completion_tokens(), + start_time=datetime(2026, 1, 1, 12, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 2), + ) + + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} + + +@pytest.mark.asyncio +async def test_async_log_success_event_counts_a_response_with_no_completion_tokens(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + + await handler.async_log_success_event( + kwargs=KWARGS, + response_obj=_chat_response_with_no_completion_tokens(), + start_time=datetime(2026, 1, 1, 12, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 2), + ) + + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 251b760a8d8..eb02459be68 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -172,8 +172,6 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"], ) async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry): - """A cached entry with no latency samples (cost-based routing shares the group's map key and writes - minute buckets only) must count as 0 latency, like an unseen deployment, instead of dividing by zero.""" cache = DualCache() handler = LowestLatencyLoggingHandler(router_cache=cache) cache.set_cache( From 5a06845db18a2e1709bcd5e696cafdd4c190bd7e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:33:15 -0700 Subject: [PATCH 30/43] fix(ci): mask only credential-length values in the e2e-changed log A one-character value in the provider secret bundle was masked too, which turned every 1 in the run log into ***, including the pass numbers and the gateway addresses, so the only public diagnostics were unreadable --- .github/e2e-stack/secrets_to_env.py | 3 ++- .../code_coverage_tests/test_e2e_changed_gate.py | 16 ++++++++++++++++ tests/e2e/CONTRIBUTING.md | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index 99e717b271d..e7870a32e42 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -8,6 +8,7 @@ from pydantic import TypeAdapter, ValidationError secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +MIN_MASKED_LENGTH: Final = 8 def main() -> int: @@ -25,7 +26,7 @@ def main() -> int: _ = sys.stderr.write("environment names or values cannot be represented in both bash and dotenv\n") return 1 for value in secrets.values(): - if value: + if len(value) >= MIN_MASKED_LENGTH: _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n") sys.stdout.flush() lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 7a82a8ebb60..2e95c33cf20 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -7,6 +7,7 @@ from typing import Final import pytest GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py" +SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py") SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") @@ -53,3 +54,18 @@ def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True) assert result.returncode == 1 + + +def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_path: Path) -> None: + env_path: Final = tmp_path / ".env" + + result: Final = subprocess.run( + [sys.executable, str(SECRETS_TO_ENV), str(env_path)], + input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "::add-mask::sk-0123456789abcdef\n" + assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 7b90b9de5c8..0b8859d9260 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -64,7 +64,7 @@ Repository admins must require the `e2e-changed-tests` status check for merging Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need -Fetched values are masked before use, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs +Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use From a9e918577bb9cc5c35bd4fcf90948aa4522ea059 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:46:51 -0700 Subject: [PATCH 31/43] ci(e2e): run the access_control canary on harness changes and name failed tests A harness-only change (proxy_client.py, conftest.py, pytest.ini, the gateway config, .github/e2e-stack, or the workflow) selected nothing, so the stack was never exercised by the change that touched it. select_tests.py keeps the changed-file rule and adds the access_control suite whenever a harness file changes. The run step now reports the pytest exit code before the evidence check, prints pytest's summary line per pass so the rerun count is visible, and assert_tests_ran.py names each failed or errored test as classname::name --- .github/e2e-stack/assert_tests_ran.py | 4 + .github/e2e-stack/select_tests.py | 31 +++++++ .github/workflows/test-e2e-changed.yml | 29 +++++-- .../test_e2e_changed_gate.py | 85 +++++++++++++++++++ tests/e2e/CONTRIBUTING.md | 6 +- 5 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 .github/e2e-stack/select_tests.py diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py index 7fd3f7c7c33..c4348c20873 100644 --- a/.github/e2e-stack/assert_tests_ran.py +++ b/.github/e2e-stack/assert_tests_ran.py @@ -20,6 +20,10 @@ def main() -> int: collected: Final = sum(case.get("file") == path for case in cases) skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases) _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n") + for case in cases: + if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")): + continue + _ = sys.stdout.write(f" failed: {case.get('classname', '')}::{case.get('name', '')}\n") if ( selected and not missing diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py new file mode 100644 index 00000000000..cbe582b5b42 --- /dev/null +++ b/.github/e2e-stack/select_tests.py @@ -0,0 +1,31 @@ +import re +import sys +from typing import Final + +SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$") +OWN_LANE: Final = re.compile( + r"^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" +) +HARNESS: Final = re.compile( + r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" + r"|^tests/e2e/gateway/" + r"|^\.github/e2e-stack/" + r"|^\.github/workflows/test-e2e-changed\.yml$" +) + + +def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]: + direct: Final = frozenset(path for path in changed if SELECTABLE.match(path) and not OWN_LANE.match(path)) + harness_changed: Final = any(HARNESS.match(path) for path in changed) + canary_tests: Final = frozenset(path for path in canary if harness_changed and SELECTABLE.match(path)) + return tuple(sorted(direct | canary_tests)) + + +def main() -> int: + changed: Final = tuple(line.strip() for line in sys.stdin if line.strip()) + _ = sys.stdout.write(" ".join(select(changed, tuple(sys.argv[1:]))) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 16d84beab2d..23ab6dfcfe4 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -15,11 +15,21 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: + contents: read pull-requests: read outputs: tests: ${{ steps.changed.outputs.tests }} any: ${{ steps.changed.outputs.any }} steps: + - name: Checkout the selector and the canary suite + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: | + .github/e2e-stack + tests/e2e/access_control + persist-credentials: false + ref: ${{ github.sha }} + - name: List the e2e test files this PR added or modified id: changed env: @@ -27,7 +37,6 @@ jobs: REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} - OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$' run: | gh api "repos/${REPO}/pulls/${PR_NUMBER}" \ --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \ @@ -36,9 +45,7 @@ jobs: --jq '.[] | select(.status != "removed") | .filename')" gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}" tests="$(printf '%s\n' "${files}" \ - | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \ - | grep -vE "${OWN_LANE}" \ - | sort -u | tr '\n' ' ' | sed 's/ $//')" || true + | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)" echo "tests=${tests}" >> "${GITHUB_OUTPUT}" if [ -n "${tests}" ]; then echo "any=true" >> "${GITHUB_OUTPUT}" @@ -170,25 +177,29 @@ jobs: read -r -a test_files <<< "${TESTS}" for pass in 1 2 3; do report="${RUNNER_TEMP}/e2e-pass-${pass}.xml" + log="${RUNNER_TEMP}/e2e-pass-${pass}.log" echo "::group::pass ${pass} of 3" set +e uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \ - -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1 + -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1 status=$? + uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}" + verified=$? set -e + grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1 echo "::endgroup::" if [ "${status}" = "5" ]; then echo "::error::the selected files collected no runnable tests, so nothing was verified" exit 1 fi - if ! uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"; then - echo "::error::pass ${pass} of 3 did not verify every selected file" - exit 1 - fi if [ "${status}" != "0" ]; then echo "::error::pass ${pass} of 3 failed with exit code ${status}" exit "${status}" fi + if [ "${verified}" != "0" ]; then + echo "::error::pass ${pass} of 3 did not verify every selected file" + exit 1 + fi echo "pass ${pass} of 3 passed" done diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 2e95c33cf20..f7c4a5e2527 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -8,6 +8,8 @@ import pytest GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py" SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py") +SELECT_TESTS: Final = GATE.with_name("select_tests.py") +CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") @@ -46,6 +48,29 @@ def test_passing_case_does_not_hide_a_failure_in_the_same_file(tmp_path: Path, o assert result.returncode == 1 +def test_failed_cases_are_named_per_selected_file(tmp_path: Path) -> None: + suite: Final = ET.Element("testsuite") + _ = ET.SubElement(suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_ok") + failed: Final = ET.SubElement( + suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_boom" + ) + _ = ET.SubElement(failed, "failure", message="secret-bearing message") + errored: Final = ET.SubElement( + suite, "testcase", file=SELECTED[1], classname="tests.e2e.access_control.test_b", name="test_setup" + ) + _ = ET.SubElement(errored, "error") + report: Final = tmp_path / "report.xml" + ET.ElementTree(suite).write(report) + + result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True) + + assert result.returncode == 1 + assert " failed: tests.e2e.access_control.test_a::test_boom\n" in result.stdout + assert " failed: tests.e2e.access_control.test_b::test_setup\n" in result.stdout + assert "test_ok" not in result.stdout + assert "secret-bearing message" not in result.stdout + + @pytest.mark.parametrize("contents", ("", "')) def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None: report: Final = tmp_path / "report.xml" @@ -69,3 +94,63 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat assert result.returncode == 0, result.stderr assert result.stdout == "::add-mask::sk-0123456789abcdef\n" assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" + + +def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: + result: Final = subprocess.run( + [sys.executable, str(SELECT_TESTS), *CANARY], + input="".join(f"{path}\n" for path in changed), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return tuple(result.stdout.split()) + + +@pytest.mark.parametrize( + ("changed", "expected"), + ( + (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)), + (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()), + (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), + (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/CLAUDE.md"), ()), + ( + ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"), + ("tests/e2e/logging/test_datadog_e2e.py",), + ), + ), +) +def test_changed_suite_files_are_selected_outside_the_own_lane( + changed: tuple[str, ...], expected: tuple[str, ...] +) -> None: + assert select_tests(changed) == expected + + +@pytest.mark.parametrize( + "harness_file", + ( + "tests/e2e/proxy_client.py", + "tests/e2e/conftest.py", + "tests/e2e/pytest.ini", + "tests/e2e/gateway/stage_mirror_ci_config.yml", + ".github/e2e-stack/up.sh", + ".github/workflows/test-e2e-changed.yml", + ), +) +def test_harness_changes_run_the_canary_suite(harness_file: str) -> None: + assert select_tests((harness_file, "litellm/router.py")) == CANARY + + +def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() -> None: + assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY + + +def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None: + assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == ( + *CANARY, + "tests/e2e/logging/test_datadog_e2e.py", + ) + + +def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None: + assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py") diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 0b8859d9260..be282e2a77e 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -56,15 +56,15 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack -Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch +Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need -Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs +Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, pytest's summary line, failed test ids, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use From b2e93ba99f3e8ba8e87809d433d2bb9db5cd791a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:58:07 -0700 Subject: [PATCH 32/43] ci(e2e): declare the embedding model the access_control canary calls The first canary run failed pass 1 because the stage-mirror config had no openai-text-embedding-3-small while test_llm_api_routes_group_grants_every_llm_endpoint calls /embeddings with it; the public log named the test, which is the behavior the previous commit added --- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/gateway/stage_mirror_ci_config.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index be282e2a77e..106cfa31f0e 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -56,7 +56,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 6b3756b5f9f..229e8514dee 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -52,6 +52,10 @@ model_list: litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY + - model_name: openai-text-embedding-3-small + litellm_params: + model: openai/text-embedding-3-small + api_key: os.environ/OPENAI_API_KEY mcp_servers: devin: From 79d47788d9eea16c0038fc7eed10af37b5f8889b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:56:39 -0700 Subject: [PATCH 33/43] fix(anthropic): stream the refusal text on bridged /v1/messages calls Both bridges opened an empty text block on a refused streaming turn and closed it without a single delta, so a client replaying that assistant turn got HTTP 400 "text content blocks must be non-empty" from Anthropic. The safeguard-refusal fallback that motivated withholding the text only runs on the awaited non-streaming response, so nothing needed it withheld Move the refusal readers into the shared messages/utils helpers so the adapters stop reaching into each other's private statics, which is also what put reportPrivateUsage over its budget --- .../adapters/streaming_iterator.py | 16 +++-- .../adapters/transformation.py | 36 ++++------ .../messages/utils.py | 70 ++++++++++++++++++- .../responses_adapters/streaming_iterator.py | 32 ++++++--- .../responses_adapters/transformation.py | 50 ++----------- .../test_streaming_iterator_first_delta.py | 8 +-- ...t_responses_adapters_streaming_iterator.py | 6 +- 7 files changed, 126 insertions(+), 92 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index adf77dc3dd5..c6d744772a5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -1006,6 +1006,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": return processed_chunk + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + ) + return cast( # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch ContentBlockDelta | MessageBlockDelta, { # mutable-ok: fresh translation payload; never mutated after construction @@ -1013,11 +1017,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": { # mutable-ok: fresh stop_details payload; never mutated after construction - "type": "refusal", - "category": None, - "explanation": "".join(self._refusal_text_parts), - }, + "stop_details": refusal_stop_details("".join(self._refusal_text_parts)), }, }, ) @@ -1098,9 +1098,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): - Different content types in the response - Specific markers in the content """ + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + from .transformation import LiteLLMAnthropicMessagesAdapter - refusal_text: Final = LiteLLMAnthropicMessagesAdapter._refusal_text(chunk.choices[0].delta) + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) if refusal_text is not None: self._refusal_text_parts.append(refusal_text) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 215e3ded908..e3bad54d599 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -117,6 +117,10 @@ from litellm.llms.anthropic.common_utils import ( from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + refusal_stop_details, +) from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, AllAnthropicPassThroughMessageValues, @@ -308,17 +312,6 @@ class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass - @staticmethod - def _refusal_text(message_or_delta: object) -> str | None: - refusal: Final = getattr(message_or_delta, "refusal", None) - if isinstance(refusal, str): - return refusal - provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None) - if isinstance(provider_specific_fields, Mapping): - provider_refusal: Final = provider_specific_fields.get("refusal") - return provider_refusal if isinstance(provider_refusal, str) else None - return None - ### FOR [BETA] `/v1/messages` endpoint support def _extract_signature_from_tool_call(self, tool_call: object) -> str | None: @@ -1325,7 +1318,7 @@ class LiteLLMAnthropicMessagesAdapter: new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) - if (refusal_text := self._refusal_text(choice.message)) is not None: + if (refusal_text := openai_chat_refusal_text(choice.message)) is not None: new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) # Handle tool calls (in parallel to text content) if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: @@ -1486,7 +1479,7 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping=tool_name_mapping, ) refusal_text: Final = next( - (text for choice in response.choices if (text := self._refusal_text(choice.message)) is not None), + (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None), None, ) @@ -1523,15 +1516,7 @@ class LiteLLMAnthropicMessagesAdapter: usage=anthropic_usage, content=anthropic_content, stop_reason=anthropic_finish_reason, - stop_details=( - { # mutable-ok: fresh refusal stop_details payload built per response - "type": "refusal", - "category": None, - "explanation": refusal_text, - } - if anthropic_finish_reason == "refusal" - else None - ), + stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None), ) applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None @@ -1572,7 +1557,7 @@ class LiteLLMAnthropicMessagesAdapter: "signature": thought_sig, } return "tool_use", cast("ContentBlockContentBlockDict", tool_block) - elif (choice.delta.content is not None and len(choice.delta.content) > 0) or self._refusal_text( + elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text( choice.delta ) is not None: return "text", TextBlock(type="text", text="") @@ -1646,7 +1631,10 @@ class LiteLLMAnthropicMessagesAdapter: elif reasoning_content: return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content) else: - return "text_delta", ContentTextBlockDelta(type="text_delta", text=text) + refusal_text: Final = "".join( + refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None + ) + return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text) def translate_streaming_openai_response_to_anthropic( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index 242300c7b6d..7545dff1408 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -1,8 +1,11 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints -from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams +from litellm.types.llms.anthropic import ( + AnthropicMessagesRequestOptionalParams, + AnthropicStopDetails, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -25,6 +28,69 @@ def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] | return stop_details if isinstance(stop_details, dict) else None +def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails: + """The ``stop_details`` object accompanying a translated ``stop_reason: "refusal"``.""" + return AnthropicStopDetails(type="refusal", category=None, explanation=explanation) + + +def _mapping_field(container: object, key: str) -> object | None: + """One key of a raw provider payload, or None when the payload is not a mapping.""" + if not isinstance(container, Mapping): + return None + return cast(Mapping[str, object], container).get(key) # cast-ok: raw payload, callers re-check every value + + +def _mapping_str_field(container: object, key: str) -> str | None: + value: Final = _mapping_field(container, key) + return value if isinstance(value, str) and value else None + + +def openai_chat_refusal_text(message_or_delta: object) -> str | None: + """ + Refusal text carried by an OpenAI Chat Completions message or streaming delta, + read from ``refusal`` or from the ``provider_specific_fields`` LiteLLM parks it + in, or None when the turn is not a refusal. + """ + refusal: Final = getattr(message_or_delta, "refusal", None) + if isinstance(refusal, str) and refusal: + return refusal + return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal") + + +def _responses_message_refusal_text(item: object) -> str | None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + if isinstance(item, ResponseOutputMessage): + return next( + (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal) and part.refusal), + None, + ) + raw_parts: Final = _mapping_field(item, "content") + if _mapping_str_field(item, "type") != "message" or not isinstance(raw_parts, Sequence): + return None + return next( + ( + refusal + for part in cast(Sequence[object], raw_parts) # cast-ok: members re-validated below + if _mapping_str_field(part, "type") == "refusal" + and isinstance(refusal := _mapping_str_field(part, "refusal"), str) + ), + None, + ) + + +def responses_output_refusal_text(output: Iterable[object]) -> str | None: + """ + Refusal text carried by an OpenAI Responses ``output`` list, in typed + (``ResponseOutputRefusal``) or raw-dictionary shape, or None when none of the + output messages refused. + """ + return next( + (text for item in output if (text := _responses_message_refusal_text(item)) is not None), + None, + ) + + def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError": """The exception a safeguard-refused Anthropic response converts into so the content-policy fallback chain can re-dispatch it.""" diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 06ffcd26d40..5bd662e0f94 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,6 +9,10 @@ from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -136,8 +140,20 @@ class AnthropicResponsesStreamWrapper: if event_type == "response.refusal.delta": delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") - if isinstance(delta, str): - self._refusal_text_parts.append(delta) + if not isinstance(delta, str) or not delta: + return + self._refusal_text_parts.append(delta) + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + block_idx = self._open_block(item_id, {"type": "text", "text": ""}) + self._chunk_queue.append( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + } + ) return # ---- text delta ---- @@ -225,9 +241,7 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () - refusal_text: Final = LiteLLMAnthropicToResponsesAPIAdapter._refusal_text_from_output(output) or ( - "".join(self._refusal_text_parts) or None - ) + refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None) status: Final = getattr(response_obj, "status", None) if response_obj is not None else None has_tool_call: Final = any( getattr(item, "type", None) == "function_call" @@ -255,12 +269,8 @@ class AnthropicResponsesStreamWrapper: "stop_reason": stop_reason, "stop_sequence": None, **( - { # mutable-ok: fresh refusal stop_details payload built per chunk - "stop_details": { # mutable-ok: fresh refusal stop_details payload built per chunk - "type": "refusal", - "category": None, - "explanation": refusal_text, - } + { # mutable-ok: fresh message_delta stop_details entry built per chunk + "stop_details": refusal_stop_details(refusal_text) } if stop_reason == "refusal" else {} # mutable-ok: empty spread placeholder for non-refusal stop diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 318065148b8..9fb44127f5b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,7 +6,7 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping from itertools import groupby from typing import Any, Final, cast @@ -19,6 +19,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, prompt_cache_key_from_user_id, @@ -69,38 +73,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage) - @staticmethod - def _refusal_text_from_output(output: Iterable[object]) -> str | None: - from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal - - def refusal_text_from_item(item: object) -> str | None: - if isinstance(item, ResponseOutputMessage): - return next( - (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal)), - None, - ) - if not isinstance(item, Mapping): - return None - item_mapping: Final = cast(Mapping[str, object], item) # cast-ok: keys re-checked before use - raw_parts: Final = item_mapping.get("content") - if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): - return None - for part in cast(Sequence[object], raw_parts): # cast-ok: members re-validated below - if not isinstance(part, Mapping): - continue - part_mapping = cast(Mapping[str, object], part) # cast-ok: keys re-checked before use - if part_mapping.get("type") != "refusal": - continue - refusal = part_mapping.get("refusal") - if isinstance(refusal, str): - return refusal - return None - - return next( - (text for item in output if (text := refusal_text_from_item(item)) is not None), - None, - ) - # ------------------------------------------------------------------ # # Request translation: Anthropic -> Responses API # # ------------------------------------------------------------------ # @@ -656,7 +628,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" - refusal_text: Final = self._refusal_text_from_output( + refusal_text: Final = responses_output_refusal_text( cast(Iterable[object], response.output) # cast-ok: output items re-validated per item ) @@ -747,13 +719,5 @@ class LiteLLMAnthropicToResponsesAPIAdapter: usage=anthropic_usage, content=content, stop_reason=stop_reason, - stop_details=( - { # mutable-ok: fresh refusal stop_details payload built per response - "type": "refusal", - "category": None, - "explanation": refusal_text, - } - if stop_reason == "refusal" - else None - ), + stop_details=(refusal_stop_details(refusal_text) if stop_reason == "refusal" else None), ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index e7381986ec2..4359ace1870 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -108,7 +108,7 @@ def _text_deltas(events: List[dict]) -> List[str]: ] -def test_streaming_chat_refusal_emits_only_refusal_stop_details(): +def test_streaming_chat_refusal_emits_refusal_text_and_stop_details(): chunks = [ _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), _make_chunk(Delta(content=None), finish_reason="stop"), @@ -117,7 +117,7 @@ def test_streaming_chat_refusal_emits_only_refusal_stop_details(): events = _drain_sync(wrapper) - assert _text_deltas(events) == [] + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"] == { "stop_reason": "refusal", @@ -130,7 +130,7 @@ def test_streaming_chat_refusal_emits_only_refusal_stop_details(): @pytest.mark.asyncio -async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): +async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async(): chunks = [ _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), _make_chunk(Delta(content=None), finish_reason="stop"), @@ -139,7 +139,7 @@ async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): events = await _drain_async(wrapper) - assert _text_deltas(events) == [] + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index d8f16dde3e7..d9df5df426a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -328,7 +328,7 @@ class TestResponseCompletedUsage: class TestRefusalStreamEvents: - def test_refusal_event_sequence_emits_only_stop_details(self): + def test_refusal_event_sequence_emits_refusal_text_and_stop_details(self): response = SimpleNamespace( status="completed", output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}], @@ -346,11 +346,13 @@ class TestRefusalStreamEvents: assert [chunk["type"] for chunk in chunks] == [ "message_start", "content_block_start", + "content_block_delta", "content_block_stop", "message_delta", "message_stop", ] - assert chunks[3]["delta"] == { + assert chunks[2]["delta"] == {"type": "text_delta", "text": "I cannot fulfill this."} + assert chunks[4]["delta"] == { "stop_reason": "refusal", "stop_sequence": None, "stop_details": { From 116f88b023996a52a46f02b2dd8bffc8c15c9246 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:03:50 -0700 Subject: [PATCH 34/43] fix(e2e-changed): keep the gate off suites the stack cannot run The selector picked up two suites that can never pass in this stack, so editing either one turned the check permanently red: the presidio masking suite calls pytest.fail without an analyzer and anonymizer that up.sh never starts, and the pipecat audio suite skips itself at import time unless the NLTK punkt_tab data is present, which nothing installs. tests/e2e/coverage_registry/test_collector.py had the same problem for a different reason. Its nested pytest.main autoloads pytest-retry from the ci group the workflow installs and dies with "INTERNALERROR: no option named 'filtered_exceptions'", so the collect-only pass now disables that plugin. The plugin's entry point is pytest-retry, not retry, so the same one-word fix lands on mutmut's pytest_add_cli_args, where "-p no:retry" was disabling nothing. Two smaller holes in the harness: a canary argument the shell never expanded used to select nothing and let the gate pass green, and a secret that cannot be represented in both bash and dotenv was rejected without naming the key. --- .github/e2e-stack/secrets_to_env.py | 13 +++-- .github/e2e-stack/select_tests.py | 23 +++++++-- pyproject.toml | 2 +- .../test_e2e_changed_gate.py | 49 ++++++++++++++++++- tests/e2e/CONTRIBUTING.md | 4 +- tests/e2e/coverage_registry/collector.py | 2 + 6 files changed, 80 insertions(+), 13 deletions(-) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index e7870a32e42..7913b25918b 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -20,10 +20,15 @@ def main() -> int: except (ValidationError, UnicodeError): _ = sys.stderr.write("expected a JSON object containing string environment values\n") return 1 - if any( - ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") for key, value in secrets.items() - ): - _ = sys.stderr.write("environment names or values cannot be represented in both bash and dotenv\n") + unusable: Final = tuple( + key + for key, value in secrets.items() + if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") + ) + if unusable: + _ = sys.stderr.write( + f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n" + ) return 1 for value in secrets.values(): if len(value) >= MIN_MASKED_LENGTH: diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index cbe582b5b42..a62358f81ff 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -3,8 +3,11 @@ import sys from typing import Final SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$") -OWN_LANE: Final = re.compile( - r"^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" +UNSUPPORTED: Final = re.compile( + r"^tests/e2e/(ui|claude_code|load)/" + r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" + r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" + r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" @@ -12,18 +15,28 @@ HARNESS: Final = re.compile( r"|^\.github/e2e-stack/" r"|^\.github/workflows/test-e2e-changed\.yml$" ) +UNEXPANDED: Final = re.compile(r"[*?\[]") + + +def is_selectable(path: str) -> bool: + return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]: - direct: Final = frozenset(path for path in changed if SELECTABLE.match(path) and not OWN_LANE.match(path)) + direct: Final = frozenset(path for path in changed if is_selectable(path)) harness_changed: Final = any(HARNESS.match(path) for path in changed) - canary_tests: Final = frozenset(path for path in canary if harness_changed and SELECTABLE.match(path)) + canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path)) return tuple(sorted(direct | canary_tests)) def main() -> int: + canary: Final = tuple(sys.argv[1:]) + unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path)) + if unexpanded: + _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n") + return 1 changed: Final = tuple(line.strip() for line in sys.stdin if line.strip()) - _ = sys.stdout.write(" ".join(select(changed, tuple(sys.argv[1:]))) + "\n") + _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n") return 0 diff --git a/pyproject.toml b/pyproject.toml index b889a3a0e60..f4f238dd4b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -388,7 +388,7 @@ mutate_only_covered_lines = true # registered hash algorithm". Nothing to do with mutation coverage, and one # erroring test is enough to end the stats phase before any mutant runs. pytest_add_cli_args = [ - "-p", "no:retry", + "-p", "no:pytest-retry", "-p", "no:rerunfailures", "-p", "no:xdist", "--ignore=tests/test_litellm/proxy/management_endpoints/test_saml_sso.py", diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index f7c4a5e2527..d2b842364a6 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -113,6 +113,16 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)), (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()), (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), + (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), + (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), + ( + ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), + ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), + ), + ( + ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",), + ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",), + ), (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/CLAUDE.md"), ()), ( ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"), @@ -120,7 +130,7 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: ), ), ) -def test_changed_suite_files_are_selected_outside_the_own_lane( +def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them( changed: tuple[str, ...], expected: tuple[str, ...] ) -> None: assert select_tests(changed) == expected @@ -154,3 +164,40 @@ def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None: def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None: assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py") + + +def test_a_canary_argument_the_shell_never_expanded_fails_the_selector() -> None: + result: Final = subprocess.run( + [sys.executable, str(SELECT_TESTS), "tests/e2e/access_control/test_*.py"], + input="tests/e2e/proxy_client.py\n", + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "tests/e2e/access_control/test_*.py" in result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize( + ("secrets", "offender", "unprintable"), + ( + ('{"AWS_ACCESS_KEY_ID": "AKIAEXAMPLE", "BAD-NAME": "shibboleth"}', "BAD-NAME", "shibboleth"), + ("""{"AWS_SECRET_ACCESS_KEY": "quote'shibboleth"}""", "AWS_SECRET_ACCESS_KEY", "shibboleth"), + ('{"DD_API_KEY": "line\\nshibboleth"}', "DD_API_KEY", "shibboleth"), + ), +) +def test_an_unusable_secret_is_named_without_printing_its_value( + tmp_path: Path, secrets: str, offender: str, unprintable: str +) -> None: + env_path: Final = tmp_path / ".env" + + result: Final = subprocess.run( + [sys.executable, str(SECRETS_TO_ENV), str(env_path)], input=secrets, capture_output=True, text=True + ) + + assert result.returncode == 1 + assert offender in result.stderr + assert unprintable not in result.stderr + assert result.stdout == "" + assert not env_path.exists() diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 106cfa31f0e..b83300cc8d5 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -56,9 +56,9 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start -Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch +Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index e20f7884f55..c5d09820fe7 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -105,6 +105,8 @@ def collect_markers(e2e_dir: Path = E2E_DIR) -> CollectedMarkers: "--continue-on-collection-errors", "-p", "no:cacheprovider", + "-p", + "no:pytest-retry", str(e2e_dir), ], plugins=[sink], From 82edb9e901c7b87a99a7b5c318b3db333ca6165b Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 6 Sep 2026 00:09:27 -0500 Subject: [PATCH 35/43] fix(vertex): preserve Lyria pricing fallback and audio MIME --- litellm/cost_calculator.py | 14 +++---- litellm/llms/vertex_ai/common_utils.py | 22 ++++++++++- .../text_to_speech/transformation.py | 4 +- .../vertex_passthrough_logging_handler.py | 6 +-- ...test_vertex_passthrough_logging_handler.py | 27 +++++++++++--- .../text_to_speech/test_transformation.py | 17 +++++++++ tests/test_litellm/test_cost_calculator.py | 37 +++++++++++++++++-- 7 files changed, 103 insertions(+), 24 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 5dcbb1d5f37..fa8e1a5264c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -495,17 +495,15 @@ def cost_per_token( # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models if call_type == "speech" or call_type == "aspeech": + if custom_llm_provider in ("vertex_ai", "vertex_ai_beta"): + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost + + lyria_generation_cost: Final = get_vertex_ai_lyria_generation_cost(model_without_prefix) + if lyria_generation_cost is not None: + return 0.0, lyria_generation_cost speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) prompt_cost: float = 0.0 completion_cost: float = 0.0 - if not speech_model_info.get("input_cost_per_character") and not speech_model_info.get("input_cost_per_token"): - output_cost_per_generation: Final = speech_model_info.get("output_cost_per_image") - speech_output_cost_per_second: Final = speech_model_info.get("output_cost_per_second") - audio_seconds_per_prediction: Final = speech_model_info.get("audio_seconds_per_prediction") - if output_cost_per_generation is not None: - return prompt_cost, float(output_cost_per_generation) - if speech_output_cost_per_second is not None and audio_seconds_per_prediction is not None: - return prompt_cost, float(speech_output_cost_per_second) * float(audio_seconds_per_prediction) cost_metric: Final = select_cost_metric_for_model(speech_model_info) if cost_metric == "cost_per_character": if prompt_characters is None: diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index b649d8dac0e..3895ea25434 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -2,6 +2,7 @@ import re from copy import deepcopy from enum import Enum from functools import lru_cache +from types import MappingProxyType from typing import Any, Final, Literal, cast, get_type_hints import httpx @@ -55,7 +56,26 @@ def _get_bundled_vertex_ai_lyria_model_info(model_key: str) -> VertexAILyriaMode def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) - return runtime_model_info or _get_bundled_vertex_ai_lyria_model_info(model_key) + bundled_model_info: Final = _get_bundled_vertex_ai_lyria_model_info(model_key) + if runtime_model_info is None: + return bundled_model_info + if bundled_model_info is None: + return runtime_model_info + return _validate_vertex_ai_lyria_model_info(MappingProxyType({**bundled_model_info, **runtime_model_info})) + + +def get_vertex_ai_lyria_generation_cost(model: str) -> float | None: + model_info: Final = get_vertex_ai_lyria_model_info(model) + if model_info is None: + return None + generation_cost: Final = model_info.get("output_cost_per_image") + if generation_cost is not None: + return generation_cost + cost_per_second: Final = model_info.get("output_cost_per_second") + seconds_per_prediction: Final = model_info.get("audio_seconds_per_prediction") + if cost_per_second is None or seconds_per_prediction is None: + return None + return cost_per_second * seconds_per_prediction class VertexAIError(BaseLLMException): diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 2047be2ea37..5a93aca3ac9 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -681,9 +681,11 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): ) # rebind-ok: interactions response supplies its audio MIME type if audio_data is None: raise ValueError(f"No generated audio found in Vertex AI {base_model} response") + decoded_audio: Final = base64.b64decode(audio_data) default_format: Final = model_info["supported_audio_formats"][0] mime_type = ( mime_type + or speech_media_type_from_audio_bytes(decoded_audio) or { # mutable-ok: short-lived lookup selects the default response MIME type; rebind-ok: absent provider MIME type falls back to model metadata "mp3": "audio/mpeg", "wav": "audio/wav", @@ -692,7 +694,7 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): response: Final = HttpxBinaryResponseContent( httpx.Response( status_code=raw_response.status_code, - content=base64.b64decode(audio_data), + content=decoded_audio, headers={ # mutable-ok: httpx requires a concrete response header dictionary "content-type": mime_type }, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index f7625d71168..afdbaf3b06c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -12,7 +12,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.vertex_ai.common_utils import ( - get_vertex_ai_lyria_model_info, + get_vertex_ai_lyria_generation_cost, get_vertex_location_from_url, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -397,9 +397,7 @@ class VertexPassthroughLoggingHandler: ) if runtime_unit_cost is not None: return runtime_unit_cost - return VertexPassthroughLoggingHandler._audio_prediction_unit_cost_from_model_info( - model_info=get_vertex_ai_lyria_model_info(model=model) - ) + return get_vertex_ai_lyria_generation_cost(model=model) @staticmethod def _audio_prediction_unit_cost_from_model_info(model_info: object) -> float | None: diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index f3d28d58c22..179301a9c81 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -146,13 +146,27 @@ def test_audio_predict_response_supports_bytes_base64_encoded( assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) -def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_map_omits_model( +@pytest.mark.parametrize( + "missing_fields", + ( + None, + ("output_cost_per_second",), + ("audio_seconds_per_prediction",), + ("output_cost_per_second", "audio_seconds_per_prediction"), + ), +) +def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_incomplete( monkeypatch: pytest.MonkeyPatch, + missing_fields: tuple[str, ...] | None, ) -> None: - stale_runtime_model_cost: Final = { - key: value for key, value in litellm.model_cost.items() if key != "vertex_ai/lyria-002" - } - monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost) + if missing_fields is None: + monkeypatch.delitem(litellm.model_cost, "vertex_ai/lyria-002") + else: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + {key: value for key, value in litellm.model_cost["vertex_ai/lyria-002"].items() if key not in missing_fields}, + ) logging_obj = MagicMock() logging_obj.model_call_details = {} response = httpx.Response( @@ -178,7 +192,8 @@ def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_map_omits_mod request_body={"instances": [{"prompt": "ambient piano"}]}, ) - assert "vertex_ai/lyria-002" not in litellm.model_cost + if missing_fields is None: + assert "vertex_ai/lyria-002" not in litellm.model_cost assert result["kwargs"]["model"] == "lyria-002" assert result["kwargs"]["response_cost"] == pytest.approx(0.06) assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 457c3f76dbb..4f4df69780a 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -169,6 +169,23 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): class TestVertexAILyriaTextToSpeechConfig: + def test_response_without_mime_type_uses_audio_container(self) -> None: + audio: Final = b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00" + raw_response: Final = httpx.Response( + 200, + json={"outputs": [{"type": "audio", "data": base64.b64encode(audio).decode()}]}, + ) + + response: Final = VertexAILyriaTextToSpeechConfig().transform_text_to_speech_response( + model="lyria-3-pro-preview", + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert response.content == audio + assert response.response.headers["content-type"] == "audio/wav" + assert response._hidden_params["audio_mime_type"] == "audio/wav" + @pytest.mark.parametrize( "model", ["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"], diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ffc71c76d03..aa630c4916b 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,6 +1,7 @@ import json from pathlib import Path +from typing import Final import pytest @@ -154,14 +155,42 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): ("vertex_ai/lyria-3-pro-preview", 0.08), ], ) -def test_vertex_lyria_speech_cost(model, expected_cost, _local_model_cost_map): - cost = completion_cost( +@pytest.mark.parametrize("runtime_state", ("complete", "missing", "routing_only", "custom_zero", "custom_price")) +@pytest.mark.parametrize("call_type", ("speech", "aspeech")) +def test_vertex_lyria_speech_cost( + model: str, + expected_cost: float, + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + runtime_state: str, + call_type: str, +) -> None: + model_info: Final = litellm.model_cost[model] + if runtime_state == "missing": + monkeypatch.delitem(litellm.model_cost, model) + elif runtime_state == "routing_only": + monkeypatch.setitem( + litellm.model_cost, + model, + { + key: value + for key, value in model_info.items() + if key not in ("output_cost_per_image", "output_cost_per_second", "audio_seconds_per_prediction") + }, + ) + elif runtime_state in ("custom_zero", "custom_price"): + cost_key: Final = "output_cost_per_image" if "output_cost_per_image" in model_info else "output_cost_per_second" + multiplier: Final = 0 if runtime_state == "custom_zero" else 2 + monkeypatch.setitem(litellm.model_cost, model, {**model_info, cost_key: model_info[cost_key] * multiplier}) + + cost: Final = completion_cost( model=model, prompt="A bright synth track", - call_type="speech", + call_type=call_type, ) - assert cost == pytest.approx(expected_cost) + expected: Final = 0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1) + assert cost == pytest.approx(expected) def test_baseten_model_api_pricing_entries(_local_model_cost_map): From fd24cce2c3d0893e70d88f1e2d6ab3427a09736c Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sun, 6 Sep 2026 00:15:08 -0500 Subject: [PATCH 36/43] test(vertex): isolate Lyria fallback from the remote catalog --- .../llms/vertex_ai/test_vertex_passthrough_logging_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 179301a9c81..2cf6689261e 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -158,6 +158,7 @@ def test_audio_predict_response_supports_bytes_base64_encoded( def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_incomplete( monkeypatch: pytest.MonkeyPatch, missing_fields: tuple[str, ...] | None, + local_model_cost_map: None, ) -> None: if missing_fields is None: monkeypatch.delitem(litellm.model_cost, "vertex_ai/lyria-002") From 6be78fa850d4afc89527488115acb5117a6a34be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:34:31 -0700 Subject: [PATCH 37/43] fix(vertex_ai): bill Lyria per generation, not per audio second Google prices Lyria per generated clip, so every Vertex Lyria entry in the price map now carries a single output_cost_per_image and both the speech and the passthrough cost paths read that one field. The old output_cost_per_second and audio_seconds_per_prediction pair assumed a 30 second clip, which does not match the 32.768 second WAV Vertex returns, and no other model in the map priced audio that way Drops max_audio_length_hours and max_audio_per_prompt from the price map, its schema, the generator, and ModelInfo, since nothing reads them, and drops the audio_mime_type hidden param for the same reason: the response already carries the resolved content type on its own header Folds the per-model bundled catalog lookups into one cached parse of the local cost map, validated with a TypeAdapter over a ReadOnly TypedDict --- ci_cd/generate_model_prices_schema.py | 12 ----- litellm/cost_calculator.py | 16 +++--- litellm/llms/vertex_ai/common_utils.py | 51 ++++++++++--------- .../text_to_speech/transformation.py | 27 +++------- litellm/main.py | 13 ++--- ...odel_prices_and_context_window_backup.json | 7 +-- .../vertex_passthrough_logging_handler.py | 28 +--------- litellm/proxy/proxy_server.py | 11 ++-- litellm/types/llms/openai.py | 3 -- litellm/types/utils.py | 3 -- litellm/utils.py | 3 -- model_prices_and_context_window.json | 7 +-- model_prices_and_context_window.schema.json | 15 ------ .../vertex_ai/test_vertex_ai_common_utils.py | 41 +++++++++++++++ ...test_vertex_passthrough_logging_handler.py | 41 ++++++++------- .../text_to_speech/test_transformation.py | 36 ++++++------- tests/test_litellm/test_cost_calculator.py | 13 +++-- tests/test_litellm/test_utils.py | 6 +-- 18 files changed, 141 insertions(+), 192 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index ee0dad25c81..ab29b70bdd4 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -121,10 +121,6 @@ ARRAY_KEYS: dict[str, JsonSchema] = { } INTEGER_KEYS: dict[str, JsonSchema] = { - "max_audio_per_prompt": { - **NONNEG_INTEGER, - "description": "Maximum number of audio outputs accepted or generated per prompt.", - }, "max_tokens": { **NONNEG_INTEGER, "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.", @@ -150,14 +146,6 @@ INTEGER_KEYS: dict[str, JsonSchema] = { } NUMBER_KEYS: dict[str, JsonSchema] = { - "audio_seconds_per_prediction": { - **NONNEG_NUMBER, - "description": "Audio duration, in seconds, produced by one prediction.", - }, - "max_audio_length_hours": { - **NONNEG_NUMBER, - "description": "Maximum generated audio duration, expressed in hours.", - }, "regional_processing_uplift_multiplier_eu": { "type": "number", "minimum": 1, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index fa8e1a5264c..eabf2c6249b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -80,6 +80,7 @@ from litellm.llms.together_ai.cost_calculator import ( get_model_params_and_category, has_together_registry_pricing, ) +from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost from litellm.llms.vertex_ai.cost_calculator import ( cost_per_character as google_cost_per_character, ) @@ -495,16 +496,17 @@ def cost_per_token( # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models if call_type == "speech" or call_type == "aspeech": - if custom_llm_provider in ("vertex_ai", "vertex_ai_beta"): - from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost - - lyria_generation_cost: Final = get_vertex_ai_lyria_generation_cost(model_without_prefix) - if lyria_generation_cost is not None: - return 0.0, lyria_generation_cost + lyria_generation_cost: Final = ( + get_vertex_ai_lyria_generation_cost(model=model_without_prefix) + if custom_llm_provider in ("vertex_ai", "vertex_ai_beta") + else None + ) + if lyria_generation_cost is not None: + return 0.0, lyria_generation_cost speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) + cost_metric: Final = select_cost_metric_for_model(speech_model_info) prompt_cost: float = 0.0 completion_cost: float = 0.0 - cost_metric: Final = select_cost_metric_for_model(speech_model_info) if cost_metric == "cost_per_character": if prompt_characters is None: raise ValueError( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3895ea25434..38cdf334cc3 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1,4 +1,5 @@ import re +from collections.abc import Mapping from copy import deepcopy from enum import Enum from functools import lru_cache @@ -29,8 +30,6 @@ class VertexAILyriaModelInfo(TypedDict): vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"]] supported_audio_formats: ReadOnly[tuple[Literal["mp3", "wav"], ...]] output_cost_per_image: NotRequired[ReadOnly[float]] - output_cost_per_second: NotRequired[ReadOnly[float]] - audio_seconds_per_prediction: NotRequired[ReadOnly[float]] _VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER: Final = TypeAdapter(VertexAILyriaModelInfo) @@ -45,37 +44,41 @@ def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyri return None -@lru_cache(maxsize=32) -def _get_bundled_vertex_ai_lyria_model_info(model_key: str) -> VertexAILyriaModelInfo | None: +@lru_cache(maxsize=1) +def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]: from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - bundled_model_info: Final = GetModelCostMap.load_local_model_cost_map().get(model_key) - return _validate_vertex_ai_lyria_model_info(bundled_model_info) + return MappingProxyType( + { + model_key: lyria_model_info + for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items() + if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None + } + ) + + +def _vertex_ai_lyria_model_key(model: str) -> str: + return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + + +def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None: + return None if model_info is None else model_info.get("output_cost_per_image") def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: - model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + model_key: Final = _vertex_ai_lyria_model_key(model) runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) - bundled_model_info: Final = _get_bundled_vertex_ai_lyria_model_info(model_key) - if runtime_model_info is None: - return bundled_model_info - if bundled_model_info is None: - return runtime_model_info - return _validate_vertex_ai_lyria_model_info(MappingProxyType({**bundled_model_info, **runtime_model_info})) + return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key) def get_vertex_ai_lyria_generation_cost(model: str) -> float | None: - model_info: Final = get_vertex_ai_lyria_model_info(model) - if model_info is None: - return None - generation_cost: Final = model_info.get("output_cost_per_image") - if generation_cost is not None: - return generation_cost - cost_per_second: Final = model_info.get("output_cost_per_second") - seconds_per_prediction: Final = model_info.get("audio_seconds_per_prediction") - if cost_per_second is None or seconds_per_prediction is None: - return None - return cost_per_second * seconds_per_prediction + model_key: Final = _vertex_ai_lyria_model_key(model) + runtime_cost: Final = _vertex_ai_lyria_generation_cost( + _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) + ) + if runtime_cost is not None: + return runtime_cost + return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key)) class VertexAIError(BaseLLMException): diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 5a93aca3ac9..f489366b8d8 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -15,6 +15,7 @@ import httpx import litellm from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( + DEFAULT_SPEECH_MEDIA_TYPE, speech_media_type_from_audio_bytes, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -571,13 +572,11 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): VertexAIInteractionsConfig, ) - resolved_project: Final = project - def mint_access_token( _credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, ) -> tuple[str, str]: - return "", project_id or resolved_project + return "", project_id or project return VertexAIInteractionsConfig(mint_access_token=mint_access_token).get_complete_url( api_base=api_base, @@ -681,24 +680,12 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): ) # rebind-ok: interactions response supplies its audio MIME type if audio_data is None: raise ValueError(f"No generated audio found in Vertex AI {base_model} response") - decoded_audio: Final = base64.b64decode(audio_data) - default_format: Final = model_info["supported_audio_formats"][0] - mime_type = ( - mime_type - or speech_media_type_from_audio_bytes(decoded_audio) - or { # mutable-ok: short-lived lookup selects the default response MIME type; rebind-ok: absent provider MIME type falls back to model metadata - "mp3": "audio/mpeg", - "wav": "audio/wav", - }[default_format] - ) - response: Final = HttpxBinaryResponseContent( + binary_data: Final = base64.b64decode(audio_data) + media_type: Final = mime_type or speech_media_type_from_audio_bytes(binary_data) or DEFAULT_SPEECH_MEDIA_TYPE + return HttpxBinaryResponseContent( httpx.Response( status_code=raw_response.status_code, - content=decoded_audio, - headers={ # mutable-ok: httpx requires a concrete response header dictionary - "content-type": mime_type - }, + content=binary_data, + headers=MappingProxyType({"content-type": media_type}), ) ) - response.set_audio_mime_type(mime_type) - return response diff --git a/litellm/main.py b/litellm/main.py index 040af897256..44afce08c0a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8260,14 +8260,11 @@ def speech( # Vertex AI Text-to-Speech (Google Cloud TTS) if text_to_speech_provider_config is None: - if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): - text_to_speech_provider_config = ( - VertexAILyriaTextToSpeechConfig() - ) # rebind-ok: model metadata selects the Lyria provider implementation - else: - text_to_speech_provider_config = ( - VertexAITextToSpeechConfig() - ) # rebind-ok: non-Lyria Vertex models use the standard TTS implementation + text_to_speech_provider_config = ( # rebind-ok: model metadata selects the Vertex TTS implementation + VertexAILyriaTextToSpeechConfig() + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model) + else VertexAITextToSpeechConfig() + ) # Cast to specific Vertex AI config type to access dispatch method vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9986d5056d1..0bc6c59a4b7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45512,12 +45512,9 @@ "supports_tool_choice": true }, "vertex_ai/lyria-002": { - "audio_seconds_per_prediction": 30, "litellm_provider": "vertex_ai", - "max_audio_length_hours": 0.009111111111111111, - "max_audio_per_prompt": 4, - "mode": "audio_speech", - "output_cost_per_second": 0.002, + "mode": "audio_speech", + "output_cost_per_image": 0.06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_audio_formats": [ "wav" diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index afdbaf3b06c..119a53c2411 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -1,6 +1,5 @@ import asyncio import re -from collections.abc import Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -346,9 +345,7 @@ class VertexPassthroughLoggingHandler: prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count( json_response=json_response ) - response_cost: Final = ( - VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) or 0.0 - ) * prediction_count + response_cost: Final = (get_vertex_ai_lyria_generation_cost(model=model) or 0.0) * prediction_count logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Vertex model logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata @@ -387,30 +384,9 @@ class VertexPassthroughLoggingHandler: ) -> bool: return ( VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0 - and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) is not None + and get_vertex_ai_lyria_generation_cost(model=model) is not None ) - @staticmethod - def _get_audio_prediction_unit_cost(model: str) -> float | None: - runtime_unit_cost: Final = VertexPassthroughLoggingHandler._audio_prediction_unit_cost_from_model_info( - model_info=litellm.model_cost.get(f"vertex_ai/{model}") - ) - if runtime_unit_cost is not None: - return runtime_unit_cost - return get_vertex_ai_lyria_generation_cost(model=model) - - @staticmethod - def _audio_prediction_unit_cost_from_model_info(model_info: object) -> float | None: - if not isinstance(model_info, Mapping): - return None - output_cost_per_second: Final = model_info.get("output_cost_per_second") - audio_seconds_per_prediction: Final = model_info.get("audio_seconds_per_prediction") - if not isinstance(output_cost_per_second, (int, float)) or not isinstance( - audio_seconds_per_prediction, (int, float) - ): - return None - return float(output_cost_per_second * audio_seconds_per_prediction) - @staticmethod def _get_audio_prediction_count( json_response: dict, # mutable-ok: counter inspects the decoded provider response dictionary without mutation diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f11cbc92224..27132c90e05 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11176,14 +11176,9 @@ async def audio_speech( upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - hidden_audio_mime_type: Final = hidden_params.get("audio_mime_type") - media_type: Final = ( - hidden_audio_mime_type - if isinstance(hidden_audio_mime_type, str) - else resolve_speech_media_type( - upstream_content_type=upstream_content_type, - response_format=requested_format if isinstance(requested_format, str) else None, - ) + media_type: Final = resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=requested_format if isinstance(requested_format, str) else None, ) return StreamingResponse( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0b13191d977..32d88da0085 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -119,9 +119,6 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): return self._hidden_params["response_cost"] = response_cost - def set_audio_mime_type(self, audio_mime_type: str) -> None: - self._hidden_params["audio_mime_type"] = audio_mime_type - class NotGiven: """ diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6c645e70bee..e29944919ba 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -312,9 +312,6 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_video_per_second: float | None # only for vertex ai models output_cost_per_audio_per_second: float | None # only for vertex ai models output_cost_per_second: float | None # for OpenAI Speech models - audio_seconds_per_prediction: ReadOnly[float | None] - max_audio_length_hours: ReadOnly[float | None] - max_audio_per_prompt: ReadOnly[int | None] output_cost_per_second_1080p: ( float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) diff --git a/litellm/utils.py b/litellm/utils.py index d6a68b3ce6e..f6b40d33a79 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5880,9 +5880,6 @@ def _get_model_info_helper( "output_cost_per_token_above_512k_tokens", None ), output_cost_per_second=_model_info.get("output_cost_per_second", None), - audio_seconds_per_prediction=_model_info.get("audio_seconds_per_prediction", None), - max_audio_length_hours=_model_info.get("max_audio_length_hours", None), - max_audio_per_prompt=_model_info.get("max_audio_per_prompt", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9986d5056d1..0bc6c59a4b7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45512,12 +45512,9 @@ "supports_tool_choice": true }, "vertex_ai/lyria-002": { - "audio_seconds_per_prediction": 30, "litellm_provider": "vertex_ai", - "max_audio_length_hours": 0.009111111111111111, - "max_audio_per_prompt": 4, - "mode": "audio_speech", - "output_cost_per_second": 0.002, + "mode": "audio_speech", + "output_cost_per_image": 0.06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_audio_formats": [ "wav" diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 58ca91f3977..6faa957dc16 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -53,11 +53,6 @@ "type": "number", "minimum": 0 }, - "audio_seconds_per_prediction": { - "type": "number", - "minimum": 0, - "description": "Audio duration, in seconds, produced by one prediction." - }, "audio_transcription_config": { "type": "string" }, @@ -368,16 +363,6 @@ "type": "string", "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers." }, - "max_audio_length_hours": { - "type": "number", - "minimum": 0, - "description": "Maximum generated audio duration, expressed in hours." - }, - "max_audio_per_prompt": { - "type": "integer", - "minimum": 0, - "description": "Maximum number of audio outputs accepted or generated per prompt." - }, "max_input_tokens": { "type": "integer", "minimum": 0, diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index d1d751989ea..48376872a72 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1696,3 +1696,44 @@ def test_vertex_text_embedding_request_includes_labels_from_metadata(): }, ) assert req.get("labels") == {"project_id": "cost-center-1"} + + +@pytest.mark.parametrize( + ("model", "expected_api"), + [ + ("lyria-002", "lyria_predict"), + ("vertex_ai/lyria-002", "lyria_predict"), + ("lyria-3-clip-preview", "lyria_interactions"), + ("lyria-3-pro-preview", "lyria_interactions"), + ], +) +def test_get_vertex_ai_lyria_model_info_resolves_audio_api(model, expected_api): + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info + + model_info = get_vertex_ai_lyria_model_info(model=model) + + assert model_info is not None + assert model_info["vertex_ai_audio_api"] == expected_api + + +@pytest.mark.parametrize("model", ["en-US-Studio-O", "gemini-2.5-flash-preview-tts", "chirp-3-hd-charon"]) +def test_get_vertex_ai_lyria_model_info_is_none_for_non_lyria_speech_models(model): + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info + + assert get_vertex_ai_lyria_model_info(model=model) is None + + +def test_get_vertex_ai_lyria_model_info_falls_back_to_bundled_map(monkeypatch): + import litellm + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info + + stale_runtime_model_cost = { + key: value for key, value in litellm.model_cost.items() if not key.startswith("vertex_ai/lyria") + } + monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost) + + model_info = get_vertex_ai_lyria_model_info(model="lyria-3-pro-preview") + + assert model_info is not None + assert model_info["vertex_ai_audio_api"] == "lyria_interactions" + assert model_info["supported_audio_formats"] == ("mp3", "wav") diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 2cf6689261e..682dadfe854 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -18,8 +18,9 @@ def test_lyria_predict_response_preserves_audio_response_and_logs_cost( litellm.model_cost, "vertex_ai/lyria-002", { - "audio_seconds_per_prediction": 30, - "output_cost_per_second": 0.002, + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + "output_cost_per_image": 0.06, }, ) logging_obj = MagicMock() @@ -79,8 +80,9 @@ def test_audio_predict_response_uses_model_map_metadata( litellm.model_cost, "vertex_ai/music-audio-preview", { - "audio_seconds_per_prediction": 12, - "output_cost_per_second": 0.5, + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + "output_cost_per_image": 0.5, }, ) logging_obj = MagicMock() @@ -109,8 +111,8 @@ def test_audio_predict_response_uses_model_map_metadata( ) assert result["kwargs"]["model"] == "music-audio-preview" - assert result["kwargs"]["response_cost"] == pytest.approx(6.0) - assert logging_obj.model_call_details["response_cost"] == pytest.approx(6.0) + assert result["kwargs"]["response_cost"] == pytest.approx(0.5) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.5) def test_audio_predict_response_supports_bytes_base64_encoded( @@ -120,8 +122,9 @@ def test_audio_predict_response_supports_bytes_base64_encoded( litellm.model_cost, "vertex_ai/lyria-002", { - "audio_seconds_per_prediction": 30, - "output_cost_per_second": 0.002, + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + "output_cost_per_image": 0.06, }, ) logging_obj = MagicMock() @@ -146,27 +149,23 @@ def test_audio_predict_response_supports_bytes_base64_encoded( assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) -@pytest.mark.parametrize( - "missing_fields", - ( - None, - ("output_cost_per_second",), - ("audio_seconds_per_prediction",), - ("output_cost_per_second", "audio_seconds_per_prediction"), - ), -) +@pytest.mark.parametrize("runtime_entry_is_missing", (True, False)) def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_incomplete( monkeypatch: pytest.MonkeyPatch, - missing_fields: tuple[str, ...] | None, + runtime_entry_is_missing: bool, local_model_cost_map: None, ) -> None: - if missing_fields is None: + if runtime_entry_is_missing: monkeypatch.delitem(litellm.model_cost, "vertex_ai/lyria-002") else: monkeypatch.setitem( litellm.model_cost, "vertex_ai/lyria-002", - {key: value for key, value in litellm.model_cost["vertex_ai/lyria-002"].items() if key not in missing_fields}, + { + key: value + for key, value in litellm.model_cost["vertex_ai/lyria-002"].items() + if key != "output_cost_per_image" + }, ) logging_obj = MagicMock() logging_obj.model_call_details = {} @@ -193,7 +192,7 @@ def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_i request_body={"instances": [{"prompt": "ambient piano"}]}, ) - if missing_fields is None: + if runtime_entry_is_missing: assert "vertex_ai/lyria-002" not in litellm.model_cost assert result["kwargs"]["model"] == "lyria-002" assert result["kwargs"]["response_cost"] == pytest.approx(0.06) diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 4f4df69780a..b5eec42b569 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -169,23 +169,6 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): class TestVertexAILyriaTextToSpeechConfig: - def test_response_without_mime_type_uses_audio_container(self) -> None: - audio: Final = b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00" - raw_response: Final = httpx.Response( - 200, - json={"outputs": [{"type": "audio", "data": base64.b64encode(audio).decode()}]}, - ) - - response: Final = VertexAILyriaTextToSpeechConfig().transform_text_to_speech_response( - model="lyria-3-pro-preview", - raw_response=raw_response, - logging_obj=MagicMock(), - ) - - assert response.content == audio - assert response.response.headers["content-type"] == "audio/wav" - assert response._hidden_params["audio_mime_type"] == "audio/wav" - @pytest.mark.parametrize( "model", ["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"], @@ -385,11 +368,11 @@ class TestVertexAILyriaTextToSpeechConfig: { "predictions": [ { - "bytesBase64Encoded": "bHlyaWEtMi1hdWRpbw==", + "bytesBase64Encoded": "UklGRiQAAABXQVZFZm10IA==", } ] }, - b"lyria-2-audio", + b"RIFF$\x00\x00\x00WAVEfmt ", "audio/wav", ), ( @@ -427,6 +410,19 @@ class TestVertexAILyriaTextToSpeechConfig: b"lyria-3-audio", "audio/mpeg", ), + ( + "lyria-3-pro-preview", + { + "outputs": [ + { + "type": "audio", + "data": "UklGRiQAAABXQVZFZm10IA==", + } + ] + }, + b"RIFF$\x00\x00\x00WAVEfmt ", + "audio/wav", + ), ], ) def test_transform_response( @@ -446,7 +442,7 @@ class TestVertexAILyriaTextToSpeechConfig: ) assert response.content == expected_audio - assert response._hidden_params["audio_mime_type"] == expected_mime_type + assert response.response.headers["content-type"] == expected_mime_type @pytest.mark.parametrize( ("model", "response_format"), diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index aa630c4916b..3236253a10a 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -172,16 +172,15 @@ def test_vertex_lyria_speech_cost( monkeypatch.setitem( litellm.model_cost, model, - { - key: value - for key, value in model_info.items() - if key not in ("output_cost_per_image", "output_cost_per_second", "audio_seconds_per_prediction") - }, + {key: value for key, value in model_info.items() if key != "output_cost_per_image"}, ) elif runtime_state in ("custom_zero", "custom_price"): - cost_key: Final = "output_cost_per_image" if "output_cost_per_image" in model_info else "output_cost_per_second" multiplier: Final = 0 if runtime_state == "custom_zero" else 2 - monkeypatch.setitem(litellm.model_cost, model, {**model_info, cost_key: model_info[cost_key] * multiplier}) + monkeypatch.setitem( + litellm.model_cost, + model, + {**model_info, "output_cost_per_image": model_info["output_cost_per_image"] * multiplier}, + ) cost: Final = completion_cost( model=model, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index a179a82fcb2..f934b2aca1e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -944,14 +944,11 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "code_interpreter_cost_per_session": {"type": "number"}, "inference_geo": {"type": "string"}, "litellm_provider": {"type": "string"}, - "max_audio_length_hours": {"type": "number"}, - "max_audio_per_prompt": {"type": "number"}, "max_input_tokens": {"type": "number"}, "max_output_tokens": {"type": "number"}, "max_tokens": {"type": "number"}, "metadata": {"type": "object"}, "provider_specific_entry": {"type": "object"}, - "audio_seconds_per_prediction": {"type": "number"}, "mode": { "type": "string", "enum": [ @@ -2869,8 +2866,7 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert lyria_2["mode"] == "audio_speech" assert clip["mode"] == "audio_speech" assert pro["mode"] == "audio_speech" - assert lyria_2["audio_seconds_per_prediction"] == 30 - assert lyria_2["output_cost_per_second"] == 0.002 + assert lyria_2["output_cost_per_image"] == 0.06 assert lyria_2["supported_modalities"] == ["text"] assert lyria_2["supported_output_modalities"] == ["audio"] assert lyria_2["supports_audio_output"] is True From b21315fe754ddcb54b6e87759b621e5e17e515ef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:36:43 -0700 Subject: [PATCH 38/43] refactor(anthropic): type the responses refusal stream iterator state Types the cached sync upstream iterator instead of holding it as Any, so the Responses to Anthropic streaming wrapper carries no untyped state. --- .../responses_adapters/streaming_iterator.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 5bd662e0f94..bcae732ad42 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -4,7 +4,7 @@ import asyncio import json import traceback from collections import deque -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger @@ -55,7 +55,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_stop = False self._chunk_queue: deque[dict[str, object]] = deque() self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks - self._sync_responses_iterator: Any = None + self._sync_responses_iterator: Iterator[object] | None = None def _make_message_start(self) -> dict[str, object]: return { @@ -312,11 +312,9 @@ class AnthropicResponsesStreamWrapper: else: if self._sync_responses_iterator is None: self._sync_responses_iterator = iter(self.responses_stream) + sync_iterator: Final = self._sync_responses_iterator missing: Final = object() - while True: - event = await asyncio.to_thread(next, self._sync_responses_iterator, missing) - if event is missing: - break + while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing: self._process_event(event) if self._chunk_queue: return self._chunk_queue.popleft() From 02b44820c4dd5da47abf11e9f1b2dbe2fdce80f5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:59:57 -0700 Subject: [PATCH 39/43] test(vertex_ai): keep imagen predict passthrough off the Lyria audio path The new Lyria passthrough branch runs before the image-generation branch and keys on the same `predictions[0].bytesBase64Encoded` shape imagen returns, so only the cost-map lookup separates them. Cover an imagen predict response end to end so a future change that drops that lookup fails here instead of misbilling images as audio. --- ...test_vertex_passthrough_logging_handler.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 682dadfe854..98010021bca 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -9,6 +9,7 @@ import litellm from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) +from litellm.types.utils import PassthroughCallTypes def test_lyria_predict_response_preserves_audio_response_and_logs_cost( @@ -197,3 +198,33 @@ def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_i assert result["kwargs"]["model"] == "lyria-002" assert result["kwargs"]["response_cost"] == pytest.approx(0.06) assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) + + +def test_image_predict_response_is_not_billed_as_audio( + local_model_cost_map: None, +) -> None: + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={"predictions": [{"bytesBase64Encoded": "frame", "mimeType": "image/png"}]}, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route=( + "/v1/projects/test/locations/us-central1/publishers/google/models/imagen-4.0-generate-001:predict" + ), + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "a red cube"}]}, + ) + + assert isinstance(result["result"], litellm.ImageResponse) + assert logging_obj.call_type == PassthroughCallTypes.passthrough_image_generation.value + assert result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["vertex_ai/imagen-4.0-generate-001"]["output_cost_per_image"] + ) From 11e45ad95361eb37c00f9cce799bf544fd9990a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:00:01 -0700 Subject: [PATCH 40/43] fix(vertex_ai): mark the Lyria 3 catalog entries text-only `vertex_ai/lyria-3-clip-preview` and `vertex_ai/lyria-3-pro-preview` were registered with `supports_vision`, `supports_image_input`, and an `image` modality, which contradicts their `gemini/lyria-3-*` siblings and makes /model/info advertise image input on text-to-music models. --- .../model_prices_and_context_window_backup.json | 14 +++++--------- model_prices_and_context_window.json | 14 +++++--------- tests/test_litellm/test_utils.py | 11 +++++++---- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0bc6c59a4b7..5bdf6ed4fb8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45513,7 +45513,7 @@ }, "vertex_ai/lyria-002": { "litellm_provider": "vertex_ai", - "mode": "audio_speech", + "mode": "audio_speech", "output_cost_per_image": 0.06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_audio_formats": [ @@ -45549,8 +45549,7 @@ "/v1/audio/speech" ], "supported_modalities": [ - "text", - "image" + "text" ], "supported_output_modalities": [ "audio" @@ -45561,11 +45560,10 @@ "supports_audio_input": false, "supports_audio_output": true, "supports_function_calling": false, - "supports_image_input": true, "supports_prompt_caching": false, "supports_response_schema": false, "supports_system_messages": false, - "supports_vision": true, + "supports_vision": false, "supports_web_search": false, "vertex_ai_audio_api": "lyria_interactions" }, @@ -45588,8 +45586,7 @@ "/v1/audio/speech" ], "supported_modalities": [ - "text", - "image" + "text" ], "supported_output_modalities": [ "audio" @@ -45600,11 +45597,10 @@ "supports_audio_input": false, "supports_audio_output": true, "supports_function_calling": false, - "supports_image_input": true, "supports_prompt_caching": false, "supports_response_schema": false, "supports_system_messages": false, - "supports_vision": true, + "supports_vision": false, "supports_web_search": false, "vertex_ai_audio_api": "lyria_interactions" }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0bc6c59a4b7..5bdf6ed4fb8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45513,7 +45513,7 @@ }, "vertex_ai/lyria-002": { "litellm_provider": "vertex_ai", - "mode": "audio_speech", + "mode": "audio_speech", "output_cost_per_image": 0.06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_audio_formats": [ @@ -45549,8 +45549,7 @@ "/v1/audio/speech" ], "supported_modalities": [ - "text", - "image" + "text" ], "supported_output_modalities": [ "audio" @@ -45561,11 +45560,10 @@ "supports_audio_input": false, "supports_audio_output": true, "supports_function_calling": false, - "supports_image_input": true, "supports_prompt_caching": false, "supports_response_schema": false, "supports_system_messages": false, - "supports_vision": true, + "supports_vision": false, "supports_web_search": false, "vertex_ai_audio_api": "lyria_interactions" }, @@ -45588,8 +45586,7 @@ "/v1/audio/speech" ], "supported_modalities": [ - "text", - "image" + "text" ], "supported_output_modalities": [ "audio" @@ -45600,11 +45597,10 @@ "supports_audio_input": false, "supports_audio_output": true, "supports_function_calling": false, - "supports_image_input": true, "supports_prompt_caching": false, "supports_response_schema": false, "supports_system_messages": false, - "supports_vision": true, + "supports_vision": false, "supports_web_search": false, "vertex_ai_audio_api": "lyria_interactions" }, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index f934b2aca1e..d0916adce9d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2887,14 +2887,17 @@ def test_vertex_ai_lyria_models_in_cost_map(): "/v1beta/interactions", "/v1/audio/speech", ] - assert clip["supported_modalities"] == ["text", "image"] - assert pro["supported_modalities"] == ["text", "image"] + assert clip["supported_modalities"] == ["text"] + assert pro["supported_modalities"] == ["text"] + assert clip["supports_vision"] is False + assert pro["supports_vision"] is False + assert "supports_image_input" not in clip + assert "supports_image_input" not in pro assert clip["supported_regions"] == ["global"] assert pro["supported_regions"] == ["global"] assert clip["supports_audio_output"] is True assert pro["supports_audio_output"] is True - assert clip["supports_image_input"] is True - assert pro["supports_image_input"] is True + def test_model_info_for_fireworks_short_form_models(): """ From c09d34fc4b84a619f10eb7e64fe44fea198a8dc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:20:19 -0700 Subject: [PATCH 41/43] fix(anthropic): stream refusals parked in provider_specific_fields The first-delta guard read `delta.refusal` directly, while the translation three lines later goes through `openai_chat_refusal_text`, which also reads the `provider_specific_fields` LiteLLM parks unrecognized fields in. A provider that sends the refusal that way had its only refusal delta skipped as blank, so the client got `stop_reason: refusal` over an empty content array, which is the symptom this PR set out to fix --- .../adapters/streaming_iterator.py | 5 ++- .../test_streaming_iterator_first_delta.py | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index c6d744772a5..b2513a5c046 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -1064,6 +1064,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): @staticmethod def _is_blank_delta(chunk: "ModelResponseStream") -> bool: from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) choice: Final = chunk.choices[0] if choice.finish_reason is not None: @@ -1073,7 +1076,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return False if getattr(delta, "content", None): return False - if getattr(delta, "refusal", None): + if openai_chat_refusal_text(delta): return False if getattr(delta, "reasoning_content", None): return False diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 4359ace1870..4c5e2101fbf 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -145,6 +145,43 @@ async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async( assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." +def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted(): + """Providers that do not populate ``delta.refusal`` (Azure o-series among + them) hand LiteLLM the refusal as an unrecognized field, which lands in + ``provider_specific_fields``. That first delta still has to stream as text, + otherwise the client gets ``stop_reason: refusal`` over an empty content + array and shows the user nothing. + """ + chunks = [ + _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.asyncio +async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted_async(): + chunks = [ + _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): chunks = [ _make_chunk( From 05cba21763ee40477c2b4b13135f10763bb89957 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:40:20 -0700 Subject: [PATCH 42/43] fix(anthropic): split refusal off a combined finish_reason chunk A fake-streamed provider hands the adapter one chunk carrying both the delta payload and the finish_reason, which is exactly what the combined chunk splitter exists for, but its content check never listed the refusal. The translation short-circuits on finish_reason, so that refusal text was dropped and the client got `stop_reason: refusal` over an empty content array, the symptom this PR set out to fix. Both refusal accumulators also drop their `mutable-ok` lists for a plain string attribute --- .../adapters/streaming_iterator.py | 19 ++++++++++++------- .../responses_adapters/streaming_iterator.py | 6 +++--- .../test_streaming_iterator_first_delta.py | 8 +++++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index b2513a5c046..9158ff4569f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -101,6 +101,10 @@ class _CombinedChunkSplitter: @staticmethod def _is_combined(chunk: "ModelResponseStream") -> bool: """True if ``chunk`` carries response content AND a finish_reason.""" + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + choices: Final = _optional_attr_sequence(chunk, "choices") if not choices: return False @@ -115,6 +119,7 @@ class _CombinedChunkSplitter: or _optional_attr(delta, "tool_calls") or _optional_attr(delta, "reasoning_content") or _optional_attr(delta, "thinking_blocks") + or openai_chat_refusal_text(delta) ) _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = ( @@ -306,7 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage - self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks + self._refusal_text: str = "" self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -1001,7 +1006,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): self, processed_chunk: ContentBlockDelta | MessageBlockDelta, ) -> ContentBlockDelta | MessageBlockDelta: - if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: + if processed_chunk.get("type") != "message_delta" or not self._refusal_text: return processed_chunk delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": @@ -1017,7 +1022,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": refusal_stop_details("".join(self._refusal_text_parts)), + "stop_details": refusal_stop_details(self._refusal_text), }, }, ) @@ -1107,13 +1112,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): from .transformation import LiteLLMAnthropicMessagesAdapter - refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) - if refusal_text is not None: - self._refusal_text_parts.append(refusal_text) - if chunk.choices[0].finish_reason is not None: return False + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) + if refusal_text is not None: + self._refusal_text = self._refusal_text + refusal_text + ( block_type, content_block_start, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index bcae732ad42..2e0a6a9df8f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -54,7 +54,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque[dict[str, object]] = deque() - self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks + self._refusal_text: str = "" self._sync_responses_iterator: Iterator[object] | None = None def _make_message_start(self) -> dict[str, object]: @@ -142,7 +142,7 @@ class AnthropicResponsesStreamWrapper: delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") if not isinstance(delta, str) or not delta: return - self._refusal_text_parts.append(delta) + self._refusal_text = self._refusal_text + delta item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: @@ -241,7 +241,7 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () - refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None) + refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None) status: Final = getattr(response_obj, "status", None) if response_obj is not None else None has_tool_call: Final = any( getattr(item, "type", None) == "function_call" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 4c5e2101fbf..fdd08eaa182 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -183,6 +183,10 @@ async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emit def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): + """Fake-streamed responses arrive as one chunk carrying both the delta and the + finish_reason. The refusal has to be split off and streamed as text, or the + client gets ``stop_reason: refusal`` over an empty content array. + """ chunks = [ _make_chunk( Delta(content=None, refusal="I cannot fulfill this request."), @@ -193,6 +197,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): events = _drain_sync(wrapper) + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." @@ -202,7 +207,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async(): chunks = [ _make_chunk( - Delta(content=None, refusal="I cannot fulfill this request."), + Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."}), finish_reason="stop", ) ] @@ -210,6 +215,7 @@ async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_as events = await _drain_async(wrapper) + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." From b9f5cd60364aae12b654c3e01aa43173e4003e71 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sat, 5 Sep 2026 23:56:09 -0700 Subject: [PATCH 43/43] ci: run unit tests on Python 3.12 (#39989) --- .github/workflows/_test-unit-base.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 6f6822a975b..75b0f93fd77 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -55,17 +55,14 @@ on: permissions: contents: read +env: + UV_PYTHON: "3.12" + jobs: run: - name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }} + name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.job-timeout-minutes }} - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - env: - UV_PYTHON: ${{ matrix.python-version }} permissions: contents: read pull-requests: read @@ -88,7 +85,7 @@ jobs: timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.UV_PYTHON }} - name: Set up uv if: steps.changes.outputs.decision != 'skip' @@ -103,9 +100,9 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.UV_CACHE_DIR }} - key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} + key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}- + ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}- - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' @@ -139,7 +136,7 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} DIST: ${{ inputs.dist }} - COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }} + COVERAGE_CORE: sysmon run: | if [ "${WORKERS}" = "0" ]; then uv run --no-sync pytest ${TEST_PATH:?} \ @@ -166,7 +163,7 @@ jobs: fi - name: Save coverage report - if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip' + if: always() && steps.changes.outputs.decision != 'skip' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}