diff --git a/.circleci/config.yml b/.circleci/config.yml index bb4ad0f4019..e2102a9ae91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ commands: parameters: category: type: enum - enum: ["backend", "client"] + enum: ["backend", "client", "provider-harness"] default: "backend" steps: - run: @@ -2918,19 +2918,30 @@ jobs: provider_replay_harness: docker: - *python312_image + - image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f working_directory: ~/project resource_class: medium + environment: + E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0 + E2E_PROVIDER_CACHE: "0" + E2E_FIXTURE_MODE: live steps: + - checkout + - skip_if_unrelated_changes: + category: provider-harness - setup_litellm_test_deps + - wait_for_service: + url: tcp://localhost:6379 - run: - name: Test provider replay harness + name: Test provider capture and replay harness command: | mkdir -p test-results/provider-replay-harness uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \ --junitxml=test-results/provider-replay-harness/junit.xml \ tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \ tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \ - tests/code_coverage_tests/test_provider_replay_harness.py + tests/code_coverage_tests/test_provider_replay_harness.py \ + tests/code_coverage_tests/test_provider_cache.py - store_test_results: path: test-results/provider-replay-harness diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 7aa0c3544ee..9dc7b76b23f 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,13 +1,19 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false has_ci=false +has_provider_harness=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue + case "$file" in + tests/e2e/*/*.py) : ;; + tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) + has_provider_harness=true ;; + esac case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; @@ -17,6 +23,9 @@ while IFS= read -r file || [ -n "$file" ]; do done case "$category" in + provider-harness) + [ "$has_provider_harness" = true ] && echo run || echo skip + ;; backend) [ "$has_backend" = true ] && echo run || echo skip ;; diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index dcf64a24399..cdadde732bd 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: path_filter.sh }" +category="${1:?usage: path_filter.sh }" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { @@ -36,5 +36,5 @@ if [ "$decision" = run ]; then run_full "$category-relevant changes detected" fi -echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful" +echo "path-filter[$category]: only unrelated changes detected; halting job as successful" circleci-agent step halt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cfa0390e836..70a50d7f06e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,7 @@ /ui/nginx.conf /ui/litellm-dashboard/src/lib/http/schema.d.ts /ui/litellm-dashboard/tsconfig.tsbuildinfo -/model_prices_and_context_window.json @mateo-berri -/litellm/model_prices_and_context_window_backup.json @mateo-berri +/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri +/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri /.github/CODEOWNERS @yuneng-berri diff --git a/.github/workflows/ai-gateway-image.yml b/.github/workflows/ai-gateway-image.yml deleted file mode 100644 index 3f690f566b0..00000000000 --- a/.github/workflows/ai-gateway-image.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: ai-gateway image - -on: - push: - paths: - - "litellm-rust/**" - - "litellm/**" - - "enterprise/**" - - "litellm-proxy-extras/**" - - "pyproject.toml" - - "rust-toolchain.toml" - - ".github/workflows/ai-gateway-image.yml" - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - paths: - - "litellm-rust/**" - - "litellm/**" - - "enterprise/**" - - "litellm-proxy-extras/**" - - "pyproject.toml" - - "rust-toolchain.toml" - - ".github/workflows/ai-gateway-image.yml" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - ai-gateway-image: - name: ai-gateway release image - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - name: Build the release image - run: docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway:${{ github.sha }} . - - name: Start the gateway and wait for readiness - env: - IMAGE: litellm-ai-gateway:${{ github.sha }} - run: | - docker run -d --name ai-gateway -p 4001:4001 \ - -e LITELLM_MASTER_KEY=sk-ci-not-a-real-key \ - -e OPENAI_API_KEY=sk-ci-not-a-real-key \ - "$IMAGE" - for _ in $(seq 1 60); do - if curl -fsS http://127.0.0.1:4001/health/readiness; then - echo "gateway is serving readiness" - exit 0 - fi - sleep 2 - done - echo "gateway never became ready" >&2 - docker logs ai-gateway >&2 - exit 1 - - name: Assert the gateway loaded the baked config - run: | - docker logs ai-gateway 2>&1 | tee gateway.log - grep 'via python config reader' gateway.log - - name: Stop the gateway - if: always() - run: docker rm -f ai-gateway || true diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 17b6481a2bf..551f783d4f9 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -70,7 +70,7 @@ env: jobs: rust-lint: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 defaults: run: working-directory: litellm-rust @@ -81,28 +81,48 @@ jobs: - run: rustup toolchain install --no-self-update - - run: cargo fmt --check + - run: cargo fmt --all --check - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - litellm-rust/target - key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-${{ github.job }}- + workspaces: litellm-rust + cache-on-failure: true - run: cargo clippy --workspace --all-targets --locked -- -D warnings - - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings - - - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings - rust-test: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 + defaults: + run: + working-directory: litellm-rust + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: rustup toolchain install --no-self-update + + - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + with: + tool: cargo-nextest@0.9.143 + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: litellm-rust + cache-on-failure: true + + - run: cargo nextest run --workspace --locked + + - run: cargo test --workspace --doc --locked + + rust-wheel: + runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -118,24 +138,10 @@ jobs: - run: rustup toolchain install --no-self-update - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - litellm-rust/target - key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-${{ github.job }}- - - - run: cargo test --workspace --locked - working-directory: litellm-rust - - - run: cargo test -p litellm-core --features bedrock-auth --locked - working-directory: litellm-rust - - - run: cargo test -p litellm-ai-gateway --features server --locked - working-directory: litellm-rust + workspaces: litellm-rust + cache-on-failure: true - run: uv build --wheel --out-dir dist diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 7e3d25e9c5d..e2a3af77594 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -462,64 +462,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core", - "base64 0.22.1", - "bytes", - "futures-util", - "http 1.4.2", - "http-body 1.1.0", - "http-body-util", - "hyper 1.10.1", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "sync_wrapper", - "tokio", - "tokio-tungstenite", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.4.2", - "http-body 1.1.0", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "azure_core" version = "1.1.0" @@ -1582,7 +1524,6 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "httparse", - "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1890,12 +1831,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "libc" version = "0.2.186" @@ -1903,40 +1838,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "litellm-ai-gateway" +name = "litellm-auth" version = "0.1.0" dependencies = [ - "axum", - "base64 0.22.1", - "futures-channel", - "futures-util", - "litellm-config", - "litellm-core", - "reqwest 0.12.28", - "rustls 0.23.42", - "rustls-native-certs", "serde", - "serde_json", - "sha2 0.10.9", "subtle", - "tokio", - "tokio-tungstenite", - "tower", - "tracing", -] - -[[package]] -name = "litellm-config" -version = "0.1.0" -dependencies = [ - "litellm-core", - "pyo3", - "serde_json", "thiserror 2.0.19", + "tokio", + "veil", ] [[package]] -name = "litellm-core" +name = "litellm-auth-aws" version = "0.1.0" dependencies = [ "aws-config", @@ -1945,13 +1858,75 @@ dependencies = [ "aws-sigv4", "aws-smithy-runtime-api", "aws-types", + "litellm-auth", + "moka", + "reqwest 0.12.28", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "litellm-auth-azure" +version = "0.1.0" +dependencies = [ "azure_core", "azure_identity", + "litellm-auth", + "moka", + "serde_json", + "sha2 0.10.9", + "strum", + "tokio", + "url", +] + +[[package]] +name = "litellm-auth-gcp" +version = "0.1.0" +dependencies = [ + "gcp_auth", + "litellm-auth", + "moka", + "serde_json", + "sha2 0.10.9", + "tokio", +] + +[[package]] +name = "litellm-cache" +version = "0.1.0" +dependencies = [ + "rstest", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", +] + +[[package]] +name = "litellm-cache-memory" +version = "0.1.0" +dependencies = [ + "litellm-cache", + "rstest", + "serde_json", + "tokio", +] + +[[package]] +name = "litellm-core" +version = "0.1.0" +dependencies = [ "base64 0.22.1", "bytes", "data-url", "futures-util", - "gcp_auth", + "litellm-auth", + "litellm-auth-aws", + "litellm-auth-azure", + "litellm-auth-gcp", "mime_guess", "moka", "rand 0.8.7", @@ -1968,8 +1943,6 @@ dependencies = [ "thiserror 2.0.19", "tokio", "tokio-tungstenite", - "tracing", - "tracing-subscriber", "url", "veil", ] @@ -1980,6 +1953,7 @@ version = "0.1.0" dependencies = [ "criterion", "futures-util", + "litellm-auth", "litellm-core", "litellm-python-interop", "litellm-token-counter", @@ -1990,7 +1964,6 @@ dependencies = [ "serde_json", "tokio", "tokio-tungstenite", - "tracing", ] [[package]] @@ -2065,12 +2038,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "memchr" version = "2.8.3" @@ -3150,15 +3117,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shlex" version = "2.0.1" @@ -3380,15 +3338,6 @@ dependencies = [ "syn 3.0.0", ] -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" version = "0.3.53" @@ -3606,7 +3555,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -3650,7 +3598,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3686,17 +3633,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "sharded-slab", - "thread_local", - "tracing-core", -] - [[package]] name = "try-lock" version = "0.2.5" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 5c72c86d6ef..879090870d8 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -1,12 +1,5 @@ [workspace] -members = [ - "crates/core", - "crates/token-counter", - "crates/config", - "crates/ai-gateway", - "crates/python-interop", - "crates/python-bridge", -] +members = ["crates/*"] resolver = "2" [workspace.package] @@ -17,14 +10,15 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] bytes = "1" -tracing = "0.1" -tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } litellm-core = { path = "crates/core" } +litellm-auth = { path = "crates/auth" } +litellm-auth-aws = { path = "crates/auth-aws" } +litellm-auth-azure = { path = "crates/auth-azure" } +litellm-auth-gcp = { path = "crates/auth-gcp" } +litellm-cache = { path = "crates/cache" } +litellm-cache-memory = { path = "crates/cache-memory" } litellm-token-counter = { path = "crates/token-counter" } -litellm-config = { path = "crates/config" } -litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } litellm-python-interop = { path = "crates/python-interop" } -axum = "0.7" pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" @@ -42,9 +36,6 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" -gcp_auth = "0.12.7" -azure_core = "1.0.0" -azure_identity = { version = "1.0.0", features = ["tokio"] } moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" diff --git a/litellm-rust/crates/ai-gateway/Cargo.toml b/litellm-rust/crates/ai-gateway/Cargo.toml deleted file mode 100644 index dfa61226d4e..00000000000 --- a/litellm-rust/crates/ai-gateway/Cargo.toml +++ /dev/null @@ -1,56 +0,0 @@ -[package] -name = "litellm-ai-gateway" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[lib] -name = "litellm_ai_gateway" - -[[bin]] -name = "litellm-ai-gateway" -path = "src/main.rs" -required-features = ["server"] - -[[bin]] -name = "trace-parity-gateway" -path = "src/bin/trace_parity_gateway.rs" -required-features = ["trace-parity"] - -[dependencies] -tracing.workspace = true -litellm-core = { workspace = true, features = ["bedrock-auth"] } -litellm-config.workspace = true -# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the -# Python proxy callbacks API. -reqwest.workspace = true -# rustls and its root store are direct dependencies so `io::tls` can build the -# one TLS config the outbound dials use; see that module for why it has to. -rustls.workspace = true -rustls-native-certs.workspace = true -# `sync` powers the bounded mpsc channel the realtime logger drains. -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "time", "sync"] } -tokio-tungstenite.workspace = true -futures-util.workspace = true -serde_json.workspace = true -base64.workspace = true -axum = { workspace = true, features = ["ws"], optional = true } -serde.workspace = true -subtle = { workspace = true, optional = true } -# sha2 hashes the master key into user_api_key_hash (matches the proxy's -# SHA-256 hash_token) so the plaintext credential never enters a log payload. -sha2 = { workspace = true, optional = true } -tower = { version = "0.5.3", features = ["util"], optional = true } - -[features] -default = [] -server = ["dep:axum", "dep:subtle", "dep:sha2"] -# Build the gateway's config from the proxy YAML via an embedded Python -# interpreter (links libpython; requires `litellm` importable at runtime). -python-config = ["litellm-config/python"] -trace-parity = ["server", "dep:tower", "litellm-core/observability"] - -[dev-dependencies] -futures-channel = "0.3" -tower = { version = "0.5.3", features = ["util"] } diff --git a/litellm-rust/crates/ai-gateway/Dockerfile b/litellm-rust/crates/ai-gateway/Dockerfile deleted file mode 100644 index 72ac25ce1d6..00000000000 --- a/litellm-rust/crates/ai-gateway/Dockerfile +++ /dev/null @@ -1,109 +0,0 @@ -# Multi-stage build for the LiteLLM Rust AI Gateway (realtime WebSocket proxy). -# -# Build context is the **repo root** so we can install `litellm` from this repo's -# source (the gateway loads its model_list via litellm.proxy.read_model_list, -# which is not in any PyPI release yet) AND build the rust workspace under -# litellm-rust/. -# -# docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway . -# -# No secrets live in this file. Runtime config (LITELLM_MASTER_KEY, -# OPENAI_API_KEY referenced by config.yaml, etc.) is injected as environment -# variables at deploy time. - -# ---- Chef ------------------------------------------------------------------- -# cargo-chef caches the dependency build so only the gateway crate recompiles on -# a source-only change. python3-dev is present in every rust stage because the -# `python-config` feature links libpython via pyo3 (even in the cook step), and -# python3-pip builds the litellm wheel in the builder stage. -FROM rust:1.98-slim-bookworm AS chef -ENV PYO3_PYTHON=python3.11 -# rustup reads rust-toolchain.toml from any parent of the working directory, so -# copying it in is what keeps every cargo call below on the repo's pinned -# channel rather than on whatever the base image happens to ship. -COPY rust-toolchain.toml /build/rust-toolchain.toml -WORKDIR /build/litellm-rust -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 python3-dev python3-pip pkg-config libssl-dev clang \ - && rm -rf /var/lib/apt/lists/* \ - && cargo install cargo-chef --locked --version 0.1.77 - -# ---- Planner ---------------------------------------------------------------- -# Produce the dependency recipe from the rust workspace manifests + Cargo.lock. -FROM chef AS planner -COPY litellm-rust/ . -RUN cargo chef prepare --recipe-path recipe.json - -# ---- Builder ---------------------------------------------------------------- -FROM chef AS builder -# Cook (compile) just the dependencies first — this layer is cached and reused -# whenever only gateway source changes. -COPY --from=planner /build/litellm-rust/recipe.json recipe.json -RUN cargo chef cook --locked --release \ - -p litellm-ai-gateway --features server,python-config \ - --recipe-path recipe.json -# Now copy the real sources and build the gateway binary. Deps are already cooked -# above, so this step only recompiles the gateway crate. -COPY litellm-rust/ . -RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config - -# The root pyproject builds with maturin against litellm-rust/crates/python-bridge, -# so the wheel is built here, next to the crate sources and the cargo toolchain, -# and the runtime stage installs the artifact instead of compiling anything. -# litellm[proxy] pins litellm-enterprise and litellm-proxy-extras to the versions -# in this repo, and those hit PyPI hours after every version bump merges, so both -# wheels are built from the repo too instead of being resolved from PyPI. -COPY pyproject.toml README.md LICENSE /build/ -COPY litellm/ /build/litellm/ -COPY enterprise/ /build/enterprise/ -COPY litellm-proxy-extras/ /build/litellm-proxy-extras/ -RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /build/dist \ - /build /build/enterprise /build/litellm-proxy-extras - -# ---- Runtime ---------------------------------------------------------------- -# python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3 -# 3.11 ABI so the embedded interpreter links and imports cleanly. -FROM python:3.11-slim-bookworm AS runtime - -# CA certificates for outbound TLS to the OpenAI realtime endpoint. -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# Install litellm (with proxy extras) FROM THIS REPO'S SOURCE so -# `import litellm.proxy.read_model_list` works — it is not on PyPI yet. The two -# sibling wheels come from the builder as well, so the pins in litellm[proxy] -# resolve against them and never wait on a PyPI publish. -COPY --from=builder /build/dist/*.whl /tmp/wheels/ -RUN wheel="$(ls /tmp/wheels/litellm-*.whl)" \ - && pip install --no-cache-dir \ - /tmp/wheels/litellm_enterprise-*.whl \ - /tmp/wheels/litellm_proxy_extras-*.whl \ - "${wheel}[proxy]" \ - && rm -rf /tmp/wheels - -# The compiled gateway binary (pure-Rust realtime hot path; Python is load-time -# only). -COPY --from=builder /build/litellm-rust/target/release/litellm-ai-gateway /usr/local/bin/litellm-ai-gateway - -# Default config.yaml. A real deploy can override this (e.g. mount a Render -# secret file at the same path) — never bake secrets into the image. -COPY litellm-rust/crates/ai-gateway/config.yaml /app/config.yaml - -# Bind to all interfaces (Render routes to 0.0.0.0:$PORT) and load the model_list -# from config.yaml via the embedded python config reader. -ENV HOST=0.0.0.0 \ - LITELLM_CONFIG_PATH=/app/config.yaml - -# Drop to a non-root user. The realtime hot path needs no root privileges, so -# running unprivileged limits blast radius if the process is ever compromised. -# The binary in /usr/local/bin is world-executable (COPY default mode 755); we -# only need /app (and the config.yaml it reads) owned by the unprivileged user. -RUN useradd --system --no-create-home --uid 10001 appuser \ - && chown -R appuser:appuser /app -USER appuser - -ENTRYPOINT ["/usr/local/bin/litellm-ai-gateway"] diff --git a/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore b/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore deleted file mode 100644 index d1386ff684d..00000000000 --- a/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore +++ /dev/null @@ -1,54 +0,0 @@ -# Dockerfile-specific ignore-file for the Rust AI Gateway build. -# -# The build context is the repo root (so the image can pip install litellm from -# source AND build the rust workspace). BuildKit honors `.dockerignore` -# next to the Dockerfile and it takes precedence over the repo-root `.dockerignore`, -# so this file shrinks the (large) repo-root context for THIS build only without -# touching the root `.dockerignore` used by the main litellm images. -# -# Strategy: ignore everything, then re-include only what the build needs: -# - litellm/ (pip install . needs the full package + proxy reader) -# - litellm-rust/ (the rust workspace; Cargo.lock + crate sources) -# - enterprise/ (litellm/proxy/enterprise symlinks into it; maturin walks it) -# - litellm-proxy-extras/ (built into a wheel alongside enterprise/ for litellm[proxy]) -# - pyproject.toml / README.md / LICENSE (packaging metadata for the wheel build) -# - rust-toolchain.toml (the pinned channel every cargo call in the build uses) -* - -# --- re-include the build inputs --- -!litellm/ -!litellm-rust/ -!enterprise/ -!litellm-proxy-extras/ -!pyproject.toml -!rust-toolchain.toml -!README.md -!LICENSE - -# --- prune heavy / irrelevant subpaths back out of the re-included trees --- -# Rust build artifacts (huge; regenerated in the builder). -**/target/ -# Committed python distribution artifacts; the wheel build does not read them. -enterprise/dist/ -litellm-proxy-extras/dist/ -# Python caches and compiled bytecode. -**/__pycache__/ -**/*.pyc -**/*.pyo -**/.pytest_cache/ -**/.ruff_cache/ -**/.mypy_cache/ -# Node / UI build output bundled under the python package (not needed to import -# litellm.proxy.read_model_list). -**/node_modules/ -litellm/proxy/_experimental/out/ -# Tests, logs, and local scratch. -**/tests/ -**/test/ -*.log -log.txt -*.tgz -# VCS / editor / CI metadata that may live under re-included trees. -**/.git/ -.git/ -**/.DS_Store diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md deleted file mode 100644 index cbcd8119546..00000000000 --- a/litellm-rust/crates/ai-gateway/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# LiteLLM Rust AI Gateway - -A minimal Axum service that fronts OpenAI's realtime API. Clients open a -WebSocket to `GET /v1/realtime`; the gateway authenticates, selects a deployment, -dials OpenAI upstream, and splices the two sockets frame-by-frame. - -## Crates - -`litellm-rust` has six crates. A crate is a layer or shared foundation, not a route: - -| Crate | Role | -|-------|------| -| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | -| litellm-token-counter | Standalone input token counting shared by host integrations without pulling in the full SDK. | -| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. | -| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | -| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. | - -Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers, token counter, and Python interop. - -- **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) -- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) -- **Health:** `GET /health/readiness`, `GET /health/liveness` -- **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging)) - -> **Realtime serving is pure Rust.** Python is used at **load time only** — to -> read the config once at boot. The realtime hot path never touches Python. - -The former `/health/gil` route and its acquisition counter were removed. They -only observed the single startup config load and did not prove that every GIL -acquisition was instrumented - -## Configuration (config.yaml) - -The gateway loads its `model_list` from a **config.yaml**, the same as the -LiteLLM proxy. Point `LITELLM_CONFIG_PATH` at the file: - -```yaml -# config.yaml -model_list: - - model_name: gpt-realtime - litellm_params: - model: openai/gpt-realtime - api_key: os.environ/OPENAI_API_KEY -``` - -```bash -LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway -``` - -At boot `litellm-config` calls into `litellm.proxy.read_model_list` and returns -resolved deployments to the gateway, which constructs the router. The Python -backend still reuses the **real proxy config reader** (`ProxyConfig.get_config`), -so everything the proxy supports in config.yaml works here too: - -- `include:` to merge in other config files, -- `os.environ/VAR` secret references (resolved via the secret manager, never - inlined), -- DB-stored models (when a database is configured). - -Secrets stay out of the config — reference them with `os.environ/...` and set -the env var at deploy time. The shipped Docker image is built with the -`python-config` feature and **bundles litellm**, so config loading works out of -the box; the default baked config lives at `/app/config.yaml` and can be -overridden at deploy time (e.g. a Render secret file mounted at the same path). - -### Environment variables - -| Var | Required | Default | Purpose | -|---|---|---|---| -| `LITELLM_CONFIG_PATH` | yes (config mode) | — | Path to the config.yaml the gateway loads its `model_list` from. The Docker image defaults this to `/app/config.yaml`. | -| `LITELLM_MASTER_KEY` | yes | — | Bearer token clients must send. Unset ⇒ all `/v1/realtime` requests are rejected (fail closed). | -| `OPENAI_API_KEY` | yes | — | Upstream OpenAI key. Referenced by config.yaml as `os.environ/OPENAI_API_KEY` for the gateway→OpenAI dial. | -| `HOST` | no | `127.0.0.1` | **Set to `0.0.0.0` in any container/deploy** or external traffic is refused. | -| `PORT` | no | `4001` | Listen port. Render and most PaaS inject this automatically. | -| `LITELLM_PROXY_BASE_URL` | no | `http://localhost:4000` | LiteLLM proxy that request logs are POSTed to. See [Request logging](#request-logging). | - -> Secrets (`LITELLM_MASTER_KEY`, `OPENAI_API_KEY`) are never baked into the image -> or `render.yaml` — inject them at deploy time only. - -### Lean env stand-in (fallback) - -If the binary is built **without** `python-config` (default features), or -`LITELLM_CONFIG_PATH` is unset, the gateway falls back to a single-deployment -stand-in built from the environment: - -| Var | Default | Purpose | -|---|---|---| -| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). | - -The default workspace build links no libpython and needs no config file. This -fallback mode only supports one hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the -stand-in only for the leanest possible build. - -## Request logging - -The gateway runs no spend logic. When a session ends it builds one -`StandardLoggingPayload` and POSTs it to `{LITELLM_PROXY_BASE_URL}/v1/rust_control_plane/logs` -(admin-only, bearer = `LITELLM_MASTER_KEY`), and the proxy replays it through its -normal callbacks (spend logs, Langfuse, etc.). The POST is non-blocking: a bounded -channel drained by a background worker, dropping with a counter if the proxy is -down. It sends one payload per session. Both env vars are in the table above. - -Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096), -`LITELLM_LOG_BATCH_SIZE` (256), `LITELLM_LOG_FLUSH_INTERVAL_MS` (500). - -## Build & run with Docker - -The image is built `--features server,python-config` and installs litellm **from this -repo's source** (the config reader is newer than any PyPI release), so the build -**context is the repo root**: - -```bash -# from the repo root -docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway . - -docker run --rm -p 4001:4001 \ - -e HOST=0.0.0.0 -e PORT=4001 \ - -e LITELLM_MASTER_KEY=sk-local \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - litellm-ai-gateway # LITELLM_CONFIG_PATH defaults to /app/config.yaml - -# smoke test -curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/health/readiness # -> 200 -curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/v1/realtime # -> 401 (auth fails closed) -``` - -On boot you should see `loaded model_list from /app/config.yaml via python -config reader` — that confirms the config path (not the env stand-in fallback). -To use your own config, mount it over the default: - -```bash -docker run --rm -p 4001:4001 \ - -e HOST=0.0.0.0 -e LITELLM_MASTER_KEY=sk-local -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -v $(pwd)/my-config.yaml:/app/config.yaml:ro \ - litellm-ai-gateway -``` - -### Cargo-only (no Docker) - -```bash -# config.yaml mode — needs litellm importable in the active python env -LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \ - cargo run --release -p litellm-ai-gateway --features server,python-config - -# env stand-in mode — no python, no config -cargo run --release -p litellm-ai-gateway --features server -``` - -## Deploy on Render - -The service is a Docker **web service**; Render terminates TLS and supports -WebSockets, so the public endpoint is `wss://.onrender.com/v1/realtime`. - -### Option A — Blueprint (`render.yaml`) - -`crates/ai-gateway/render.yaml` describes the service (Docker runtime, -`healthCheckPath: /health/readiness`, repo-root `dockerContext: .`, -`dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile`, -`LITELLM_CONFIG_PATH: /app/config.yaml`). `LITELLM_MASTER_KEY` and -`OPENAI_API_KEY` are `sync: false` — set them in the dashboard after the first -deploy. To use a non-default model_list, mount a **Render Secret File** at -`/app/config.yaml`. Point a Render Blueprint at this repo/branch and apply. - -### Option B — Render API - -```bash -# create a Docker web service from this repo+branch, then set env vars: -curl -X POST https://api.render.com/v1/services \ - -H "Authorization: Bearer $RENDER_API_KEY" -H "Content-Type: application/json" \ - -d '{ - "type": "web_service", "name": "litellm-rust-ai-gateway", - "ownerId": "", "repo": "https://github.com/BerriAI/litellm", - "branch": "", - "serviceDetails": { - "env": "docker", - "envSpecificDetails": { - "dockerfilePath": "./litellm-rust/crates/ai-gateway/Dockerfile", - "dockerContext": "." - }, - "healthCheckPath": "/health/readiness" - } - }' -# then set env vars LITELLM_MASTER_KEY, OPENAI_API_KEY, HOST=0.0.0.0, -# LITELLM_CONFIG_PATH=/app/config.yaml -``` - -Health check path **must** be `/health/readiness`. `autoDeploy` is off by default -in the blueprint — trigger deploys manually (or flip it on) to pick up new commits. - -## Scaling - -Concurrency is what matters, not total connections: each in-flight session holds -one client socket + one upstream socket. To scale, raise the instance count / -enable autoscaling on the Render service (e.g. baseline 10, max 100). Each -instance needs file descriptors for `2 × peak_concurrent_sessions` — raise -`ulimit -n` if you push very high concurrency. - -## Latency note - -The gateway adds the cost of one extra hop: client→gateway, then a fresh -gateway→OpenAI realtime handshake (TLS + WS upgrade + `session.created`). In -benchmarks this is ~100–150 ms of added session-establishment time; first-audio -and steady-state streaming add no measurable overhead. To minimize it, deploy the -gateway in the Render region with the lowest RTT to OpenAI's realtime endpoint. diff --git a/litellm-rust/crates/ai-gateway/config.yaml b/litellm-rust/crates/ai-gateway/config.yaml deleted file mode 100644 index 321801f6862..00000000000 --- a/litellm-rust/crates/ai-gateway/config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Sample realtime config for the LiteLLM Rust AI Gateway. -# -# litellm-config resolves this model_list at boot through the Python config -# reader (litellm.proxy.read_model_list), then the gateway builds its router. -# Includes, environment secrets, and database-stored models still work. -# -# Secrets are referenced (never inlined) via os.environ/. A real deploy can -# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH). -model_list: - - model_name: gpt-realtime - litellm_params: - model: openai/gpt-realtime - api_key: os.environ/OPENAI_API_KEY diff --git a/litellm-rust/crates/ai-gateway/render.yaml b/litellm-rust/crates/ai-gateway/render.yaml deleted file mode 100644 index 4170849f65d..00000000000 --- a/litellm-rust/crates/ai-gateway/render.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Render blueprint for the LiteLLM Rust AI Gateway (realtime WebSocket proxy). -# -# Single instance for now (no autoscaling). The public endpoint is a -# WebSocket served over TLS: wss://.onrender.com/v1/realtime -# -# Paths are relative to the **repo root** (Render's convention). The build -# context is the repo root so the image can install litellm from source — the -# gateway loads its model_list via litellm.proxy.read_model_list at boot. -# -# Secrets (LITELLM_MASTER_KEY, OPENAI_API_KEY) are marked sync: false — set -# them in the Render dashboard or via the API, never inline here. -services: - - type: web - name: litellm-rust-ai-gateway - runtime: docker - plan: standard - dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile - dockerContext: . - healthCheckPath: /health/readiness - numInstances: 1 - envVars: - # The gateway loads its model_list from this config.yaml via the embedded - # python config reader. The image bakes a default config at /app/config.yaml; - # a real deploy can override it by mounting a Render secret file at this - # same path (Dashboard → Environment → Secret Files) — never inline secrets. - - key: LITELLM_CONFIG_PATH - value: /app/config.yaml - - key: HOST - value: 0.0.0.0 - # Bearer token clients must send on /v1/realtime (fail closed if unset). - - key: LITELLM_MASTER_KEY - sync: false - # Referenced by config.yaml as os.environ/OPENAI_API_KEY for the upstream dial. - - key: OPENAI_API_KEY - sync: false diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs deleted file mode 100644 index b17f17de11f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ /dev/null @@ -1,288 +0,0 @@ -use litellm_core::audio_transcription::{ - AudioTranscriptionRequest as CoreAudioTranscriptionRequest, ProviderAudioTranscriptionRequest, - prepare_audio_transcription_provider_call, -}; -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::Error; -use serde_json::{Map, Value, json}; -use std::future::Future; -use std::pin::Pin; - -use super::types::PreparedAudioTranscriptionRequest; -use crate::integrations::custom_guardrail::{ - CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, -}; -use crate::integrations::custom_logger::{ - CallType, CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::{ - RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, -}; - -pub(crate) struct AudioTranscriptionLifecycleHooks { - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, - request_metadata: RequestMetadata, -} - -type AudioFuture<'a, T> = Pin> + Send + 'a>>; -type AudioLogFuture<'a> = Pin + Send + 'a>>; - -impl AudioTranscriptionLifecycleHooks { - pub(crate) fn new( - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, - request_metadata: RequestMetadata, - ) -> Self { - Self { - logger_runner, - guardrail_runner, - request_metadata, - } - } - - async fn run_pre_call_guardrails( - &self, - request: PreparedAudioTranscriptionRequest, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(request); - } - let (guardrail_request, _) = self - .guardrail_runner - .run_pre_call( - &guardrail_context(&self.request_metadata), - GuardrailRequest::new(json!({ - "model": request.model, - "custom_llm_provider": request.custom_llm_provider, - "audio": request.audio, - "optional_params": request.optional_params, - })), - ) - .await - .map_err(guardrail_error_to_core_error)?; - let Value::Object(mut data) = guardrail_request.data else { - return Err(Error::InvalidRequest( - "audio transcription pre_call guardrail must return an object".to_string(), - )); - }; - let audio = data.remove("audio").ok_or_else(|| { - Error::InvalidRequest("audio transcription guardrail removed audio".to_string()) - })?; - let optional_params = match data.remove("optional_params") { - Some(Value::Object(value)) => value, - Some(_) => { - return Err(Error::InvalidRequest( - "audio transcription optional_params must be an object".to_string(), - )); - } - None => Map::new(), - }; - Ok(PreparedAudioTranscriptionRequest { - audio, - optional_params, - ..request - }) - } - - async fn prepare_provider_request( - &self, - request: PreparedAudioTranscriptionRequest, - ) -> Result { - let PreparedAudioTranscriptionRequest { - model, - custom_llm_provider, - audio, - api_key, - api_base, - extra_headers, - optional_params, - timeout, - .. - } = request; - let provider_request = - prepare_audio_transcription_provider_call(CoreAudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: Some(&custom_llm_provider), - extra_headers, - optional_params, - timeout, - })?; - self.run_during_call_guardrails(provider_request).await - } - - async fn run_during_call_guardrails( - &self, - request: ProviderAudioTranscriptionRequest, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(request); - } - let (guardrail_request, _) = self - .guardrail_runner - .run_during_call( - &guardrail_context(&self.request_metadata), - GuardrailRequest::new(json!({ - "model": request.model(), - "custom_llm_provider": request.custom_llm_provider(), - "url": request.url(), - "body": request.body(), - })), - ) - .await - .map_err(guardrail_error_to_core_error)?; - let Value::Object(mut data) = guardrail_request.data else { - return Err(Error::InvalidRequest( - "audio transcription during_call guardrail must return an object".to_string(), - )); - }; - let body = data.remove("body").ok_or_else(|| { - Error::InvalidRequest("audio transcription guardrail removed body".to_string()) - })?; - Ok(request.with_body(body)) - } - - fn logging_payload( - &self, - context: &CallLifecycleContext, - timing: &CallLifecycleTiming, - ) -> StandardLoggingPayload { - StandardLoggingPayload { - id: context.litellm_call_id.clone(), - litellm_call_id: context.litellm_call_id.clone(), - call_type: context.call_type.clone(), - model: context.model.clone(), - custom_llm_provider: context.custom_llm_provider.clone(), - response_cost: 0.0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - start_time: timing.start_time, - end_time: timing.end_time, - stream: false, - metadata: StandardLoggingMetadata { - user_api_key_hash: self.request_metadata.user_api_key_hash.clone(), - user_api_key_user_id: self.request_metadata.user_api_key_user_id.clone(), - user_api_key_team_id: self.request_metadata.user_api_key_team_id.clone(), - ..Default::default() - }, - messages: None, - } - } -} - -impl CallLifecycleHooks - for AudioTranscriptionLifecycleHooks -{ - type PreCallFuture<'a> = AudioFuture<'a, PreparedAudioTranscriptionRequest>; - type DuringCallFuture<'a> = AudioFuture<'a, ProviderAudioTranscriptionRequest>; - type SuccessFuture<'a> = AudioLogFuture<'a>; - type FailureFuture<'a> = AudioLogFuture<'a>; - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedAudioTranscriptionRequest, - ) -> Self::PreCallFuture<'a> { - Box::pin(async move { self.run_pre_call_guardrails(request).await }) - } - - fn async_during_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedAudioTranscriptionRequest, - ) -> Self::DuringCallFuture<'a> { - Box::pin(async move { self.prepare_provider_request(request).await }) - } - - fn async_log_success_event<'a>( - &'a self, - context: &'a CallLifecycleContext, - response: &'a Value, - timing: &'a CallLifecycleTiming, - ) -> Self::SuccessFuture<'a> { - Box::pin(async move { - if self.logger_runner.is_empty() { - return; - } - self.logger_runner - .async_log_success_event( - &ModelCallDetails::from_standard_logging_payload( - self.logging_payload(context, timing), - ), - &CallbackValue::new("audio_transcription", response.clone()), - CallbackTiming::new(timing.start_time, timing.end_time), - ) - .await; - }) - } - - fn async_log_failure_event<'a>( - &'a self, - context: &'a CallLifecycleContext, - error: &'a Error, - timing: &'a CallLifecycleTiming, - ) -> Self::FailureFuture<'a> { - Box::pin(async move { - if self.logger_runner.is_empty() { - return; - } - let logging_error = LoggingError { - message: error.to_string(), - kind: core_error_kind(error).to_string(), - }; - self.logger_runner - .async_log_failure_event( - &ModelCallDetails::from_standard_logging_payload( - self.logging_payload(context, timing), - ) - .with_failure_error(logging_error.clone()), - Some(&CallbackValue::new( - "error", - json!({"message": logging_error.message, "kind": logging_error.kind}), - )), - CallbackTiming::new(timing.start_time, timing.end_time), - ) - .await; - }) - } -} - -fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { - GuardrailContext { - call_type: CallType::Other("audio_transcription".to_string()), - selected_guardrails: Vec::new(), - metadata: std::collections::HashMap::new(), - user_api_key_hash: metadata.user_api_key_hash.clone(), - user_api_key_user_id: metadata.user_api_key_user_id.clone(), - user_api_key_team_id: metadata.user_api_key_team_id.clone(), - trace_parent: None, - } -} - -fn guardrail_error_to_core_error(error: GuardrailError) -> Error { - Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) -} - -fn core_error_kind(error: &Error) -> &'static str { - match error { - Error::Auth(_) - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureDocumentIntelligenceCredentials - | Error::MissingReductoApiKey => "AuthError", - Error::InvalidProvider(_) => "InvalidProvider", - Error::InvalidRequest(_) => "InvalidRequest", - Error::InvalidType { .. } => "InvalidType", - Error::MissingField(_) | Error::MissingDocumentUrl => "MissingField", - Error::Http { .. } => "HttpError", - Error::InvalidResponse(_) => "InvalidResponse", - Error::Network(_) => "NetworkError", - Error::Connect(_) => "ConnectError", - Error::Routing(_) => "RoutingError", - Error::Unsupported(_) => "UnsupportedRequest", - } -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs deleted file mode 100644 index 03d621b8414..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -use litellm_core::Error; -use litellm_core::audio_transcription::execute_audio_transcription_provider_call; -use litellm_core::call_lifecycle::CallLifecycle; -use serde_json::Value; - -mod hooks; -mod prepare; -mod types; - -pub use types::AudioTranscriptionRequest; - -use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; - -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { - let PreparedAudioTranscriptionCall { request, hooks } = - prepare_audio_transcription_call(request); - CallLifecycle::default() - .run_request(request, &hooks, execute_audio_transcription_provider_call) - .await -} - -#[cfg(test)] -mod tests; diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs deleted file mode 100644 index a475d58635f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; - -use super::hooks::AudioTranscriptionLifecycleHooks; -use super::types::{AudioTranscriptionRequest, PreparedAudioTranscriptionRequest}; -use crate::integrations::custom_guardrail::CustomGuardrailRunner; -use crate::integrations::custom_logger::CustomLoggerRunner; - -pub(crate) struct PreparedAudioTranscriptionCall { - pub(crate) request: PreparedAudioTranscriptionRequest, - pub(crate) hooks: AudioTranscriptionLifecycleHooks, -} - -pub(crate) fn prepare_audio_transcription_call( - request: AudioTranscriptionRequest<'_>, -) -> PreparedAudioTranscriptionCall { - let call_id = request - .litellm_call_id - .map(str::to_string) - .unwrap_or_else(new_audio_transcription_call_id); - let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) - .unwrap_or(CustomLlmProvider { - model: request.model, - custom_llm_provider: "bedrock", - }); - PreparedAudioTranscriptionCall { - request: PreparedAudioTranscriptionRequest { - model: provider_info.model.to_string(), - custom_llm_provider: provider_info.custom_llm_provider.to_string(), - litellm_call_id: call_id, - audio: request.audio, - api_key: request.api_key.map(str::to_string), - api_base: request.api_base.map(str::to_string), - extra_headers: request.extra_headers, - optional_params: request.optional_params, - timeout: request.timeout, - }, - hooks: AudioTranscriptionLifecycleHooks::new( - CustomLoggerRunner::new(request.callbacks), - CustomGuardrailRunner::new(request.guardrails), - request.request_metadata, - ), - } -} - -fn new_audio_transcription_call_id() -> String { - static COUNTER: AtomicU64 = AtomicU64::new(1); - let sequence = COUNTER.fetch_add(1, Ordering::Relaxed); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |duration| duration.as_nanos()); - format!("audio-transcription-{timestamp}-{sequence}") -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs deleted file mode 100644 index 5df04708b7d..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::io::{Read, Write}; -use std::net::TcpListener; -use std::thread; - -use serde_json::{Map, json}; - -use super::{AudioTranscriptionRequest, audio_transcription}; - -#[tokio::test] -async fn bedrock_request_is_signed_and_contains_audio() { - let listener = TcpListener::bind("127.0.0.1:0").expect("listener"); - let address = listener.local_addr().expect("address"); - let server = thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("connection"); - let mut request = Vec::new(); - let mut buffer = [0_u8; 16_384]; - let count = stream.read(&mut buffer).expect("request"); - request.extend_from_slice(&buffer[..count]); - let request = String::from_utf8_lossy(&request); - assert!(request.contains("POST /model/mistral.voxtral-mini-3b-2507/converse")); - assert!(request.contains("authorization: AWS4-HMAC-SHA256")); - assert!(request.contains("x-amz-date:")); - assert!(request.contains("\"bytes\":\"AQI=\"")); - assert!(request.contains("Transcribe the audio. Respond with only the transcript.")); - let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 53\r\nConnection: close\r\n\r\n{\"output\":{\"message\":{\"content\":[{\"text\":\"hello\"}]}}}"; - stream.write_all(response).expect("response"); - }); - - let optional_params = Map::from_iter([ - ("aws_access_key_id".to_string(), json!("access-key")), - ("aws_secret_access_key".to_string(), json!("secret-key")), - ("aws_region_name".to_string(), json!("us-east-1")), - ]); - let api_base = format!("http://{address}"); - let response = audio_transcription(AudioTranscriptionRequest { - model: "mistral.voxtral-mini-3b-2507", - audio: json!({"data": "AQI=", "format": "wav", "filename": "audio.wav"}), - api_key: None, - api_base: Some(&api_base), - custom_llm_provider: Some("bedrock"), - extra_headers: None, - optional_params, - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - .expect("transcription"); - assert_eq!(response, json!({"text": "hello"})); - server.join().expect("server"); -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs deleted file mode 100644 index b470638264e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; -use serde_json::{Map, Value}; - -use crate::integrations::custom_guardrail::CustomGuardrail; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; - -pub struct AudioTranscriptionRequest<'a> { - pub model: &'a str, - pub audio: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub optional_params: Map, - pub timeout: Option, - pub callbacks: Vec>, - pub guardrails: Vec>, - pub request_metadata: RequestMetadata, - pub litellm_call_id: Option<&'a str>, -} - -pub(crate) struct PreparedAudioTranscriptionRequest { - pub(crate) model: String, - pub(crate) custom_llm_provider: String, - pub(crate) litellm_call_id: String, - pub(crate) audio: Value, - pub(crate) api_key: Option, - pub(crate) api_base: Option, - pub(crate) extra_headers: Option>, - pub(crate) optional_params: Map, - pub(crate) timeout: Option, -} - -impl CallLifecycleRequest for PreparedAudioTranscriptionRequest { - fn lifecycle_context(&self) -> CallLifecycleContext { - CallLifecycleContext::new( - "audio_transcription", - self.model.clone(), - self.custom_llm_provider.clone(), - self.litellm_call_id.clone(), - ) - } -} diff --git a/litellm-rust/crates/ai-gateway/src/auth/mod.rs b/litellm-rust/crates/ai-gateway/src/auth/mod.rs deleted file mode 100644 index b09d8285c3a..00000000000 --- a/litellm-rust/crates/ai-gateway/src/auth/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Gateway authentication, as an axum **extractor** (the idiomatic pattern — -//! keeps handlers clean and auth testable). -//! -//! For now this is a single **master key**: any caller presenting it as -//! `Authorization: Bearer ` may invoke the gateway. Per-key auth, budgets, -//! and rate limits are delegated to the Python proxy in a later phase. -//! -//! A handler opts in by adding [`RequireMasterKey`] to its arguments; auth then -//! runs during extraction, before the handler body. Routes never re-implement it. - -use axum::extract::FromRequestParts; -use axum::http::StatusCode; -use axum::http::header::AUTHORIZATION; -use axum::http::request::Parts; -use sha2::{Digest, Sha256}; -use subtle::ConstantTimeEq; - -use crate::state::AppState; - -/// SHA-256 hex digest of a token — the exact transform the Python proxy applies -/// (`litellm.proxy.utils.hash_token`). -/// -/// STRICT REQUIREMENT: a raw key (`LITELLM_MASTER_KEY`, a virtual key, …) must -/// **never** leave this gateway in a log payload. Spend logs and every callback -/// integration receive `user_api_key_hash`, so that field must be this hash, not -/// the credential. Hashing here also means the value matches the key's hash in -/// `LiteLLM_SpendLogs.api_key`, so realtime spend joins with the rest of LiteLLM. -pub fn hash_token(token: &str) -> String { - let digest = Sha256::digest(token.as_bytes()); - let mut hex = String::with_capacity(digest.len() * 2); - for byte in digest { - use std::fmt::Write; - let _ = write!(hex, "{byte:02x}"); - } - hex -} - -/// Extractor that requires the configured master key as a bearer token. -/// -/// Rejections: `500` when no master key is configured (permanent -/// misconfiguration, not a transient outage); `401` on a missing/incorrect -/// token. The comparison is constant-time. -pub struct RequireMasterKey; - -#[axum::async_trait] -impl FromRequestParts for RequireMasterKey { - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &AppState, - ) -> Result { - let Some(expected) = state.master_key.as_deref() else { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - "gateway auth not configured (set LITELLM_MASTER_KEY)".to_string(), - )); - }; - let provided = parts - .headers - .get(AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.strip_prefix("Bearer ")) - .map(str::trim); - match provided { - Some(token) if bool::from(token.as_bytes().ct_eq(expected.as_bytes())) => Ok(Self), - _ => Err(( - StatusCode::UNAUTHORIZED, - "missing or invalid bearer token".to_string(), - )), - } - } -} - -#[cfg(test)] -mod tests { - use super::hash_token; - - #[test] - fn hash_token_matches_python_sha256_hexdigest() { - // Must equal hashlib.sha256("sk-1234".encode()).hexdigest() — the value - // the proxy stores in LiteLLM_SpendLogs.api_key. - assert_eq!( - hash_token("sk-1234"), - "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b" - ); - // 64 lowercase hex chars, and never the raw input. - let h = hash_token("sk-secret"); - assert_eq!(h.len(), 64); - assert!(h.chars().all(|c| c.is_ascii_hexdigit())); - assert_ne!(h, "sk-secret"); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs b/litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs deleted file mode 100644 index e247c650fad..00000000000 --- a/litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::io::Read; - -use serde::Deserialize; -use serde_json::Value; - -#[derive(Deserialize)] -struct Input { - path: String, - model_alias: String, - provider_model: String, - api_base: String, - body: Value, -} - -#[tokio::main] -async fn main() { - let mut input = String::new(); - if let Err(error) = std::io::stdin().read_to_string(&mut input) { - fail(error); - } - let input: Input = match serde_json::from_str(&input) { - Ok(input) => input, - Err(error) => fail(error), - }; - let result = litellm_ai_gateway::trace_parity::traced_request( - input.path, - input.model_alias, - input.provider_model, - input.api_base, - input.body, - ) - .await; - match serde_json::to_string(&result) { - Ok(result) => println!("{result}"), - Err(error) => fail(error), - } -} - -fn fail(error: impl std::fmt::Display) -> ! { - eprintln!("{error}"); - std::process::exit(1) -} diff --git a/litellm-rust/crates/ai-gateway/src/client.rs b/litellm-rust/crates/ai-gateway/src/client.rs deleted file mode 100644 index ff2606f0229..00000000000 --- a/litellm-rust/crates/ai-gateway/src/client.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::sync::OnceLock; -use std::time::Duration; - -const HTTP_CLIENT_TIMEOUT_SECS: u64 = 600; - -pub(crate) fn http_client() -> &'static reqwest::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - reqwest::Client::builder() - .timeout(Duration::from_secs(HTTP_CLIENT_TIMEOUT_SECS)) - .build() - .expect("failed to build reqwest client") - }) -} diff --git a/litellm-rust/crates/ai-gateway/src/constants.rs b/litellm-rust/crates/ai-gateway/src/constants.rs deleted file mode 100644 index 78af374bf70..00000000000 --- a/litellm-rust/crates/ai-gateway/src/constants.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Crate-level constants for the ai-gateway. -//! -//! Per `litellm-rust/CLAUDE.md`, magic numbers and fixed strings live here -//! (the Rust mirror of Python's `litellm/constants.py`), not inline in feature -//! modules. Env-overridable tunables keep their `DEFAULT_*` value here; the env -//! read + fallback happens at the host/config layer. - -/// Default LiteLLM control-plane base URL for request-log egress when -/// `LITELLM_PROXY_BASE_URL` is unset. -pub(crate) const DEFAULT_PROXY_BASE_URL: &str = "http://localhost:4000"; - -/// The logs ingest path appended to the proxy base. Not a tunable; it is the -/// proxy's API contract (the rust-control-plane router on the Python proxy). -pub(crate) const RUST_CONTROL_PLANE_LOGS_PATH: &str = "/v1/rust_control_plane/logs"; - -/// Default bounded channel depth for the log-egress worker. -/// Override: `LITELLM_LOG_CHANNEL_CAPACITY`. -pub(crate) const DEFAULT_CHANNEL_CAPACITY: usize = 4096; - -/// Default max records POSTed per request to the control plane. -/// Override: `LITELLM_LOG_BATCH_SIZE`. -pub(crate) const DEFAULT_MAX_BATCH_SIZE: usize = 256; - -/// Default partial-batch flush cadence, in ms. -/// Override: `LITELLM_LOG_FLUSH_INTERVAL_MS`. -pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500; - -/// Provider attributed to realtime sessions in the logging payload. -#[cfg(feature = "server")] -pub(crate) const DEFAULT_PROVIDER: &str = "openai"; - -pub(crate) const DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS: u64 = 10; -pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300; - -/// HTTP path for the non-streaming Anthropic Messages route. -#[cfg(feature = "server")] -pub(crate) const MESSAGES_ROUTE_PATH: &str = "/v1/messages"; - -/// Request headers owned by the gateway and never forwarded upstream. -#[cfg(feature = "server")] -pub(crate) const MESSAGES_HEADERS_NOT_FORWARDED: &[&str] = - &["authorization", "connection", "content-length", "host"]; diff --git a/litellm-rust/crates/ai-gateway/src/integrations/README.md b/litellm-rust/crates/ai-gateway/src/integrations/README.md deleted file mode 100644 index 16a162dac57..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# LiteLLM Rust integrations - -This directory contains Rust-native equivalents of LiteLLM integration hooks. -The first supported surfaces are terminal custom loggers and pre/during-call -custom guardrails. - -## File layout - -Every integration is a folder: - -- `mod.rs` contains the implementation, trait, runner, or adapter -- `types.rs` contains the integration-local request, response, error, and future - types - -Do not add new flat integration files such as `custom_logger.rs`. Shared wire -contracts that are used by multiple integrations can stay in -`integrations/types.rs`. - -Call ordering and lifecycle timing live in `litellm-core/src/call_lifecycle`. -Call-type modules, such as OCR, adapt their request and response shapes into -that generic lifecycle runner. - -## CustomLogger - -Implement `CustomLogger` when Rust code needs to observe terminal success or -failure events. Method names intentionally match Python `CustomLogger` names. - -```rust -use litellm_ai_gateway::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, -}; - -struct RecordingLogger; - -impl CustomLogger for RecordingLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: &'a CallbackValue, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - let model = &model_call_details.model; - let provider = &model_call_details.custom_llm_provider; - let call_type = model_call_details.call_type.to_string(); - let request_id = model_call_details.request_id.as_deref(); - let response_object = &response_obj.object; - let duration = timing.end_time - timing.start_time; - let standard_payload = model_call_details.standard_logging_payload.as_ref(); - - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: Option<&'a CallbackValue>, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - let error = model_call_details.failure_error.as_ref(); - let response_object = response_obj.map(|value| value.object.as_str()); - let duration = timing.end_time - timing.start_time; - - Ok(()) - }) - } -} -``` - -Use `CustomLoggerRunner` to fan out terminal events to configured loggers. The -runner is a no-op when no loggers are configured, which is the expected fast -path for requests without callbacks. - -## CustomGuardrail - -Implement `CustomGuardrail` when Rust code needs to run pre-call or native -during-call checks. Method names intentionally match Python `CustomGuardrail` -entrypoints inherited from Python `CustomLogger`. - -```rust -use litellm_ai_gateway::integrations::custom_guardrail::{ - CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailEventHook, - GuardrailFuture, GuardrailRequest, -}; - -struct BlocklistedPromptGuardrail; - -impl CustomGuardrail for BlocklistedPromptGuardrail { - fn guardrail_name(&self) -> &str { - "blocklisted-prompt" - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &[GuardrailEventHook::PreCall] - } - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - if request.data.to_string().contains("blocked phrase") { - return Ok(GuardrailDecision::Block( - litellm_ai_gateway::integrations::custom_guardrail::GuardrailError::blocked( - "blocked phrase detected", - ), - )); - } - Ok(GuardrailDecision::Allow(request)) - }) - } -} -``` - -Use `CustomGuardrailRunner::run_pre_call` for `pre_call` guardrails and -`CustomGuardrailRunner::run_during_call` for `during_call` guardrails. A -`GuardrailDecision::Mask` continues with modified request data. -`GuardrailDecision::Block` short-circuits the provider call. - -## Current boundary - -These are Rust-only primitives. Python callback and guardrail adapters are a -separate layer that should implement these Rust traits instead of changing the -runner interfaces. diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs deleted file mode 100644 index e5d4ce3a708..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! Rust mirror of Python `CustomGuardrail` entrypoints used by the proxy. -//! -//! This module is intentionally Rust-only: Python/PyO3 adapters are a later -//! layer that should implement this trait rather than changing the runner. - -use std::future::Future; -use std::sync::Arc; - -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; - -pub mod types; - -pub use types::{ - GuardrailContext, GuardrailDecision, GuardrailDispatchReport, GuardrailError, - GuardrailEventHook, GuardrailFuture, GuardrailRequest, -}; - -pub trait CustomGuardrail: Send + Sync { - fn guardrail_name(&self) -> &str; - - fn supported_event_hooks(&self) -> &[GuardrailEventHook]; - - /// Python 1:1 name: `async_pre_call_hook(user_api_key_dict, cache, data, call_type)`. - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { Ok(GuardrailDecision::Allow(request)) }) - } - - /// Python 1:1 name: `async_moderation_hook(data, user_api_key_dict, call_type)`. - fn async_moderation_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { Ok(GuardrailDecision::Allow(request)) }) - } -} - -pub struct CustomGuardrailRunner { - guardrails: Vec>, -} - -impl CustomGuardrailRunner { - pub fn new(guardrails: Vec>) -> Self { - Self { guardrails } - } - - pub fn is_empty(&self) -> bool { - self.guardrails.is_empty() - } - - pub async fn run_pre_call( - &self, - context: &GuardrailContext, - request: GuardrailRequest, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - self.run_hook(GuardrailEventHook::PreCall, context, request) - .await - } - - pub async fn run_during_call( - &self, - context: &GuardrailContext, - request: GuardrailRequest, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - self.run_hook(GuardrailEventHook::DuringCall, context, request) - .await - } - - pub async fn run_before_provider( - &self, - event_hook: GuardrailEventHook, - context: &GuardrailContext, - request: GuardrailRequest, - provider: F, - ) -> Result - where - F: FnOnce(GuardrailRequest) -> Fut, - Fut: Future>, - { - let (request, _) = self.run_hook(event_hook, context, request).await?; - provider(request).await - } - - pub async fn run_pre_call_with_failure_logging( - &self, - context: &GuardrailContext, - request: GuardrailRequest, - logger_runner: &CustomLoggerRunner, - model_call_details: &ModelCallDetails, - timing: CallbackTiming, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - match self.run_pre_call(context, request).await { - Ok(result) => Ok(result), - Err(error) => { - let failure_details = model_call_details.clone().with_failure_error(LoggingError { - message: error.message.clone(), - kind: error.kind.clone(), - }); - let response_obj = CallbackValue::new( - "guardrail_error", - serde_json::json!({ - "message": error.message, - "kind": error.kind, - }), - ); - logger_runner - .async_log_failure_event(&failure_details, Some(&response_obj), timing) - .await; - Err(error) - } - } - } - - async fn run_hook( - &self, - event_hook: GuardrailEventHook, - context: &GuardrailContext, - mut request: GuardrailRequest, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - if self.guardrails.is_empty() { - return Ok((request, GuardrailDispatchReport::default())); - } - - let mut report = GuardrailDispatchReport::default(); - for guardrail in &self.guardrails { - if !self.should_run(guardrail.as_ref(), event_hook, context) { - continue; - } - - report.invoked += 1; - let decision = match event_hook { - GuardrailEventHook::PreCall => { - guardrail - .async_pre_call_hook(context, request.clone()) - .await? - } - GuardrailEventHook::DuringCall => { - guardrail - .async_moderation_hook(context, request.clone()) - .await? - } - }; - match decision.into_request() { - Ok(next_request) => request = next_request, - Err(error) => return Err(error), - } - } - - Ok((request, report)) - } - - fn should_run( - &self, - guardrail: &dyn CustomGuardrail, - event_hook: GuardrailEventHook, - context: &GuardrailContext, - ) -> bool { - let supports_hook = guardrail.supported_event_hooks().contains(&event_hook); - let selected = context.selected_guardrails.is_empty() - || context - .selected_guardrails - .iter() - .any(|name| name == guardrail.guardrail_name()); - supports_hook && selected - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::integrations::custom_logger::{CallType, CallbackValue, CustomLogger, LogFuture}; - use crate::integrations::types::{StandardLoggingMetadata, StandardLoggingPayload}; - use serde_json::json; - use std::sync::Mutex; - - #[derive(Clone)] - enum TestDecision { - Allow, - Mask, - Block, - } - - struct RecordingCustomGuardrail { - name: String, - hooks: Vec, - decision: TestDecision, - calls: Mutex>, - } - - impl RecordingCustomGuardrail { - fn new(name: &str, hooks: Vec, decision: TestDecision) -> Self { - Self { - name: name.to_string(), - hooks, - decision, - calls: Mutex::new(Vec::new()), - } - } - - fn calls(&self) -> Vec<&'static str> { - self.calls.lock().unwrap().clone() - } - - fn decision(&self, mut request: GuardrailRequest) -> GuardrailDecision { - match self.decision { - TestDecision::Allow => GuardrailDecision::Allow(request), - TestDecision::Mask => { - request.data["masked"] = json!(true); - GuardrailDecision::Mask(request) - } - TestDecision::Block => { - GuardrailDecision::Block(GuardrailError::blocked("blocked by guardrail")) - } - } - } - } - - impl CustomGuardrail for RecordingCustomGuardrail { - fn guardrail_name(&self) -> &str { - &self.name - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &self.hooks - } - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - self.calls.lock().unwrap().push("async_pre_call_hook"); - Ok(self.decision(request)) - }) - } - - fn async_moderation_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - self.calls.lock().unwrap().push("async_moderation_hook"); - Ok(self.decision(request)) - }) - } - } - - #[tokio::test] - async fn pre_call_dispatches_to_async_pre_call_hook() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "pre", - vec![GuardrailEventHook::PreCall], - TestDecision::Allow, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail.clone()]); - let context = - GuardrailContext::new(CallType::Ocr).with_selected_guardrails(vec!["pre".to_string()]); - let request = GuardrailRequest::new(json!({"messages": ["hello"]})); - - let (result, report) = runner - .run_pre_call(&context, request) - .await - .expect("guardrail allows request"); - - assert_eq!(report.invoked, 1); - assert_eq!(result.data["messages"], json!(["hello"])); - assert_eq!(guardrail.calls(), vec!["async_pre_call_hook"]); - } - - #[tokio::test] - async fn during_call_dispatches_to_async_moderation_hook() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "during", - vec![GuardrailEventHook::DuringCall], - TestDecision::Allow, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail.clone()]); - let context = GuardrailContext::new(CallType::Completion) - .with_selected_guardrails(vec!["during".to_string()]); - let request = GuardrailRequest::new(json!({"prompt": "hello"})); - - let (_result, report) = runner - .run_during_call(&context, request) - .await - .expect("guardrail allows request"); - - assert_eq!(report.invoked, 1); - assert_eq!(guardrail.calls(), vec!["async_moderation_hook"]); - } - - #[tokio::test] - async fn mask_decision_continues_with_updated_request() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "masker", - vec![GuardrailEventHook::PreCall], - TestDecision::Mask, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail]); - let context = GuardrailContext::new(CallType::Ocr); - let request = GuardrailRequest::new(json!({"document": "secret"})); - - let (result, report) = runner - .run_pre_call(&context, request) - .await - .expect("mask continues"); - - assert_eq!(report.invoked, 1); - assert_eq!(result.data["masked"], json!(true)); - } - - #[tokio::test] - async fn block_decision_short_circuits_and_logs_failure() { - struct RecordingFailureLogger { - errors: Mutex>, - } - - impl CustomLogger for RecordingFailureLogger { - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.errors.lock().unwrap().push( - model_call_details - .failure_error - .as_ref() - .map(|error| error.kind.clone()) - .unwrap_or_default(), - ); - Ok(()) - }) - } - } - - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "blocker", - vec![GuardrailEventHook::PreCall], - TestDecision::Block, - )); - let guardrail_runner = CustomGuardrailRunner::new(vec![guardrail]); - let logger = Arc::new(RecordingFailureLogger { - errors: Mutex::new(Vec::new()), - }); - let logger_runner = CustomLoggerRunner::new(vec![logger.clone()]); - let context = GuardrailContext::new(CallType::Ocr); - let details = ModelCallDetails::from_standard_logging_payload(StandardLoggingPayload { - id: "req_ocr".to_string(), - litellm_call_id: "req_ocr".to_string(), - call_type: "ocr".to_string(), - model: "mistral-ocr-latest".to_string(), - custom_llm_provider: "mistral".to_string(), - response_cost: 0.0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - start_time: 1.0, - end_time: 1.0, - stream: false, - metadata: StandardLoggingMetadata::default(), - messages: None, - }); - - let err = guardrail_runner - .run_pre_call_with_failure_logging( - &context, - GuardrailRequest::new(json!({"document": "bad"})), - &logger_runner, - &details, - CallbackTiming::new(1.0, 2.0), - ) - .await - .expect_err("guardrail blocks request"); - - assert_eq!(err.kind, "GuardrailBlocked"); - assert_eq!( - logger.errors.lock().unwrap().as_slice(), - ["GuardrailBlocked"] - ); - } - - #[tokio::test] - async fn block_decision_short_circuits_later_guardrails_and_provider_work() { - let blocking_guardrail = Arc::new(RecordingCustomGuardrail::new( - "blocker", - vec![GuardrailEventHook::PreCall], - TestDecision::Block, - )); - let later_guardrail = Arc::new(RecordingCustomGuardrail::new( - "later", - vec![GuardrailEventHook::PreCall], - TestDecision::Allow, - )); - let runner = - CustomGuardrailRunner::new(vec![blocking_guardrail.clone(), later_guardrail.clone()]); - let provider_called = Arc::new(Mutex::new(false)); - let provider_called_for_closure = provider_called.clone(); - - let result = runner - .run_before_provider( - GuardrailEventHook::PreCall, - &GuardrailContext::new(CallType::Completion), - GuardrailRequest::new(json!({"prompt": "blocked"})), - move |_request| async move { - *provider_called_for_closure.lock().unwrap() = true; - Ok("provider response") - }, - ) - .await; - - assert!(result.is_err()); - assert_eq!(blocking_guardrail.calls(), vec!["async_pre_call_hook"]); - assert_eq!(later_guardrail.calls(), Vec::<&'static str>::new()); - assert!(!*provider_called.lock().unwrap()); - } - - #[tokio::test] - async fn run_before_provider_returns_provider_guardrail_error_directly() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "allow", - vec![GuardrailEventHook::PreCall], - TestDecision::Allow, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail]); - - let result = runner - .run_before_provider( - GuardrailEventHook::PreCall, - &GuardrailContext::new(CallType::Completion), - GuardrailRequest::new(json!({"prompt": "allowed"})), - |_request| async move { - Err::<&'static str, GuardrailError>(GuardrailError::blocked( - "provider-side guardrail error", - )) - }, - ) - .await; - - let err = result.expect_err("provider error is returned directly"); - assert_eq!(err.kind, "GuardrailBlocked"); - assert_eq!(err.message, "provider-side guardrail error"); - } - - #[tokio::test] - async fn no_guardrails_fast_path_dispatches_nothing() { - let runner = CustomGuardrailRunner::new(Vec::new()); - let context = GuardrailContext::new(CallType::Ocr); - let request = GuardrailRequest::new(json!({"document": "ok"})); - - let (result, report) = runner - .run_pre_call(&context, request) - .await - .expect("no guardrails allow request"); - - assert!(runner.is_empty()); - assert_eq!(report, GuardrailDispatchReport::default()); - assert_eq!(result.data["document"], json!("ok")); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs deleted file mode 100644 index 825e56cc0d7..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; - -use serde_json::Value; - -use crate::integrations::custom_logger::CallType; - -pub type GuardrailFuture<'a> = - Pin> + Send + 'a>>; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GuardrailEventHook { - PreCall, - DuringCall, -} - -impl GuardrailEventHook { - pub fn as_str(&self) -> &'static str { - match self { - Self::PreCall => "pre_call", - Self::DuringCall => "during_call", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GuardrailError { - pub message: String, - pub kind: String, -} - -impl GuardrailError { - pub fn blocked(message: impl Into) -> Self { - Self { - message: message.into(), - kind: "GuardrailBlocked".to_string(), - } - } -} - -impl std::fmt::Display for GuardrailError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.kind, self.message) - } -} - -impl std::error::Error for GuardrailError {} - -#[derive(Clone, Debug)] -pub struct GuardrailContext { - pub call_type: CallType, - pub selected_guardrails: Vec, - pub metadata: HashMap, - pub user_api_key_hash: Option, - pub user_api_key_user_id: Option, - pub user_api_key_team_id: Option, - pub trace_parent: Option, -} - -impl GuardrailContext { - pub fn new(call_type: CallType) -> Self { - Self { - call_type, - selected_guardrails: Vec::new(), - metadata: HashMap::new(), - user_api_key_hash: None, - user_api_key_user_id: None, - user_api_key_team_id: None, - trace_parent: None, - } - } - - pub fn with_selected_guardrails(mut self, selected_guardrails: Vec) -> Self { - self.selected_guardrails = selected_guardrails; - self - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct GuardrailRequest { - pub data: Value, -} - -impl GuardrailRequest { - pub fn new(data: Value) -> Self { - Self { data } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum GuardrailDecision { - Allow(GuardrailRequest), - Mask(GuardrailRequest), - Block(GuardrailError), -} - -impl GuardrailDecision { - pub(super) fn into_request(self) -> Result { - match self { - Self::Allow(request) | Self::Mask(request) => Ok(request), - Self::Block(error) => Err(error), - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct GuardrailDispatchReport { - pub invoked: usize, -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs deleted file mode 100644 index 792717dacfc..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs +++ /dev/null @@ -1,317 +0,0 @@ -//! The `CustomLogger` trait — the Rust mirror of Python -//! `litellm/integrations/custom_logger.py::CustomLogger`. -//! -//! The Python-named async terminal methods are the public Rust callback shape. - -use std::sync::Arc; - -pub mod types; - -pub use types::{ - CallType, CallbackDispatchReport, CallbackTiming, CallbackValue, LogError, LogFuture, - LoggingError, ModelCallDetails, -}; - -pub trait CustomLogger: Send + Sync { - /// Python 1:1 name: `async_log_success_event(model_call_details, response_obj, start_time, end_time)`. - fn async_log_success_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Ok(()) }) - } - - /// Python 1:1 name: `async_log_failure_event(model_call_details, response_obj, start_time, end_time)`. - fn async_log_failure_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Ok(()) }) - } -} - -pub struct CustomLoggerRunner { - loggers: Vec>, -} - -impl CustomLoggerRunner { - pub fn new(loggers: Vec>) -> Self { - Self { loggers } - } - - pub fn is_empty(&self) -> bool { - self.loggers.is_empty() - } - - pub async fn async_log_success_event( - &self, - model_call_details: &ModelCallDetails, - response_obj: &CallbackValue, - timing: CallbackTiming, - ) -> CallbackDispatchReport { - if self.loggers.is_empty() { - return CallbackDispatchReport::default(); - } - - let mut report = CallbackDispatchReport::default(); - for logger in &self.loggers { - report.invoked += 1; - if let Err(err) = logger - .async_log_success_event(model_call_details, response_obj, timing) - .await - { - report.dropped += 1; - eprintln!("litellm-ai-gateway: async_log_success_event dropped: {err}"); - } - } - report - } - - pub async fn async_log_failure_event( - &self, - model_call_details: &ModelCallDetails, - response_obj: Option<&CallbackValue>, - timing: CallbackTiming, - ) -> CallbackDispatchReport { - if self.loggers.is_empty() { - return CallbackDispatchReport::default(); - } - - let mut report = CallbackDispatchReport::default(); - for logger in &self.loggers { - report.invoked += 1; - if let Err(err) = logger - .async_log_failure_event(model_call_details, response_obj, timing) - .await - { - report.dropped += 1; - eprintln!("litellm-ai-gateway: async_log_failure_event dropped: {err}"); - } - } - report - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::integrations::types::{StandardLoggingMetadata, StandardLoggingPayload}; - use serde_json::json; - use std::sync::Mutex; - - #[derive(Clone, Debug, PartialEq)] - struct RecordedEvent { - hook: &'static str, - model: String, - provider: String, - call_type: String, - request_id: Option, - litellm_call_id: Option, - user_id: Option, - response_object: Option, - error_kind: Option, - start_time: f64, - end_time: f64, - standard_logging_model: Option, - } - - #[derive(Default)] - struct RecordingCustomLogger { - events: Mutex>, - } - - impl RecordingCustomLogger { - fn events(&self) -> Vec { - self.events.lock().unwrap().clone() - } - } - - impl CustomLogger for RecordingCustomLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: &'a CallbackValue, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push(RecordedEvent { - hook: "async_log_success_event", - model: model_call_details.model.clone(), - provider: model_call_details.custom_llm_provider.clone(), - call_type: model_call_details.call_type.to_string(), - request_id: model_call_details.request_id.clone(), - litellm_call_id: model_call_details.litellm_call_id.clone(), - user_id: model_call_details.metadata.user_api_key_user_id.clone(), - response_object: Some(response_obj.object.clone()), - error_kind: None, - start_time: timing.start_time, - end_time: timing.end_time, - standard_logging_model: model_call_details - .standard_logging_payload - .as_ref() - .map(|payload| payload.model.clone()), - }); - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: Option<&'a CallbackValue>, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push(RecordedEvent { - hook: "async_log_failure_event", - model: model_call_details.model.clone(), - provider: model_call_details.custom_llm_provider.clone(), - call_type: model_call_details.call_type.to_string(), - request_id: model_call_details.request_id.clone(), - litellm_call_id: model_call_details.litellm_call_id.clone(), - user_id: model_call_details.metadata.user_api_key_user_id.clone(), - response_object: response_obj.map(|value| value.object.clone()), - error_kind: model_call_details - .failure_error - .as_ref() - .map(|error| error.kind.clone()), - start_time: timing.start_time, - end_time: timing.end_time, - standard_logging_model: model_call_details - .standard_logging_payload - .as_ref() - .map(|payload| payload.model.clone()), - }); - Ok(()) - }) - } - } - - fn payload(call_type: &str, model: &str, provider: &str) -> StandardLoggingPayload { - StandardLoggingPayload { - id: format!("req_{call_type}"), - litellm_call_id: format!("call_{call_type}"), - call_type: call_type.to_string(), - model: model.to_string(), - custom_llm_provider: provider.to_string(), - response_cost: 0.25, - prompt_tokens: 3, - completion_tokens: 4, - total_tokens: 7, - start_time: 10.0, - end_time: 11.5, - stream: false, - metadata: StandardLoggingMetadata { - user_api_key_hash: Some("hash".to_string()), - user_api_key_user_id: Some("user".to_string()), - user_api_key_team_id: Some("team".to_string()), - ..Default::default() - }, - messages: Some(json!([{"role": "user", "content": "read this"}])), - } - } - - #[tokio::test] - async fn rust_custom_logger_reads_success_payload_for_ocr() { - let logger = Arc::new(RecordingCustomLogger::default()); - let runner = CustomLoggerRunner::new(vec![logger.clone()]); - let details = ModelCallDetails::from_standard_logging_payload(payload( - "ocr", - "mistral-ocr-latest", - "mistral", - )); - let response = CallbackValue::new("ocr", json!({"pages": [{"markdown": "ok"}]})); - let report = runner - .async_log_success_event(&details, &response, CallbackTiming::new(10.0, 11.5)) - .await; - - assert_eq!(report.invoked, 1); - assert_eq!(report.dropped, 0); - assert_eq!( - logger.events(), - vec![RecordedEvent { - hook: "async_log_success_event", - model: "mistral-ocr-latest".to_string(), - provider: "mistral".to_string(), - call_type: "ocr".to_string(), - request_id: Some("req_ocr".to_string()), - litellm_call_id: Some("call_ocr".to_string()), - user_id: Some("user".to_string()), - response_object: Some("ocr".to_string()), - error_kind: None, - start_time: 10.0, - end_time: 11.5, - standard_logging_model: Some("mistral-ocr-latest".to_string()), - }] - ); - } - - #[tokio::test] - async fn rust_custom_logger_reads_failure_payload_for_non_ocr_call_type() { - let logger = Arc::new(RecordingCustomLogger::default()); - let runner = CustomLoggerRunner::new(vec![logger.clone()]); - let details = ModelCallDetails::from_standard_logging_payload(payload( - "acompletion", - "gpt-4.1-mini", - "openai", - )) - .with_failure_error(LoggingError { - message: "provider failed".to_string(), - kind: "ProviderError".to_string(), - }); - let response = CallbackValue::new("error", json!({"message": "provider failed"})); - let report = runner - .async_log_failure_event(&details, Some(&response), CallbackTiming::new(2.0, 3.0)) - .await; - - assert_eq!(report.invoked, 1); - assert_eq!(report.dropped, 0); - assert_eq!( - logger.events(), - vec![RecordedEvent { - hook: "async_log_failure_event", - model: "gpt-4.1-mini".to_string(), - provider: "openai".to_string(), - call_type: "acompletion".to_string(), - request_id: Some("req_acompletion".to_string()), - litellm_call_id: Some("call_acompletion".to_string()), - user_id: Some("user".to_string()), - response_object: Some("error".to_string()), - error_kind: Some("ProviderError".to_string()), - start_time: 2.0, - end_time: 3.0, - standard_logging_model: Some("gpt-4.1-mini".to_string()), - }] - ); - } - - #[tokio::test] - async fn no_callback_fast_path_dispatches_nothing() { - let runner = CustomLoggerRunner::new(Vec::new()); - let details = ModelCallDetails::new("mistral-ocr-latest", "mistral", CallType::Ocr); - let response = CallbackValue::new("ocr", json!({})); - - let report = runner - .async_log_success_event(&details, &response, CallbackTiming::new(1.0, 1.5)) - .await; - - assert!(runner.is_empty()); - assert_eq!(report, CallbackDispatchReport::default()); - } - - #[test] - fn with_standard_logging_payload_keeps_top_level_fields_in_sync() { - let details = ModelCallDetails::new("old-model", "old-provider", CallType::Completion) - .with_standard_logging_payload(payload("ocr", "mistral-ocr-latest", "mistral")); - - assert_eq!(details.model, "mistral-ocr-latest"); - assert_eq!(details.custom_llm_provider, "mistral"); - assert_eq!(details.call_type, CallType::Ocr); - assert_eq!(details.request_id, Some("req_ocr".to_string())); - assert_eq!(details.litellm_call_id, Some("call_ocr".to_string())); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs deleted file mode 100644 index ba7d67bd46e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; - -use serde_json::Value; - -use crate::integrations::types::{StandardLoggingMetadata, StandardLoggingPayload}; - -pub type LogFuture<'a> = Pin> + Send + 'a>>; - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct CallbackDispatchReport { - pub invoked: usize, - pub dropped: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum CallType { - Ocr, - Realtime, - Completion, - Acompletion, - ChatCompletion, - Other(String), -} - -impl CallType { - pub fn as_str(&self) -> &str { - match self { - Self::Ocr => "ocr", - Self::Realtime => "realtime", - Self::Completion => "completion", - Self::Acompletion => "acompletion", - Self::ChatCompletion => "chat_completion", - Self::Other(value) => value.as_str(), - } - } -} - -impl From<&str> for CallType { - fn from(value: &str) -> Self { - match value { - "ocr" => Self::Ocr, - "realtime" => Self::Realtime, - "completion" => Self::Completion, - "acompletion" => Self::Acompletion, - "chat_completion" => Self::ChatCompletion, - other => Self::Other(other.to_string()), - } - } -} - -impl std::fmt::Display for CallType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct CallbackTiming { - pub start_time: f64, - pub end_time: f64, -} - -impl CallbackTiming { - pub fn new(start_time: f64, end_time: f64) -> Self { - Self { - start_time, - end_time, - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct CallbackValue { - pub object: String, - pub value: Value, -} - -impl CallbackValue { - pub fn new(object: impl Into, value: Value) -> Self { - Self { - object: object.into(), - value, - } - } -} - -#[derive(Clone, Debug)] -pub struct ModelCallDetails { - pub model: String, - pub custom_llm_provider: String, - pub call_type: CallType, - pub metadata: StandardLoggingMetadata, - pub extra_metadata: HashMap, - pub request_id: Option, - pub litellm_call_id: Option, - pub response_cost: Option, - pub standard_logging_payload: Option, - pub failure_error: Option, -} - -impl ModelCallDetails { - pub fn new( - model: impl Into, - custom_llm_provider: impl Into, - call_type: CallType, - ) -> Self { - Self { - model: model.into(), - custom_llm_provider: custom_llm_provider.into(), - call_type, - metadata: StandardLoggingMetadata::default(), - extra_metadata: HashMap::new(), - request_id: None, - litellm_call_id: None, - response_cost: None, - standard_logging_payload: None, - failure_error: None, - } - } - - pub fn from_standard_logging_payload(payload: StandardLoggingPayload) -> Self { - let request_id = Some(payload.id.clone()); - let litellm_call_id = Some(payload.litellm_call_id.clone()); - let response_cost = Some(payload.response_cost); - let metadata = payload.metadata.clone(); - Self { - model: payload.model.clone(), - custom_llm_provider: payload.custom_llm_provider.clone(), - call_type: CallType::from(payload.call_type.as_str()), - metadata, - extra_metadata: HashMap::new(), - request_id, - litellm_call_id, - response_cost, - standard_logging_payload: Some(payload), - failure_error: None, - } - } - - pub fn with_standard_logging_payload(mut self, payload: StandardLoggingPayload) -> Self { - self.model = payload.model.clone(); - self.custom_llm_provider = payload.custom_llm_provider.clone(); - self.call_type = CallType::from(payload.call_type.as_str()); - self.request_id = Some(payload.id.clone()); - self.litellm_call_id = Some(payload.litellm_call_id.clone()); - self.response_cost = Some(payload.response_cost); - self.metadata = payload.metadata.clone(); - self.standard_logging_payload = Some(payload); - self - } - - pub fn with_failure_error(mut self, error: LoggingError) -> Self { - self.failure_error = Some(error); - self - } -} - -#[derive(Clone, Debug)] -pub struct LoggingError { - pub message: String, - pub kind: String, -} - -#[derive(Clone, Debug)] -pub struct LogError { - pub message: String, - pub kind: String, -} - -impl LogError { - pub fn channel_full() -> Self { - Self { - message: "logging channel is full; dropping record".to_string(), - kind: "ChannelFull".to_string(), - } - } - - pub fn channel_closed() -> Self { - Self { - message: "logging channel is closed; worker has shut down".to_string(), - kind: "ChannelClosed".to_string(), - } - } -} - -impl std::fmt::Display for LogError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.kind, self.message) - } -} - -impl std::error::Error for LogError {} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs deleted file mode 100644 index 3dad18cb7a3..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs +++ /dev/null @@ -1,197 +0,0 @@ -//! A `CustomLogger` that ships finished events to the LiteLLM Python proxy's -//! `/v1/rust_control_plane/logs` endpoint. -//! -//! The callback path is non-blocking: `async_log_success_event` / -//! `async_log_failure_event` -//! build a `LogRecord` and `try_send` it onto a bounded channel, returning a -//! `LogError` (never panicking, never awaiting) if the channel is full or the -//! worker has gone away. A spawned background worker drains the channel, batches -//! records into `{"records":[...]}`, and POSTs them to the proxy with a pooled -//! `reqwest::Client`. - -use std::sync::Arc; -use std::time::Duration; - -use reqwest::Client; -use tokio::sync::mpsc::{self, Receiver, Sender}; -use tokio::time::interval; - -use crate::constants::{DEFAULT_PROXY_BASE_URL, RUST_CONTROL_PLANE_LOGS_PATH}; -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, LogError, LogFuture, LoggingError, - ModelCallDetails, -}; -use types::{CallbackLogsRequest, EgressTunables, LogRecord}; - -pub mod types; - -/// Ships realtime logging events to the LiteLLM Python proxy. -pub struct LiteLLMPythonProxyAPILogger { - sink: Sender, -} - -impl LiteLLMPythonProxyAPILogger { - /// Spawn the background worker and return a logger handle. `base` is the - /// proxy base URL (no trailing path); `master_key` is sent as a bearer token. - pub fn start(base: String, master_key: String) -> Arc { - let tunables = EgressTunables::from_env(); - let (sink, receiver) = mpsc::channel::(tunables.channel_capacity); - let url = format!( - "{}{}", - base.trim_end_matches('/'), - RUST_CONTROL_PLANE_LOGS_PATH - ); - let client = Client::new(); - tokio::spawn(worker_loop( - receiver, - client, - url, - master_key, - tunables.max_batch_size, - tunables.flush_interval, - )); - Arc::new(Self { sink }) - } - - /// Build a logger from the environment: `LITELLM_PROXY_BASE_URL` (default - /// `http://localhost:4000`) and `LITELLM_MASTER_KEY`. - /// - /// `LITELLM_PROXY_BASE_URL` is treated as the full base and the route is - /// appended verbatim, so if the proxy runs under a `SERVER_ROOT_PATH` - /// (e.g. served at `https://host/litellm`), include it in the base - /// (`LITELLM_PROXY_BASE_URL=https://host/litellm`) and the POST lands at - /// `https://host/litellm/v1/rust_control_plane/logs`. - pub fn from_env() -> Arc { - let base = std::env::var("LITELLM_PROXY_BASE_URL") - .ok() - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| DEFAULT_PROXY_BASE_URL.to_string()); - let key = std::env::var("LITELLM_MASTER_KEY").unwrap_or_default(); - Self::start(base, key) - } - - fn enqueue(&self, record: LogRecord) -> Result<(), LogError> { - self.sink.try_send(record).map_err(|err| match err { - mpsc::error::TrySendError::Full(_) => LogError::channel_full(), - mpsc::error::TrySendError::Closed(_) => LogError::channel_closed(), - }) - } -} - -impl CustomLogger for LiteLLMPythonProxyAPILogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - if let Some(payload) = &model_call_details.standard_logging_payload { - self.enqueue(LogRecord { - status: "success".to_string(), - payload: payload.clone(), - error: None, - })?; - } - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - if let Some(payload) = &model_call_details.standard_logging_payload { - let fallback_error; - let error = match &model_call_details.failure_error { - Some(error) => error, - None => { - fallback_error = LoggingError { - message: "callback failure event".to_string(), - kind: "CallbackFailure".to_string(), - }; - &fallback_error - } - }; - self.enqueue(LogRecord { - status: "failure".to_string(), - payload: payload.clone(), - error: Some(format!("{}: {}", error.kind, error.message)), - })?; - } - Ok(()) - }) - } -} - -/// Drain the channel, batching records and POSTing them to the proxy. Exits when -/// the channel is closed (all senders dropped) and drained. -async fn worker_loop( - mut receiver: Receiver, - client: Client, - url: String, - master_key: String, - max_batch_size: usize, - flush_interval: Duration, -) { - let mut ticker = interval(flush_interval); - let mut batch: Vec = Vec::with_capacity(max_batch_size); - - loop { - tokio::select! { - maybe_record = receiver.recv() => { - match maybe_record { - Some(record) => { - batch.push(record); - if batch.len() >= max_batch_size { - flush(&client, &url, &master_key, &mut batch).await; - } - } - None => { - // Channel closed: flush remaining and exit. - flush(&client, &url, &master_key, &mut batch).await; - break; - } - } - } - _ = ticker.tick() => { - flush(&client, &url, &master_key, &mut batch).await; - } - } - } -} - -/// POST the current batch (if any), clearing it. Errors are logged, not fatal. -async fn flush(client: &Client, url: &str, master_key: &str, batch: &mut Vec) { - if batch.is_empty() { - return; - } - let records = std::mem::take(batch) - .into_iter() - .map(LogRecord::into_callback_record) - .collect(); - let body = CallbackLogsRequest { records }; - - let response = client - .post(url) - .bearer_auth(master_key) - .json(&body) - .send() - .await; - - match response { - Ok(resp) if resp.status().is_success() => {} - Ok(resp) => { - eprintln!( - "litellm-ai-gateway: callback logs POST returned {} to {url}", - resp.status() - ); - } - Err(err) => { - eprintln!("litellm-ai-gateway: callback logs POST failed to {url}: {err}"); - } - } -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs deleted file mode 100644 index 481a437747f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::time::Duration; - -use serde::Serialize; - -use crate::constants::{ - DEFAULT_CHANNEL_CAPACITY, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_MAX_BATCH_SIZE, -}; -use crate::integrations::types::StandardLoggingPayload; - -#[derive(Serialize)] -pub struct CallbackLogsRequest { - pub records: Vec, -} - -#[derive(Serialize)] -pub struct CallbackLogRecord { - pub status: String, - pub standard_logging_payload: StandardLoggingPayload, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Clone, Debug)] -pub struct LogRecord { - pub status: String, - pub payload: StandardLoggingPayload, - pub error: Option, -} - -impl LogRecord { - pub fn into_callback_record(self) -> CallbackLogRecord { - CallbackLogRecord { - status: self.status, - standard_logging_payload: self.payload, - error: self.error, - } - } -} - -pub(super) struct EgressTunables { - pub channel_capacity: usize, - pub max_batch_size: usize, - pub flush_interval: Duration, -} - -impl EgressTunables { - pub fn from_env() -> Self { - Self { - channel_capacity: env_positive( - "LITELLM_LOG_CHANNEL_CAPACITY", - DEFAULT_CHANNEL_CAPACITY, - ), - max_batch_size: env_positive("LITELLM_LOG_BATCH_SIZE", DEFAULT_MAX_BATCH_SIZE), - flush_interval: Duration::from_millis(env_positive( - "LITELLM_LOG_FLUSH_INTERVAL_MS", - DEFAULT_FLUSH_INTERVAL_MS, - )), - } - } -} - -fn env_positive(name: &str, default: T) -> T -where - T: std::str::FromStr + PartialOrd + From, -{ - let zero = T::from(0u8); - std::env::var(name) - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|n| *n > zero) - .unwrap_or(default) -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/mod.rs deleted file mode 100644 index c62f1821ef8..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Pure-Rust logging integrations. Names map 1:1 to Python -//! `litellm/integrations/`: -//! - [`custom_guardrail::CustomGuardrail`] — the guardrail callback trait -//! - [`custom_logger::CustomLogger`] — the callback trait -//! - [`litellm_python_proxy_api::LiteLLMPythonProxyAPILogger`] — ships events -//! to the Python proxy's `/v1/rust_control_plane/logs` endpoint -//! - [`types`] — the typed `StandardLoggingPayload` wire contract - -pub mod custom_guardrail; -pub mod custom_logger; -pub mod litellm_python_proxy_api; -pub mod types; diff --git a/litellm-rust/crates/ai-gateway/src/integrations/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/types.rs deleted file mode 100644 index 34dce93d8e0..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/types.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! Typed payloads for the LiteLLM `/v1/callbacks/logs` realtime-logging contract. -//! -//! Field names below are the EXACT JSON keys the Python replay path + spend-logs -//! builder read. Note the deliberate mix: -//! - `startTime` / `endTime` are camelCase (epoch f64 seconds) -//! - `response_cost` / `prompt_tokens` / etc. are snake_case -//! -//! Mirrors Python `litellm/integrations/` + the proxy `CallbackLogsRequest` -//! contract 1:1. - -use serde::Serialize; -use serde_json::Value; -use std::collections::HashMap; - -/// Cumulative token usage for a realtime session. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Usage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, -} - -/// Cost-attribution metadata threaded from the authenticated request. -#[derive(Clone, Debug, Default)] -pub struct RequestMetadata { - pub user_api_key_hash: Option, - pub user_api_key_user_id: Option, - pub user_api_key_team_id: Option, -} - -/// The self-describing payload. Field names are the EXACT JSON keys the Python -/// replay path + spend-logs builder read. -#[derive(Clone, Debug, Serialize)] -pub struct StandardLoggingPayload { - pub id: String, - pub litellm_call_id: String, - - /// e.g. "realtime", "acompletion". Falls back to "acompletion" if absent. - pub call_type: String, - - pub model: String, - pub custom_llm_provider: String, - - /// Spend ($) written to LiteLLM_SpendLogs.spend. - pub response_cost: f64, - - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - - /// EPOCH SECONDS as float — camelCase keys, NOT snake_case. - #[serde(rename = "startTime")] - pub start_time: f64, - #[serde(rename = "endTime")] - pub end_time: f64, - - pub stream: bool, - - pub metadata: StandardLoggingMetadata, - - /// Optional; stored as request input on the spend log row. - #[serde(skip_serializing_if = "Option::is_none")] - pub messages: Option, -} - -/// Cost-attribution keys. The replayer maps these into litellm_params.metadata, -/// which the spend-logs builder reads to set user / team_id / organization_id. -#[derive(Clone, Debug, Serialize, Default)] -pub struct StandardLoggingMetadata { - pub user_api_key_hash: Option, // -> SpendLogs.api_key - pub user_api_key_user_id: Option, // -> SpendLogs.user - pub user_api_key_team_id: Option, // -> SpendLogs.team_id - - // Optional but read by the builder; include when known: - #[serde(skip_serializing_if = "Option::is_none")] - pub user_api_key_alias: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_api_key_org_id: Option, // -> SpendLogs.organization_id - #[serde(skip_serializing_if = "Option::is_none")] - pub user_api_key_end_user_id: Option, // -> SpendLogs.end_user - #[serde(skip_serializing_if = "Option::is_none")] - pub spend_logs_metadata: Option>, -} diff --git a/litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs b/litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs deleted file mode 100644 index 80d9e401a5f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::audio_transcription::{AudioTranscriptionRequest, audio_transcription}; diff --git a/litellm-rust/crates/ai-gateway/src/io/mod.rs b/litellm-rust/crates/ai-gateway/src/io/mod.rs deleted file mode 100644 index 7098d67993f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod audio_transcription; -pub mod ocr; -pub mod realtime; -pub mod realtime_pool; -pub mod responses_ws; -pub(crate) mod tls; diff --git a/litellm-rust/crates/ai-gateway/src/io/ocr.rs b/litellm-rust/crates/ai-gateway/src/io/ocr.rs deleted file mode 100644 index 2fc82f0b61f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/ocr.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::ocr::{OcrRequest, ocr}; diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs deleted file mode 100644 index 1aa31adcc38..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ /dev/null @@ -1,418 +0,0 @@ -//! End-to-end OpenAI realtime invocation. -//! -//! The host-facing entry point opens the WebSocket to OpenAI, then splices a -//! client realtime stream to the upstream, driving typed events through the pure -//! `OPENAI_REALTIME_CONFIG` transforms. -//! Network, auth header, key resolution, and wire (de)serialization live here so -//! the `transformation` module stays pure and typed. -//! -//! The dial and splice steps are factored out ([`dial_upstream`], [`splice`]) so -//! the connection pool ([`crate::io::realtime_pool`]) can pre-establish an upstream, -//! buffer its `session.created`, and later hand the live socket to the same -//! splice loop a fresh dial uses. - -use std::time::Duration; - -use futures_util::stream::{SplitSink, SplitStream}; -use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::AuthError; -use litellm_core::auth::error::MissingCredential; -use litellm_core::error::Error; -use litellm_core::realtime::transformation::RealtimeProviderConfig; -use litellm_core::realtime::types::RealtimeEvent; -use tokio::net::TcpStream; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::tungstenite::http::HeaderValue; -use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; -use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; - -use litellm_core::providers::openai::realtime::transformation::OPENAI_REALTIME_CONFIG; - -use crate::io::tls::connect_upstream; - -/// Environment variable holding the OpenAI API key (last-resort fallback). -const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; - -/// Default **idle** timeout: if neither side sends a frame for this long, the -/// session is reaped. It resets on any activity, so it does not cap a healthy -/// (continuously streaming) session — it only frees a stalled one (e.g. a -/// half-open upstream that keeps the socket open but stops sending). -const DEFAULT_IDLE_TIMEOUT_SECS: u64 = 300; - -/// The concrete upstream WebSocket type (TLS or plain). Shared by the dial path -/// and the pool so warm sockets and fresh sockets are the exact same type. -pub type UpstreamWs = WebSocketStream>; -pub(crate) type UpstreamTx = SplitSink; -pub(crate) type UpstreamRx = SplitStream; - -/// Resolve the OpenAI API key from the explicit param or the environment. -/// -/// Blank/whitespace values are treated as absent (guard at resolution time). -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| { - std::env::var(OPENAI_API_KEY_ENV) - .ok() - .filter(|key| !key.trim().is_empty()) - }) - .ok_or_else(|| Error::from(AuthError::from(MissingCredential::OpenAiRealtimeApiKey))) -} - -/// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. -/// -/// This is the dial half of [`realtime`], factored out so the pool can -/// pre-establish sockets ahead of any client. `api_key` here is already resolved -/// (non-blank) — the pool resolves it once when it is created. -pub(crate) async fn dial_upstream( - model: &str, - api_key: &str, - api_base: Option<&str>, -) -> Result { - let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); - - let mut request = url - .as_str() - .into_client_request() - .map_err(|err| Error::Network(err.to_string()))?; - // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers - // beta_api_shape_disabled, so we do not send it. - request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|err| Error::Auth(err.to_string()))?, - ); - - let (upstream, _response) = connect_upstream(request) - .await - .map_err(|err| Error::Network(err.to_string()))?; - Ok(upstream) -} - -/// Read the next text frame from the upstream and decode it as a typed event. -/// -/// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an -/// error on a non-text frame, a closed socket, or undecodable JSON so the pool can -/// discard a misbehaving socket rather than warm it. -pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result { - loop { - let message = upstream_rx - .next() - .await - .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))? - .map_err(|err| Error::Network(err.to_string()))?; - match message { - Message::Text(text) => { - return serde_json::from_str(&text) - .map_err(|err| Error::InvalidResponse(err.to_string())); - } - // Ignore protocol frames (ping/pong) while waiting for the first event. - Message::Ping(_) | Message::Pong(_) => continue, - Message::Close(_) => { - return Err(Error::Network( - "upstream closed before first event".to_string(), - )); - } - _ => continue, - } - } -} - -/// Splice an already-connected upstream to the client streams. -/// -/// `prelude` is relayed to the client first (the pool passes the buffered -/// `session.created` here; the fresh-dial path passes `None` and lets the upstream -/// deliver it). Then a single select loop forwards both directions through the -/// transforms until either side closes or the idle timeout fires. -/// `observe` is invoked on **upstream→client** events only (the trusted side that -/// carries `session.created` and `response.done` usage) — never on client events, -/// so a client cannot fabricate usage into its own logs. -#[allow(clippy::too_many_arguments)] -pub(crate) async fn splice( - model: &str, - mut upstream_tx: UpstreamTx, - mut upstream_rx: UpstreamRx, - prelude: Option, - idle_timeout: Option, - mut observe: impl FnMut(&RealtimeEvent) + Send, - mut client_in: In, - mut client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - let config = &OPENAI_REALTIME_CONFIG; - - // Relay a buffered backend event (warm handoff's session.created) first, so a - // warm session looks identical to a fresh one from the client's view. - if let Some(event) = prelude { - for outbound in config.transform_realtime_response(&event, model)?.events { - client_out - .send(outbound) - .await - .map_err(|err| Error::Network(err.to_string()))?; - } - } - - let idle = idle_timeout.unwrap_or(Duration::from_secs(DEFAULT_IDLE_TIMEOUT_SECS)); - - // One loop forwarding both directions. The `sleep(idle)` arm is rebuilt every - // iteration, so any frame (either way) resets it — it fires only when the - // session has been fully idle for `idle`, reaping a stalled connection - // (task + upstream TCP socket) instead of leaking it. - loop { - tokio::select! { - // client -> upstream - client_event = client_in.next() => { - let Some(event) = client_event else { break }; // client disconnected - // NOTE: do NOT observe client events. session.created / response.done - // (carrying usage) are server→client events; observing the client arm - // would let an authenticated client POST a fabricated response.done and - // inflate its own spend log. Logging observes upstream events only. - for outbound in config.transform_realtime_request(&event, model)?.events { - let payload = serde_json::to_string(&outbound) - .map_err(|err| Error::InvalidResponse(err.to_string()))?; - upstream_tx - .send(Message::Text(payload)) - .await - .map_err(|err| Error::Network(err.to_string()))?; - } - } - // upstream -> client - upstream_message = upstream_rx.next() => { - let Some(message) = upstream_message else { break }; // upstream closed - match message.map_err(|err| Error::Network(err.to_string()))? { - Message::Text(text) => { - let event: RealtimeEvent = serde_json::from_str(&text) - .map_err(|err| Error::InvalidResponse(err.to_string()))?; - observe(&event); - for outbound in config.transform_realtime_response(&event, model)?.events { - client_out - .send(outbound) - .await - .map_err(|err| Error::Network(err.to_string()))?; - } - } - Message::Close(_) => break, - _ => {} - } - } - // idle timeout: no activity from either side within `idle` - _ = tokio::time::sleep(idle) => break, - } - } - Ok(()) -} - -/// Splice a client realtime stream to OpenAI: forward client events upstream -/// (via `transform_realtime_request`) and backend events downstream (via -/// `transform_realtime_response`). Returns when either side closes. -/// -/// Generic over the client transport (typed events) so this crate stays -/// framework-agnostic; the gateway adapts its axum socket to these. This is the -/// fresh-dial path: dial, then splice. The pool's warm-handoff path skips the dial -/// and calls [`splice`] directly with a buffered `session.created`. -#[allow(clippy::too_many_arguments)] -pub async fn realtime( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, - idle_timeout: Option, - observe: impl FnMut(&RealtimeEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - let api_key = resolve_api_key(api_key)?; - let upstream = dial_upstream(model, &api_key, api_base).await?; - let (upstream_tx, upstream_rx) = upstream.split(); - splice( - model, - upstream_tx, - upstream_rx, - None, - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} - -/// Splice a pre-warmed upstream (taken from [`crate::io::realtime_pool`]) to the -/// client. Relays the buffered `session.created` first, then splices exactly like -/// the fresh-dial path — so a warm session is indistinguishable from a fresh one. -#[allow(clippy::too_many_arguments)] -pub async fn realtime_warm( - model: &str, - handoff: crate::io::realtime_pool::WarmHandoff, - idle_timeout: Option, - observe: impl FnMut(&RealtimeEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - splice( - model, - handoff.tx, - handoff.rx, - Some(handoff.session_created), - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} - -#[cfg(test)] -mod tests { - use super::*; - - fn event(raw: &str) -> RealtimeEvent { - serde_json::from_str(raw).expect("valid event json") - } - - /// The realtime dial has to reach a `wss://` upstream without a process-wide - /// crypto provider installed, which is what dialing through `io::tls` buys. - #[tokio::test] - async fn dial_upstream_over_wss_reports_an_error_instead_of_panicking() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind a loopback port"); - let port = listener - .local_addr() - .expect("read the bound address") - .port(); - tokio::spawn(async move { - while let Ok((stream, _peer)) = listener.accept().await { - drop(stream); - } - }); - - let result = dial_upstream( - "gpt-realtime", - "sk-test", - Some(&format!("wss://127.0.0.1:{port}")), - ) - .await; - - assert!(matches!(result, Err(Error::Network(_)))); - } - - #[test] - fn resolve_api_key_prefers_param_then_blank_falls_through() { - assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test"); - // A blank param with no env set should error. - if std::env::var(OPENAI_API_KEY_ENV).is_err() { - assert!(resolve_api_key(Some(" ")).is_err()); - } - } - - /// Live end-to-end check against OpenAI. Ignored by default (CI never runs - /// it); run explicitly with `OPENAI_API_KEY` set: - /// `cargo test -p litellm-ai-gateway --features server realtime_invokes_openai -- --ignored --nocapture` - #[tokio::test] - #[ignore = "hits the live OpenAI realtime API; needs OPENAI_API_KEY"] - async fn realtime_invokes_openai_and_responds() { - use futures_channel::mpsc; - - let key = - std::env::var(OPENAI_API_KEY_ENV).expect("set OPENAI_API_KEY to run this ignored test"); - - // client -> provider (we hold `client_tx` to push events upstream) - let (mut client_tx, client_in) = mpsc::unbounded::(); - // provider -> client (we hold `backend_rx` to read backend events) - let (client_out, mut backend_rx) = mpsc::unbounded::(); - - // Clone the key so the spawned task owns its `String` (no borrow across await). - let key_owned = key.clone(); - let call = tokio::spawn(async move { - realtime( - "gpt-realtime", - Some(&key_owned), - None, - None, - |_| {}, - client_in, - client_out, - ) - .await - }); - - // 1. First backend event should be session.created. - let first = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()) - .await - .expect("timed out waiting for session.created") - .expect("backend stream closed before session.created"); - assert_eq!( - first.event_type, "session.created", - "expected session.created, got: {}", - first.event_type - ); - - // 2. Ask for a short audio response. - client_tx - .send(event( - r#"{"type":"conversation.item.create","item":{"type":"message","role":"user","content":[{"type":"input_text","text":"Say hi."}]}}"#, - )) - .await - .expect("send conversation.item.create"); - client_tx - .send(event(r#"{"type":"response.create"}"#)) - .await - .expect("send response.create"); - - // 3. Read backend events; require a non-empty audio delta, then response.done. - let mut saw_audio_delta = false; - let mut saw_done = false; - for _ in 0..500 { - let next = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()).await; - let event = match next { - Ok(Some(event)) => event, - Ok(None) => break, - Err(_) => panic!("timed out waiting for backend events"), - }; - match event.event_type.as_str() { - "response.output_audio.delta" => { - let delta = event - .data - .get("delta") - .and_then(|value| value.as_str()) - .unwrap_or(""); - if !delta.is_empty() { - saw_audio_delta = true; - } - } - "response.done" => { - saw_done = true; - break; - } - _ => {} - } - } - - assert!( - saw_audio_delta, - "expected a response.output_audio.delta with non-empty delta" - ); - assert!(saw_done, "expected a response.done event"); - - // Drop the client sender so the provider's to_upstream side finishes. - drop(client_tx); - let _ = call.await; - } -} diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs deleted file mode 100644 index 49e9c459a88..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs +++ /dev/null @@ -1,712 +0,0 @@ -//! Pre-warmed upstream realtime connection pool. -//! -//! The gateway's realtime overhead lives entirely in session establishment: on -//! every client connect it dials a fresh upstream WS to OpenAI and waits for -//! `session.created` before it can serve. This pool keeps a small set of upstream -//! sockets **already connected and already past `session.created`** so a connect -//! can be served from a warm socket and the handshake is off the critical path. -//! -//! Layering: this lives in the gateway's `io` module next to the dial/splice it -//! reuses. The gateway holds an `Arc` in its state and asks for a -//! warm socket per connect; on a miss it fresh-dials exactly as before. The pool -//! is a latency optimization, never a correctness dependency — see the gateway's -//! `src/routes/realtime/README.md`. -//! -//! ## Caveats (enforced here) -//! - One warm socket serves exactly one session (realtime isn't multiplexed), so -//! the pool is sized to the connect *rate*, not concurrent connections. -//! - `session.created` is pre-read once and buffered; nothing else is read from a -//! warm socket before handoff, so a warm session starts at OpenAI defaults just -//! like a fresh one (`session.update` semantics unchanged). -//! - Warm sockets are short-lived (`max_idle`) and liveness-checked at handoff to -//! bound idle billing / dodge OpenAI's idle timeout. -//! - On miss or dead socket the caller fresh-dials; the pool never blocks or fails -//! a connect because it is empty. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -use futures_util::StreamExt; -use litellm_core::Error; -use litellm_core::realtime::types::RealtimeEvent; - -use crate::io::realtime::{ - UpstreamRx, UpstreamTx, UpstreamWs, dial_upstream, read_event, resolve_api_key, -}; - -/// Default target warm sockets per key when pooling is enabled. -pub const DEFAULT_POOL_SIZE: usize = 4; - -/// Default max time a warm socket may sit before it is closed and replaced. -pub const DEFAULT_MAX_IDLE: Duration = Duration::from_secs(30); - -/// Env var: target warm sockets per key. `0` disables pooling (fresh-dial only). -pub const POOL_SIZE_ENV: &str = "REALTIME_POOL_SIZE"; - -/// Env var: max warm-socket idle lifetime, in seconds. -pub const MAX_IDLE_ENV: &str = "REALTIME_POOL_MAX_IDLE_SECS"; - -/// How often the background replenisher wakes to top up and reap stale sockets. -const REPLENISH_TICK: Duration = Duration::from_millis(250); - -/// Backoff floor after a key's warm-up dials all fail. The first failed pass -/// waits this long before retrying that key. -const BACKOFF_BASE: Duration = Duration::from_millis(500); - -/// Backoff ceiling. A key that keeps failing (invalid credentials, an -/// unreachable upstream) is retried at most once per this interval — instead of -/// firing `needed` concurrent TLS dials every 250 ms tick, which would hammer -/// the upstream and risk rate-limit exhaustion that degrades valid cold-path -/// traffic. Backoff resets the moment a dial for the key succeeds. -const BACKOFF_MAX: Duration = Duration::from_secs(30); - -/// Identifies an upstream connection: the tuple that fully determines the dial. -/// `api_key` is included so a warm socket is only ever reused for the same key -/// (no cross-tenant reuse). -#[derive(Clone, PartialEq, Eq, Hash)] -pub struct UpstreamKey { - pub model: String, - pub api_key: String, - pub api_base: Option, -} - -impl std::fmt::Debug for UpstreamKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("UpstreamKey") - .field("model", &self.model) - .field("api_key", &"[REDACTED]") - .field("api_base", &self.api_base) - .finish() - } -} - -/// A warm upstream: split halves + the buffered `session.created` + when it was -/// warmed (for `max_idle` expiry). -struct WarmConnection { - tx: UpstreamTx, - rx: UpstreamRx, - session_created: RealtimeEvent, - warmed_at: Instant, -} - -/// A live upstream taken from the pool, ready to splice. The caller relays -/// `session_created` to the client first, then splices `(tx, rx)` as usual. -pub struct WarmHandoff { - pub tx: UpstreamTx, - pub rx: UpstreamRx, - pub session_created: RealtimeEvent, -} - -/// Pool configuration, resolved once at startup from the environment. -#[derive(Clone, Copy, Debug)] -pub struct PoolConfig { - /// Target warm sockets per key. `0` disables pooling. - pub target_size: usize, - /// Max time a warm socket may sit before it is closed and replaced. - pub max_idle: Duration, -} - -impl Default for PoolConfig { - fn default() -> Self { - Self { - target_size: DEFAULT_POOL_SIZE, - max_idle: DEFAULT_MAX_IDLE, - } - } -} - -impl PoolConfig { - /// Read config from the environment, falling back to defaults. An invalid - /// value warns and uses the default rather than failing startup. - pub fn from_env() -> Self { - let target_size = match std::env::var(POOL_SIZE_ENV) { - Ok(raw) => raw.trim().parse().unwrap_or_else(|_| { - eprintln!("warning: {POOL_SIZE_ENV}={raw:?} is not a valid size; using {DEFAULT_POOL_SIZE}"); - DEFAULT_POOL_SIZE - }), - Err(_) => DEFAULT_POOL_SIZE, - }; - let max_idle = match std::env::var(MAX_IDLE_ENV) { - Ok(raw) => raw - .trim() - .parse() - .map(Duration::from_secs) - .unwrap_or_else(|_| { - eprintln!( - "warning: {MAX_IDLE_ENV}={raw:?} is not a valid number of seconds; using {}s", - DEFAULT_MAX_IDLE.as_secs() - ); - DEFAULT_MAX_IDLE - }), - Err(_) => DEFAULT_MAX_IDLE, - }; - Self { - target_size, - max_idle, - } - } - - /// Whether pooling is on (`target_size > 0`). - pub fn enabled(&self) -> bool { - self.target_size > 0 - } -} - -/// Per-key warm sockets, behind a single `Mutex`. Realtime warm sockets are few -/// (the pool is small), so a plain mutex over a `VecDeque`-ish `Vec` is simpler -/// and faster than sharding; contention is negligible at this scale. -type Warm = HashMap>; - -/// Per-key replenish backoff. Absent (or `consecutive_failures == 0`) means the -/// key is healthy and replenished every tick. After a pass whose dials all fail, -/// `retry_after` is pushed out with exponential backoff so a broken key (invalid -/// credentials, unreachable upstream) is not re-dialed on every 250 ms tick. -#[derive(Default)] -struct Backoff { - /// Don't attempt warm-up dials for this key until this instant. `None` = - /// eligible now. - retry_after: Option, - consecutive_failures: u32, -} - -type Backoffs = HashMap; - -/// Pre-warmed upstream realtime connection pool. -/// -/// Cheap to clone-via-`Arc`. The background replenisher is spawned by -/// [`RealtimePool::spawn`]; a pool built with [`RealtimePool::disabled`] never -/// warms anything and every `take` misses (callers fresh-dial). -pub struct RealtimePool { - config: PoolConfig, - warm: Mutex, - /// Per-key replenish backoff so a broken key doesn't trigger unbounded - /// concurrent dials every tick. Separate lock from `warm` so the request - /// hot path (`take`) never contends on it. - backoff: Mutex, -} - -impl RealtimePool { - /// A disabled pool: no background task, every `take` returns `None`. - pub fn disabled() -> Arc { - Arc::new(Self { - config: PoolConfig { - target_size: 0, - ..PoolConfig::default() - }, - warm: Mutex::new(HashMap::new()), - backoff: Mutex::new(HashMap::new()), - }) - } - - /// Build a pool from config **without** the background replenisher. The pool - /// only warms when [`RealtimePool::warm_now`] is called. Used by deterministic - /// unit tests; production uses [`RealtimePool::spawn`]. - #[cfg(test)] - fn new_unspawned(config: PoolConfig) -> Arc { - Arc::new(Self { - config, - warm: Mutex::new(HashMap::new()), - backoff: Mutex::new(HashMap::new()), - }) - } - - /// Build a pool from config and, if enabled, spawn the background replenisher. - /// Returns the shared handle the gateway stores in its state. - pub fn spawn(config: PoolConfig) -> Arc { - let pool = Arc::new(Self { - config, - warm: Mutex::new(HashMap::new()), - backoff: Mutex::new(HashMap::new()), - }); - if config.enabled() { - let weak = Arc::downgrade(&pool); - tokio::spawn(async move { - let mut tick = tokio::time::interval(REPLENISH_TICK); - tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - tick.tick().await; - // Stop once the gateway has dropped its handle. - let Some(pool) = weak.upgrade() else { break }; - pool.replenish_all().await; - } - }); - } - pool - } - - /// Resolved config (test/inspection). - pub fn config(&self) -> PoolConfig { - self.config - } - - /// Register a key so the replenisher starts warming it. Idempotent. The - /// gateway calls this once per known deployment at startup; the pool only - /// warms keys it has seen, so it never dials a model nobody asked for. - pub fn register(&self, key: UpstreamKey) { - if !self.config.enabled() { - return; - } - self.warm.lock().unwrap().entry(key).or_default(); - } - - /// Take a warm, live socket for `key`, or `None` on miss / dead socket. - /// - /// Pops the freshest non-expired socket and liveness-checks it; a socket that - /// is too old or already dead is dropped (closing it) and the next candidate - /// tried. Never blocks: if nothing warm is live, returns `None` so the caller - /// fresh-dials. - pub fn take(&self, key: &UpstreamKey) -> Option { - if !self.config.enabled() { - return None; - } - loop { - let mut candidate = { - let mut warm = self.warm.lock().unwrap(); - let bucket = warm.get_mut(key)?; - bucket.pop()? - }; - // Discard sockets past their warm lifetime (idle-billing guard). - if candidate.warmed_at.elapsed() > self.config.max_idle { - continue; // drops `candidate`, closing the socket - } - // Liveness: a non-blocking check that the socket hasn't already - // delivered a Close/Err. A warm socket should be silent after - // session.created, so anything pending means it is unhealthy. - if is_dead(&mut candidate.rx) { - continue; - } - return Some(WarmHandoff { - tx: candidate.tx, - rx: candidate.rx, - session_created: candidate.session_created, - }); - } - } - - /// One replenish pass over every registered key: reap stale sockets, then - /// dial up to `target_size`. Dials run concurrently; failures are swallowed - /// (a key that can't be warmed just keeps fresh-dialing on the request path) - /// and put the key into exponential backoff so a broken key isn't re-dialed - /// on every tick. - async fn replenish_all(&self) { - let keys: Vec = { self.warm.lock().unwrap().keys().cloned().collect() }; - for key in keys { - self.reap_stale(&key); - // Skip keys still in backoff from a prior all-failed pass — this is - // what bounds dials against an invalid/unreachable key to once per - // `BACKOFF_MAX` instead of `needed` dials every 250 ms tick. - if self.in_backoff(&key) { - continue; - } - let needed = { - let warm = self.warm.lock().unwrap(); - let have = warm.get(&key).map(Vec::len).unwrap_or(0); - self.config.target_size.saturating_sub(have) - }; - if needed == 0 { - continue; - } - // Dial the missing sockets CONCURRENTLY. A sequential loop here makes - // a full refill cost `needed × handshake` (~needed × 350 ms), which - // can't keep up with a high connect rate — the pool drains faster - // than it refills and most connects miss. Firing the dials together - // refills in ~one handshake window, keeping warm supply ≈ peak - // concurrent connects so the sub-ms warm handoff becomes the median, - // not the lucky-hit tail. - let dials = (0..needed).map(|_| warm_one(&key)); - let results = futures_util::future::join_all(dials).await; - let mut any_ok = false; - // `.flatten()` keeps only the successful dials; a key that can't be - // warmed just keeps fresh-dialing on the request path. - for conn in results.into_iter().flatten() { - any_ok = true; - self.warm - .lock() - .unwrap() - .entry(key.clone()) - .or_default() - .push(conn); - } - // Reset backoff on any success; otherwise grow it. We only ever enter - // backoff when a pass that *attempted* dials produced none — a `needed - // == 0` pass is handled by the `continue` above and never touches it. - self.record_replenish_outcome(&key, any_ok); - } - } - - /// Whether `key` is currently in a backoff window (a prior pass failed and - /// the retry time hasn't arrived). Eligible keys are pruned from the backoff - /// map so it doesn't grow unbounded for healthy keys. - fn in_backoff(&self, key: &UpstreamKey) -> bool { - let mut backoff = self.backoff.lock().unwrap(); - match backoff.get(key).and_then(|b| b.retry_after) { - Some(retry_after) if Instant::now() < retry_after => true, - Some(_) => { - // Window elapsed — allow the attempt. Keep the failure count so a - // still-broken key backs off further, but clear the gate so this - // tick proceeds. - if let Some(b) = backoff.get_mut(key) { - b.retry_after = None; - } - false - } - None => false, - } - } - - /// Update a key's backoff after a replenish attempt. Success clears it; - /// failure grows the retry delay exponentially up to `BACKOFF_MAX`. - fn record_replenish_outcome(&self, key: &UpstreamKey, any_ok: bool) { - let mut backoff = self.backoff.lock().unwrap(); - if any_ok { - backoff.remove(key); - return; - } - let entry = backoff.entry(key.clone()).or_default(); - entry.consecutive_failures = entry.consecutive_failures.saturating_add(1); - // Exponential: BASE * 2^(failures-1), saturating at MAX. `min` of the - // shift exponent keeps the doubling from overflowing. - let shift = (entry.consecutive_failures - 1).min(16); - let delay = BACKOFF_BASE.saturating_mul(1u32 << shift).min(BACKOFF_MAX); - entry.retry_after = Some(Instant::now() + delay); - } - - /// Drop sockets past `max_idle` or already dead for a key. - fn reap_stale(&self, key: &UpstreamKey) { - let mut warm = self.warm.lock().unwrap(); - if let Some(bucket) = warm.get_mut(key) { - bucket.retain_mut(|conn| { - conn.warmed_at.elapsed() <= self.config.max_idle && !is_dead(&mut conn.rx) - }); - } - } - - /// Test/inspection: number of warm sockets currently held for `key`. - #[cfg(test)] - pub fn warm_len(&self, key: &UpstreamKey) -> usize { - self.warm - .lock() - .unwrap() - .get(key) - .map(Vec::len) - .unwrap_or(0) - } - - /// Test/inspection: consecutive replenish failures recorded for `key` (0 if - /// the key is healthy / has no backoff entry). - #[cfg(test)] - pub fn backoff_failures(&self, key: &UpstreamKey) -> u32 { - self.backoff - .lock() - .unwrap() - .get(key) - .map(|b| b.consecutive_failures) - .unwrap_or(0) - } - - /// Test helper: synchronously warm `target_size` sockets for `key` (no - /// background task). Lets tests assert handoff behavior deterministically. - #[cfg(test)] - pub async fn warm_now(&self, key: &UpstreamKey) { - let needed = { - let warm = self.warm.lock().unwrap(); - let have = warm.get(key).map(Vec::len).unwrap_or(0); - self.config.target_size.saturating_sub(have) - }; - for _ in 0..needed { - if let Ok(conn) = warm_one(key).await { - self.warm - .lock() - .unwrap() - .entry(key.clone()) - .or_default() - .push(conn); - } - } - } - - /// Test helper: insert an already-built warm connection (used to inject a - /// dead socket and assert it is discarded at handoff). - #[cfg(test)] - fn insert_warm(&self, key: UpstreamKey, conn: WarmConnection) { - self.warm.lock().unwrap().entry(key).or_default().push(conn); - } -} - -/// Dial one upstream and pre-read its `session.created` into a [`WarmConnection`]. -/// -/// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends -/// unprompted is `session.created`; we buffer exactly that and read nothing more. -async fn warm_one(key: &UpstreamKey) -> Result { - let upstream: UpstreamWs = - dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; - let (tx, mut rx) = upstream.split(); - let session_created = read_event(&mut rx).await?; - Ok(WarmConnection { - tx, - rx, - session_created, - warmed_at: Instant::now(), - }) -} - -/// Resolve a deployment's API key into the pool key, returning `None` when no key -/// can be resolved (those deployments simply aren't pooled — the request path -/// still fresh-dials and surfaces the auth error there). -pub fn upstream_key( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, -) -> Option { - let api_key = resolve_api_key(api_key).ok()?; - Some(UpstreamKey { - model: model.to_string(), - api_key, - api_base: api_base.map(str::to_string), - }) -} - -/// Non-blocking liveness check: poll the upstream once. A warm socket is silent -/// after `session.created`, so a pending `Close`/`Err`/`None` means it is dead. -/// A pending data frame (shouldn't happen pre-handoff) is also treated as -/// unhealthy — we'd rather discard and fresh-dial than hand over a socket in an -/// unexpected state. `Pending` (the healthy case) returns `false`. -fn is_dead(rx: &mut UpstreamRx) -> bool { - use futures_util::Stream; - use futures_util::task::noop_waker_ref; - use std::pin::Pin; - use std::task::{Context, Poll}; - - let mut cx = Context::from_waker(noop_waker_ref()); - match Pin::new(rx).poll_next(&mut cx) { - Poll::Pending => false, - Poll::Ready(None) => true, - Poll::Ready(Some(Err(_))) => true, - // Any frame arriving before handoff is unexpected for a silent warm - // socket; treat it as unhealthy. - Poll::Ready(Some(Ok(_))) => true, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use futures_util::SinkExt; - use std::net::SocketAddr; - use tokio::net::TcpListener; - use tokio_tungstenite::tungstenite::Message; - - /// An in-process fake OpenAI realtime WS server. On connect it sends - /// `session.created`; on `response.create` it sends `response.created` + - /// `response.output_audio.delta` + `response.done`. Returns its `ws://` base. - async fn spawn_fake_openai() -> String { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr: SocketAddr = listener.local_addr().unwrap(); - tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(handle_fake_conn(stream)); - } - }); - format!("ws://{addr}") - } - - async fn handle_fake_conn(stream: tokio::net::TcpStream) { - let mut ws = match tokio_tungstenite::accept_async(stream).await { - Ok(ws) => ws, - Err(_) => return, - }; - // Unprompted session.created, exactly like OpenAI. - let _ = ws - .send(Message::Text( - r#"{"type":"session.created","session":{"id":"sess_fake"}}"#.to_string(), - )) - .await; - while let Some(Ok(msg)) = ws.next().await { - if let Message::Text(text) = msg - && text.contains("response.create") - { - for frame in [ - r#"{"type":"response.created"}"#, - r#"{"type":"response.output_audio.delta","delta":"AAAA"}"#, - r#"{"type":"response.done"}"#, - ] { - let _ = ws.send(Message::Text(frame.to_string())).await; - } - } - } - } - - fn test_config() -> PoolConfig { - PoolConfig { - target_size: 2, - max_idle: Duration::from_secs(30), - } - } - - fn key_for(base: &str) -> UpstreamKey { - UpstreamKey { - model: "gpt-realtime".to_string(), - api_key: "sk-test".to_string(), - api_base: Some(base.to_string()), - } - } - - #[tokio::test] - async fn warm_handoff_relays_buffered_session_created() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - pool.warm_now(&key).await; - assert_eq!(pool.warm_len(&key), 2); - - let handoff = pool.take(&key).expect("a warm socket should be available"); - assert_eq!(handoff.session_created.event_type, "session.created"); - assert_eq!( - handoff - .session_created - .data - .get("session") - .and_then(|s| s.get("id")) - .and_then(|v| v.as_str()), - Some("sess_fake") - ); - // Taking one leaves one. - assert_eq!(pool.warm_len(&key), 1); - } - - #[tokio::test] - async fn pool_miss_returns_none_for_fresh_dial_fallback() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - // Registered but never warmed → empty bucket → miss. - pool.register(key.clone()); - assert!(pool.take(&key).is_none()); - - // Unknown key → miss. - let other = key_for("ws://127.0.0.1:1"); - assert!(pool.take(&other).is_none()); - } - - #[tokio::test] - async fn disabled_pool_never_hands_off() { - let pool = RealtimePool::disabled(); - let key = key_for("ws://127.0.0.1:1"); - pool.register(key.clone()); - assert_eq!(pool.warm_len(&key), 0); - assert!(pool.take(&key).is_none()); - } - - #[tokio::test] - async fn dead_warm_socket_is_discarded() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - // Build one real warm connection, then kill the upstream by dropping the - // server side: easiest is to dial, read session.created, then close our - // own rx's peer. Instead we forge "dead" via an already-closed socket: - // dial a connection and immediately send a Close from the client side so - // the server closes back, then warm it. Simpler: warm normally, then - // mark it stale by backdating warmed_at past max_idle and confirm it's - // dropped — that exercises the same discard path. - let mut conn = warm_one(&key).await.expect("warm one"); - conn.warmed_at = Instant::now() - Duration::from_secs(3600); // past max_idle - pool.insert_warm(key.clone(), conn); - assert_eq!(pool.warm_len(&key), 1); - - // take() must discard the stale socket and report a miss. - assert!(pool.take(&key).is_none()); - assert_eq!(pool.warm_len(&key), 0); - } - - #[tokio::test] - async fn background_replenisher_tops_up_registered_key() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::spawn(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - // Wait (bounded) for the background task to reach the target size. - let mut warmed = 0; - for _ in 0..40 { - tokio::time::sleep(Duration::from_millis(50)).await; - warmed = pool.warm_len(&key); - if warmed >= test_config().target_size { - break; - } - } - assert_eq!( - warmed, - test_config().target_size, - "background replenisher should warm up to target_size" - ); - let handoff = pool.take(&key).expect("a warm socket should be available"); - assert_eq!(handoff.session_created.event_type, "session.created"); - } - - #[tokio::test] - async fn closed_upstream_socket_is_detected_dead() { - // A genuinely dead socket: dial the fake, read session.created, then drop - // the server by closing from our side and waiting for the close to land. - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - let mut conn = warm_one(&key).await.expect("warm one"); - // Close the upstream from the client side; the server echoes a close. - let _ = conn.tx.send(Message::Close(None)).await; - // Give the close a moment to arrive on rx. - tokio::time::sleep(Duration::from_millis(50)).await; - pool.insert_warm(key.clone(), conn); - - // Liveness check at take() should detect the close and discard it. - assert!(pool.take(&key).is_none()); - assert_eq!(pool.warm_len(&key), 0); - } - - #[tokio::test] - async fn broken_key_backs_off_instead_of_dialing_every_tick() { - // A key whose upstream is unreachable: every warm-up dial fails. - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for("ws://127.0.0.1:1"); // nothing listens here - pool.register(key.clone()); - - // First pass attempts dials, they all fail → key enters backoff, no warm - // sockets, one recorded failure. - pool.replenish_all().await; - assert_eq!(pool.warm_len(&key), 0); - assert_eq!(pool.backoff_failures(&key), 1); - assert!( - pool.in_backoff(&key), - "a key whose dials all failed must be in backoff" - ); - - // An immediate next pass must be SKIPPED (still in the backoff window), so - // it does NOT fire another round of dials — the failure count is unchanged. - pool.replenish_all().await; - assert_eq!( - pool.backoff_failures(&key), - 1, - "replenish during the backoff window must not re-dial the broken key" - ); - } - - #[tokio::test] - async fn healthy_key_never_enters_backoff_and_clears_after_recovery() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - // A reachable upstream: the pass succeeds, so the key is never backed off. - pool.replenish_all().await; - assert_eq!(pool.warm_len(&key), test_config().target_size); - assert_eq!(pool.backoff_failures(&key), 0); - assert!(!pool.in_backoff(&key)); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs deleted file mode 100644 index f86dd778424..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ /dev/null @@ -1,485 +0,0 @@ -use std::time::Duration; - -use futures_util::stream::{SplitSink, SplitStream}; -use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::AuthError; -use litellm_core::Error; -use litellm_core::auth::error::MissingCredential; -use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; -use litellm_core::responses::types::ResponsesWsEvent; -use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::tungstenite::http::HeaderValue; -use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; - -use litellm_core::responses::websocket::{ResponsesUpstreamWs, connect_upstream}; - -use crate::constants::{ - DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS, DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS, -}; - -const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; -type UpstreamTx = SplitSink; -type UpstreamRx = SplitStream; - -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { - api_key - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .or_else(|| { - std::env::var(OPENAI_API_KEY_ENV) - .ok() - .filter(|value| !value.trim().is_empty()) - }) - .ok_or_else(|| Error::from(AuthError::from(MissingCredential::OpenAiResponsesApiKey))) -} - -async fn dial_upstream( - model: &str, - api_key: &str, - api_base: Option<&str>, -) -> Result { - let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); - let mut request = url - .as_str() - .into_client_request() - .map_err(|error| Error::Network(error.to_string()))?; - request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|error| Error::Auth(error.to_string()))?, - ); - let result = tokio::time::timeout( - Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS), - connect_upstream(request), - ) - .await - .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?; - result - .map(|(socket, _)| socket) - .map_err(|error| match *error { - tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { - status: response.status().as_u16(), - body: String::new(), - }, - other => Error::Network(other.to_string()), - }) -} - -pub struct ResponsesWebSocketStreaming; - -impl ResponsesWebSocketStreaming { - pub async fn bidirectional_forward( - model: &str, - upstream_tx: UpstreamTx, - upstream_rx: UpstreamRx, - idle_timeout: Option, - observe: impl FnMut(&ResponsesWsEvent) + Send, - client_in: In, - client_out: Out, - ) -> Result<(), Error> - where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, - { - splice( - model, - upstream_tx, - upstream_rx, - idle_timeout, - observe, - client_in, - client_out, - ) - .await - } -} - -pub(crate) async fn splice( - model: &str, - mut upstream_tx: UpstreamTx, - mut upstream_rx: UpstreamRx, - idle_timeout: Option, - mut observe: impl FnMut(&ResponsesWsEvent) + Send, - mut client_in: In, - mut client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - let idle = - idle_timeout.unwrap_or_else(|| Duration::from_secs(DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS)); - loop { - tokio::select! { - event = client_in.next() => { - let Some(event) = event else { break }; - for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&event, model)? - .events - { - let payload = serde_json::to_string(&outbound) - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - upstream_tx.send(Message::Text(payload)) - .await - .map_err(|error| Error::Network(error.to_string()))?; - } - } - message = upstream_rx.next() => { - let Some(message) = message else { break }; - match message.map_err(|error| Error::Network(error.to_string()))? { - Message::Text(text) => { - let event = serde_json::from_str::(&text) - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - observe(&event); - for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_response(&event, model)? - .events - { - client_out.send(outbound) - .await - .map_err(|error| Error::Network(error.to_string()))?; - } - } - Message::Close(_) => break, - _ => {} - } - } - _ = tokio::time::sleep(idle) => break, - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn async_responses_websocket( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, - first_frame: Option, - idle_timeout: Option, - mut observe: impl FnMut(&ResponsesWsEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - let key = resolve_api_key(api_key)?; - let upstream = dial_upstream(model, &key, api_base).await?; - let (mut upstream_tx, upstream_rx) = upstream.split(); - if let Some(first_frame) = first_frame { - for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&first_frame, model)? - .events - { - let payload = serde_json::to_string(&outbound) - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - upstream_tx - .send(Message::Text(payload)) - .await - .map_err(|error| Error::Network(error.to_string()))?; - } - } - ResponsesWebSocketStreaming::bidirectional_forward( - model, - upstream_tx, - upstream_rx, - idle_timeout, - &mut observe, - client_in, - client_out, - ) - .await -} - -#[allow(clippy::too_many_arguments)] -pub async fn responses_ws( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, - first_frame: Option, - idle_timeout: Option, - observe: impl FnMut(&ResponsesWsEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - async_responses_websocket( - model, - api_key, - api_base, - first_frame, - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} - -#[cfg(test)] -mod tests { - use super::*; - use futures_channel::mpsc; - use futures_util::{SinkExt, StreamExt}; - use litellm_core::responses::types::ResponsesWsEventType; - use serde_json::json; - use tokio::io::AsyncWriteExt; - use tokio::net::TcpListener; - use tokio_tungstenite::accept_async; - - /// The Responses dial has to reach a `wss://` upstream without a process-wide - /// crypto provider installed, which is what dialing through `io::tls` buys. - #[tokio::test] - async fn dial_upstream_over_wss_reports_an_error_instead_of_panicking() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind a loopback port"); - let port = listener - .local_addr() - .expect("read the bound address") - .port(); - tokio::spawn(async move { - while let Ok((stream, _peer)) = listener.accept().await { - drop(stream); - } - }); - - let result = - dial_upstream("gpt-5", "sk-test", Some(&format!("wss://127.0.0.1:{port}"))).await; - - assert!(matches!(result, Err(Error::Network(_)))); - } - - async fn websocket_base() -> (String, tokio::task::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("local address"); - let task = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let mut socket = accept_async(stream).await.expect("websocket handshake"); - while let Some(Ok(Message::Text(text))) = socket.next().await { - let request: serde_json::Value = serde_json::from_str(&text).expect("request json"); - let model = request - .get("model") - .and_then(serde_json::Value::as_str) - .or_else(|| { - request - .get("response") - .and_then(serde_json::Value::as_object) - .and_then(|response| { - response.get("model").and_then(serde_json::Value::as_str) - }) - }) - .expect("enforced model"); - socket - .send(Message::Text( - json!({ - "type": "response.created", - "response": { - "id": format!("resp-{model}"), - "model": model, - "extra": "preserved" - } - }) - .to_string(), - )) - .await - .expect("created event"); - socket - .send(Message::Text( - json!({ - "type": "response.completed", - "response": { - "id": format!("resp-{model}"), - "model": model, - "usage": { - "input_tokens": 1, - "output_tokens": 2, - "total_tokens": 3 - } - } - }) - .to_string(), - )) - .await - .expect("completed event"); - } - }); - (format!("http://{address}"), task) - } - - fn event(value: serde_json::Value) -> ResponsesWsEvent { - serde_json::from_value(value).expect("event") - } - - #[test] - fn explicit_nonblank_key_wins() { - assert_eq!( - resolve_api_key(Some(" explicit ")).expect("key"), - "explicit" - ); - } - - #[test] - fn blank_key_is_not_accepted_without_environment_key() { - if std::env::var(OPENAI_API_KEY_ENV).is_err() { - assert!(resolve_api_key(Some(" ")).is_err()); - } - } - - #[tokio::test] - async fn forwards_events_sequentially_and_enforces_model() { - let (api_base, server) = websocket_base().await; - let (client_tx, client_rx) = mpsc::unbounded(); - let (output_tx, mut output_rx) = mpsc::unbounded(); - let (observed_tx, observed_rx) = mpsc::unbounded(); - client_tx - .unbounded_send(event(json!({ - "type": "response.create", - "model": "wrong" - }))) - .expect("first request"); - client_tx - .unbounded_send(event(json!({ - "type": "response.create", - "response": {"model": "also-wrong"} - }))) - .expect("second request"); - - let task = tokio::spawn(async move { - responses_ws( - "authorized-model", - Some("test-key"), - Some(&api_base), - None, - Some(Duration::from_secs(1)), - move |event| { - observed_tx - .unbounded_send(event.clone()) - .expect("observe event"); - }, - client_rx, - output_tx, - ) - .await - }); - - let first = output_rx.next().await.expect("first output"); - let second = output_rx.next().await.expect("second output"); - let third = output_rx.next().await.expect("third output"); - let fourth = output_rx.next().await.expect("fourth output"); - drop(client_tx); - task.await.expect("splice task").expect("successful splice"); - server.await.expect("server task"); - - assert_eq!(first.event_type, ResponsesWsEventType::ResponseCreated); - assert_eq!(first.model(), Some("authorized-model")); - assert_eq!(first.data["response"]["extra"], "preserved"); - assert_eq!(second.event_type, ResponsesWsEventType::ResponseCompleted); - assert_eq!(third.event_type, ResponsesWsEventType::ResponseCreated); - assert_eq!(fourth.event_type, ResponsesWsEventType::ResponseCompleted); - let observed: Vec<_> = observed_rx.collect().await; - assert_eq!(observed.len(), 4); - assert!( - observed - .iter() - .all(|event| event.event_type != ResponsesWsEventType::ResponseCreate) - ); - } - - #[tokio::test] - async fn idle_timeout_ends_without_upstream_events() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let _socket = accept_async(stream).await.expect("handshake"); - tokio::time::sleep(Duration::from_secs(1)).await; - }); - let (_client_tx, client_rx) = mpsc::unbounded::(); - let (output_tx, mut output_rx) = mpsc::unbounded(); - let result = responses_ws( - "model", - Some("key"), - Some(&format!("http://{address}")), - None, - Some(Duration::from_millis(20)), - |_| {}, - client_rx, - output_tx, - ) - .await; - assert!(result.is_ok()); - assert!(output_rx.next().await.is_none()); - server.abort(); - } - - #[tokio::test] - async fn dial_http_status_is_preserved() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("accept"); - stream - .write_all(b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n") - .await - .expect("response"); - }); - let (_client_tx, client_rx) = mpsc::unbounded::(); - let (output_tx, _output_rx) = mpsc::unbounded(); - let error = responses_ws( - "model", - Some("key"), - Some(&format!("http://{address}")), - None, - Some(Duration::from_millis(20)), - |_| {}, - client_rx, - output_tx, - ) - .await - .expect_err("status error"); - assert!(matches!(error, Error::Http { status: 401, .. })); - server.await.expect("server task"); - } - - #[tokio::test] - async fn dial_http_500_status_is_preserved() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("accept"); - stream - .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n") - .await - .expect("response"); - }); - let (_client_tx, client_rx) = mpsc::unbounded::(); - let (output_tx, _output_rx) = mpsc::unbounded(); - let error = responses_ws( - "model", - Some("key"), - Some(&format!("http://{address}")), - None, - Some(Duration::from_millis(20)), - |_| {}, - client_rx, - output_tx, - ) - .await - .expect_err("status error"); - assert!(matches!(error, Error::Http { status: 500, .. })); - server.await.expect("server task"); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/io/tls.rs b/litellm-rust/crates/ai-gateway/src/io/tls.rs deleted file mode 100644 index a2562f60345..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/tls.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Outbound WebSocket dials over a TLS config this crate builds once and owns. -//! -//! `reqwest/rustls-tls` enables `rustls/ring` and `litellm-core`'s `bedrock-auth` -//! enables `rustls/aws-lc-rs`, so the bare `ClientConfig::builder()` that -//! `tokio-tungstenite` uses when handed no connector panics rather than guess -//! between them. Naming ring on a connector of our own settles that for these -//! dials without touching the process-wide default, and building the config -//! once keeps the platform trust store, which `tokio-tungstenite` would -//! otherwise re-read on every dial, off the dial path. - -use std::io; -use std::sync::{Arc, OnceLock}; - -use rustls::{ClientConfig, RootCertStore}; -use tokio::net::TcpStream; -use tokio_tungstenite::tungstenite::Error; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::tungstenite::error::TlsError; -use tokio_tungstenite::tungstenite::handshake::client::Response; -use tokio_tungstenite::{ - Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, -}; - -static TLS_CONFIG: OnceLock> = OnceLock::new(); - -fn build_config() -> Result> { - let native = rustls_native_certs::load_native_certs(); - let roots = { - let mut store = RootCertStore::empty(); - let (added, _ignored) = store.add_parsable_certificates(native.certs); - if added == 0 { - return Err(Box::new(Error::Io(io::Error::other(format!( - "no usable native root certificates: {:?}", - native.errors - ))))); - } - store - }; - - ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) - .with_safe_default_protocol_versions() - .map(|builder| builder.with_root_certificates(roots).with_no_client_auth()) - .map_err(|error| Box::new(Error::Tls(TlsError::Rustls(error)))) -} - -fn tls_config() -> Result, Box> { - if let Some(config) = TLS_CONFIG.get() { - return Ok(Arc::clone(config)); - } - let built = Arc::new(build_config()?); - Ok(Arc::clone(TLS_CONFIG.get_or_init(|| built))) -} - -pub(crate) async fn connect_upstream( - request: R, -) -> Result<(WebSocketStream>, Response), Box> -where - R: IntoClientRequest + Unpin, -{ - let request = request.into_client_request().map_err(Box::new)?; - let connector = match request.uri().scheme_str() { - Some("wss") => Some(Connector::Rustls(tls_config()?)), - _ => None, - }; - connect_async_tls_with_config(request, None, false, connector) - .await - .map_err(Box::new) -} - -#[cfg(test)] -mod tests { - use super::build_config; - - #[test] - fn builds_a_usable_config_with_both_provider_features_enabled() { - let config = build_config().expect("a client config"); - - assert!(!config.crypto_provider().cipher_suites.is_empty()); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/lib.rs b/litellm-rust/crates/ai-gateway/src/lib.rs deleted file mode 100644 index 08fbde564ed..00000000000 --- a/litellm-rust/crates/ai-gateway/src/lib.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! LiteLLM AI Gateway library. -//! -//! Two layers, split by feature so the Python `cdylib` can depend on the I/O -//! without pulling in the HTTP server: -//! -//! - Call-type modules such as [`ocr`]: provider transforms, lifecycle hooks, -//! and provider I/O. Always available — no feature required. These predate the -//! rule that a route's entrypoint and handler live in `litellm-core` (see -//! `litellm_core::messages`) and move there as they are touched. -//! - [`io`]: compatibility exports and realtime WebSocket splice helpers. -//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling -//! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway` -//! binary turns on. - -pub mod audio_transcription; -mod client; -pub mod io; -pub mod ocr; - -#[cfg(feature = "server")] -pub mod auth; -#[cfg(feature = "server")] -pub mod routes; -#[cfg(feature = "server")] -pub mod state; -#[cfg(feature = "trace-parity")] -pub mod trace_parity; - -mod constants; -pub mod integrations; -#[cfg(feature = "server")] -mod realtime; diff --git a/litellm-rust/crates/ai-gateway/src/main.rs b/litellm-rust/crates/ai-gateway/src/main.rs deleted file mode 100644 index 88d7b1dbcf8..00000000000 --- a/litellm-rust/crates/ai-gateway/src/main.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! LiteLLM AI Gateway — a minimal Axum server fronting the Rust router. -//! -//! Flow: client → `POST /v1/realtime` → `router.realtime()` selects a deployment -//! (simple-shuffle) → `io::realtime::realtime()` invokes OpenAI. The -//! server owns transport + config; routing lives in the `router` crate. -//! -//! The binary requires the `server` feature (declared in `Cargo.toml` via -//! `required-features`), so cargo skips it unless that feature is on. Everything -//! the binary needs lives in the library (`litellm_ai_gateway`); `main` just -//! wires startup. - -use std::sync::Arc; - -use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key}; -use litellm_ai_gateway::routes; -use litellm_ai_gateway::state::AppState; -#[cfg(feature = "python-config")] -use litellm_config::load_model_list; -use litellm_core::router::{Deployment, LiteLLMParams, Router}; - -use litellm_ai_gateway::integrations::custom_logger::CustomLogger; -use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger; - -/// Bind to localhost by default so the gateway is not a public, unauthenticated -/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`). -const DEFAULT_HOST: &str = "127.0.0.1"; -const DEFAULT_PORT: u16 = 4001; - -#[tokio::main] -async fn main() { - // Trim before storing so it matches the trimmed bearer token in `auth` - // (avoids a silent auth failure when the env var has surrounding whitespace). - let master_key: Option> = std::env::var("LITELLM_MASTER_KEY") - .ok() - .map(|key| key.trim().to_string()) - .filter(|key| !key.is_empty()) - .map(Arc::from); - if master_key.is_none() { - eprintln!( - "warning: LITELLM_MASTER_KEY is not set; /v1/realtime will reject all requests (fail closed)" - ); - } - - // Spawn the realtime-logging worker (drains a channel → POSTs batches to the - // Python proxy's /v1/callbacks/logs). Built here so the spawn lands on the - // tokio runtime. `from_env` reads LITELLM_PROXY_BASE_URL + LITELLM_MASTER_KEY. - let proxy_logger = LiteLLMPythonProxyAPILogger::from_env(); - let loggers: Vec> = vec![proxy_logger]; - - let router = Arc::new(build_router()); - - // Build the pre-warmed realtime pool and register each deployment's upstream - // so the background replenisher starts warming it. `REALTIME_POOL_SIZE=0` - // yields a disabled pool → every connect fresh-dials (original behavior). - let pool_config = PoolConfig::from_env(); - let realtime_pool = RealtimePool::spawn(pool_config); - if pool_config.enabled() { - register_deployments(&router, &realtime_pool); - eprintln!( - "realtime connection pool enabled: target {} warm sockets/key, max idle {}s", - pool_config.target_size, - pool_config.max_idle.as_secs() - ); - } else { - eprintln!( - "realtime connection pool disabled (REALTIME_POOL_SIZE=0); fresh-dialing each connect" - ); - } - - let state = AppState { - router, - master_key, - loggers: Arc::new(loggers), - realtime_pool, - }; - - let host = std::env::var("HOST").unwrap_or_else(|_| DEFAULT_HOST.to_string()); - let port = resolve_port(); - - let listener = tokio::net::TcpListener::bind((host.as_str(), port)) - .await - .expect("failed to bind listener"); - eprintln!("litellm-ai-gateway listening on {host}:{port}"); - axum::serve(listener, routes::app(state)) - .await - .expect("server error"); -} - -/// Register every deployment's upstream key with the pool so the replenisher -/// pre-warms it. Mirrors `service::run`'s key derivation (strip `openai/`, resolve -/// api_key); deployments whose key can't be resolved are skipped (they fresh-dial -/// and surface the auth error on the request path, as before). -fn register_deployments(router: &Router, pool: &RealtimePool) { - for deployment in router.deployments() { - let params = &deployment.litellm_params; - let provider_model = params - .model - .strip_prefix("openai/") - .unwrap_or(¶ms.model); - if let Some(key) = upstream_key( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - ) { - pool.register(key); - } - } -} - -/// Resolve `PORT`, warning (rather than silently defaulting) on an invalid value. -fn resolve_port() -> u16 { - match std::env::var("PORT") { - Ok(raw) => raw.parse().unwrap_or_else(|_| { - eprintln!("warning: PORT={raw:?} is not a valid port; using {DEFAULT_PORT}"); - DEFAULT_PORT - }), - Err(_) => DEFAULT_PORT, - } -} - -/// Build the router. With the `python-config` feature and `LITELLM_CONFIG_PATH` -/// set, load the resolved `model_list` from the proxy config via the embedded -/// Python reader (load time only). Otherwise fall back to the env stand-in. -fn build_router() -> Router { - #[cfg(feature = "python-config")] - if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") { - match load_model_list(std::path::Path::new(&config_path)) { - Ok(deployments) => { - eprintln!("loaded model_list from {config_path} via python config reader"); - return Router::new(deployments); - } - Err(err) => { - eprintln!("config load failed ({err}); falling back to env deployment"); - } - } - } - build_router_from_env() -} - -/// Build a minimal single-deployment `model_list` from the environment. -/// -/// A real deployment loads `model_list` from config; this is the minimal stand-in -/// so the gateway has one OpenAI deployment to route to. -fn build_router_from_env() -> Router { - let model = - std::env::var("OPENAI_REALTIME_MODEL").unwrap_or_else(|_| "gpt-realtime".to_string()); - let api_key = std::env::var("OPENAI_API_KEY").ok(); - if api_key.is_none() { - eprintln!( - "warning: OPENAI_API_KEY is not set; realtime requests will fail with auth errors" - ); - } - let deployment = Deployment { - model_name: model.clone(), - litellm_params: LiteLLMParams { - model, - api_key, - api_base: None, - }, - }; - Router::new(vec![deployment]) -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs deleted file mode 100644 index fb63a02f7ad..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -use litellm_core::Error; -use litellm_core::ocr::{ - OcrClient, - wire::{OcrWireRequest, decode_request}, -}; -use serde_json::Value; - -mod types; - -pub use types::OcrRequest; - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub async fn ocr(request: OcrRequest<'_>) -> Result { - core_ocr(request).await -} - -async fn core_ocr(request: OcrRequest<'_>) -> Result { - validate_host_hooks(&request)?; - let client = OcrClient::new(crate::client::http_client().clone())?; - let core_request = decode_request(OcrWireRequest { - model: request.model.to_string(), - document: request.document, - api_key: request.api_key.map(str::to_string), - api_base: request.api_base.map(str::to_string), - custom_llm_provider: request.custom_llm_provider.map(str::to_string), - extra_headers: request.extra_headers, - optional_params: request.optional_params, - input_sources: Default::default(), - timeout_seconds: request.timeout.map(|timeout| timeout.as_secs_f64()), - })?; - client - .perform(core_request) - .await - .map(|response| response.into_json()) -} - -fn validate_host_hooks(request: &OcrRequest<'_>) -> Result<(), Error> { - if !request.guardrails.is_empty() { - return Err(Error::Unsupported( - "OCR host guardrails are not wired to the core path", - )); - } - if !request.callbacks.is_empty() { - return Err(Error::Unsupported( - "OCR host callbacks are not wired to the core path", - )); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use litellm_core::ocr::wire::is_supported_request; - use serde_json::{Map, json}; - - use super::{OcrRequest, validate_host_hooks}; - use crate::integrations::custom_guardrail::{CustomGuardrail, GuardrailEventHook}; - use crate::integrations::custom_logger::CustomLogger; - - struct TestGuardrail; - - impl CustomGuardrail for TestGuardrail { - fn guardrail_name(&self) -> &str { - "test" - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &[] - } - } - - struct TestLogger; - - impl CustomLogger for TestLogger {} - - fn request() -> OcrRequest<'static> { - OcrRequest { - model: "model", - document: json!({"type":"image_url","image_url":"data:image/png;base64,YQ=="}), - api_key: None, - api_base: None, - custom_llm_provider: Some("mistral"), - extra_headers: None, - optional_params: Map::new(), - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - } - } - - #[test] - fn core_activation_includes_migrated_providers() { - assert!(is_supported_request("model", Some("mistral"))); - assert!(is_supported_request("pixtral-12b", Some("azure_ai"))); - assert!(is_supported_request( - "doc-intelligence/prebuilt-layout", - Some("azure_ai") - )); - assert!(is_supported_request("parse-v3", Some("reducto"))); - assert!(is_supported_request("mistral-ocr", Some("vertex_ai"))); - assert!(is_supported_request("deepseek-ocr", Some("vertex_ai"))); - } - - #[test] - fn core_path_rejects_unwired_guardrails() { - let request = OcrRequest { - guardrails: vec![Arc::new(TestGuardrail)], - ..request() - }; - let error = validate_host_hooks(&request).unwrap_err(); - assert!(error.to_string().contains("guardrails are not wired")); - } - - #[test] - fn core_path_rejects_unwired_callbacks() { - let request = OcrRequest { - callbacks: vec![Arc::new(TestLogger)], - ..request() - }; - let error = validate_host_hooks(&request).unwrap_err(); - assert!(error.to_string().contains("callbacks are not wired")); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/types.rs b/litellm-rust/crates/ai-gateway/src/ocr/types.rs deleted file mode 100644 index e96d2df1adb..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/types.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use serde_json::{Map, Value}; - -use crate::integrations::custom_guardrail::CustomGuardrail; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; - -pub struct OcrRequest<'a> { - pub model: &'a str, - pub document: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub optional_params: Map, - pub timeout: Option, - pub callbacks: Vec>, - pub guardrails: Vec>, - pub request_metadata: RequestMetadata, - pub litellm_call_id: Option<&'a str>, -} diff --git a/litellm-rust/crates/ai-gateway/src/realtime/mod.rs b/litellm-rust/crates/ai-gateway/src/realtime/mod.rs deleted file mode 100644 index 82be596ba86..00000000000 --- a/litellm-rust/crates/ai-gateway/src/realtime/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Realtime logging collector. Observes the realtime event stream and emits a -//! `StandardLoggingPayload` to the registered callbacks on session close. - -pub mod streaming; diff --git a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs deleted file mode 100644 index c0d72e90b77..00000000000 --- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs +++ /dev/null @@ -1,414 +0,0 @@ -//! `RealTimeStreaming` — the realtime logging collector. -//! -//! Mirrors Python `litellm.realtime_api.main.RealTimeStreaming`: it observes the -//! event stream in O(1) (never buffering frames), accumulating just the fields -//! the spend log needs (model, id, cumulative usage), then on session close -//! builds a `StandardLoggingPayload` and fans it out to every registered -//! `CustomLogger`. - -use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; - -use litellm_core::realtime::types::RealtimeEvent; -use serde_json::Value; - -use crate::constants::DEFAULT_PROVIDER; -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::{ - RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, Usage, -}; - -/// Current wall-clock time as epoch seconds (float), matching the Python -/// `startTime`/`endTime` contract. -fn epoch_seconds() -> f64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs_f64()) - .unwrap_or(0.0) -} - -/// Status of a finished realtime session, mapped to the callback record status. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum SessionStatus { - Success, - Failure, -} - -/// Accumulates realtime session state and emits a logging payload on close. -pub struct RealTimeStreaming { - callbacks: Vec>, - /// REQUEST-ID RULE: the SpendLogs `request_id` == the OpenAI realtime session - /// id (`sess_…`), captured from `session.created`. Both `id` and - /// `litellm_call_id` are set to that value so the Python writer logs the same - /// id regardless of which field it reads. The gateway-generated `rt-…` id - /// (the constructor seed) is only a fallback for sessions that fail before - /// `session.created` arrives. - litellm_call_id: String, - /// See the request-id rule above — mirrors `litellm_call_id`. - id: String, - model: String, - custom_llm_provider: String, - usage: Usage, - response_cost: f64, - start_time: f64, - end_time: f64, - metadata: RequestMetadata, - /// Count of logging callbacks that failed to enqueue (non-fatal). - dropped: u64, -} - -impl RealTimeStreaming { - /// Create a collector for one session. `litellm_call_id` is the gateway's - /// per-connection id; `model` is the requested model (a sane default until - /// `session.created` reports the upstream model). - pub fn new( - callbacks: Vec>, - litellm_call_id: String, - model: String, - metadata: RequestMetadata, - ) -> Self { - let now = epoch_seconds(); - Self { - callbacks, - id: litellm_call_id.clone(), - litellm_call_id, - model, - custom_llm_provider: DEFAULT_PROVIDER.to_string(), - usage: Usage::default(), - response_cost: 0.0, - start_time: now, - end_time: now, - metadata, - dropped: 0, - } - } - - /// Number of logging callbacks that failed to enqueue so far (test/observ.). - #[allow(dead_code)] - pub fn dropped(&self) -> u64 { - self.dropped - } - - /// Observe one realtime event. O(1): updates accumulated state only; never - /// buffers frames. Safe to call on every event in either direction. - pub fn observe(&mut self, event: &RealtimeEvent) { - match event.event_type.as_str() { - "session.created" | "session.updated" => self.on_session(event), - "response.done" => self.on_response_done(event), - _ => {} - } - } - - /// `session.created` / `session.updated` → capture upstream id + model. - /// Per the request-id rule, the OpenAI session id becomes BOTH `id` and - /// `litellm_call_id`, replacing the gateway-generated fallback. - fn on_session(&mut self, event: &RealtimeEvent) { - let session = event.data.get("session").and_then(Value::as_object); - if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) - && !id.is_empty() - { - self.id = id.to_string(); - self.litellm_call_id = id.to_string(); - } - if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) - && !model.is_empty() - { - self.model = model.to_string(); - } - } - - /// `response.done` → add this response's usage to the cumulative totals. - fn on_response_done(&mut self, event: &RealtimeEvent) { - let usage = event - .data - .get("response") - .and_then(Value::as_object) - .and_then(|r| r.get("usage")) - .and_then(Value::as_object); - let Some(usage) = usage else { return }; - - let input = usage.get("input_tokens").and_then(Value::as_u64); - let output = usage.get("output_tokens").and_then(Value::as_u64); - let total = usage.get("total_tokens").and_then(Value::as_u64); - - if let Some(input) = input { - self.usage.prompt_tokens += input; - } - if let Some(output) = output { - self.usage.completion_tokens += output; - } - // Prefer the upstream-reported total; otherwise derive it. - match total { - Some(total) => self.usage.total_tokens += total, - None => { - self.usage.total_tokens += input.unwrap_or(0) + output.unwrap_or(0); - } - } - } - - /// Set the per-session response cost ($). Cost computation is Python-side in - /// the proxy; the gateway forwards 0.0 by default and lets the proxy price. - /// Public API (exercised in tests) for the future path where the gateway - /// prices realtime sessions itself. - #[allow(dead_code)] - pub fn set_response_cost(&mut self, cost: f64) { - self.response_cost = cost; - } - - /// Build the `StandardLoggingPayload` from accumulated state. - pub fn build_payload(&self) -> StandardLoggingPayload { - StandardLoggingPayload { - id: self.id.clone(), - litellm_call_id: self.litellm_call_id.clone(), - call_type: "realtime".to_string(), - model: self.model.clone(), - custom_llm_provider: self.custom_llm_provider.clone(), - response_cost: self.response_cost, - prompt_tokens: self.usage.prompt_tokens, - completion_tokens: self.usage.completion_tokens, - total_tokens: self.usage.total_tokens, - start_time: self.start_time, - end_time: self.end_time, - stream: true, - metadata: StandardLoggingMetadata { - user_api_key_hash: self.metadata.user_api_key_hash.clone(), - user_api_key_user_id: self.metadata.user_api_key_user_id.clone(), - user_api_key_team_id: self.metadata.user_api_key_team_id.clone(), - ..Default::default() - }, - messages: None, - } - } - - /// Finish the session: stamp the end time and fan the payload out to every - /// callback. On a logger enqueue error we bump a non-fatal counter (the - /// realtime session has already ended; a dropped log must never propagate). - pub async fn log_messages(&mut self, status: SessionStatus) { - self.end_time = epoch_seconds(); - let payload = self.build_payload(); - let timing = CallbackTiming::new(payload.start_time, payload.end_time); - let runner = CustomLoggerRunner::new(self.callbacks.clone()); - - match status { - SessionStatus::Success => { - let response = CallbackValue::new("realtime", serde_json::Value::Null); - let report = runner - .async_log_success_event( - &ModelCallDetails::from_standard_logging_payload(payload), - &response, - timing, - ) - .await; - self.dropped += report.dropped as u64; - } - SessionStatus::Failure => { - let error = LoggingError { - message: "realtime session ended in failure".to_string(), - kind: "RealtimeSessionError".to_string(), - }; - let response = CallbackValue::new( - "error", - serde_json::json!({ - "message": error.message, - "kind": error.kind, - }), - ); - let report = runner - .async_log_failure_event( - &ModelCallDetails::from_standard_logging_payload(payload) - .with_failure_error(error), - Some(&response), - timing, - ) - .await; - self.dropped += report.dropped as u64; - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::integrations::custom_logger::LogError; - use crate::integrations::custom_logger::LogFuture; - use std::sync::atomic::{AtomicU64, Ordering}; - - fn event(raw: &str) -> RealtimeEvent { - serde_json::from_str(raw).expect("valid event json") - } - - /// A test logger that records the last payload it saw. - #[derive(Default)] - struct CapturingLogger { - calls: AtomicU64, - last_model: std::sync::Mutex>, - last_total_tokens: AtomicU64, - } - - impl CustomLogger for CapturingLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - let payload = model_call_details - .standard_logging_payload - .as_ref() - .expect("standard logging payload"); - self.calls.fetch_add(1, Ordering::SeqCst); - *self.last_model.lock().unwrap() = Some(payload.model.clone()); - self.last_total_tokens - .store(payload.total_tokens, Ordering::SeqCst); - Ok(()) - }) - } - } - - #[tokio::test] - async fn observe_accumulates_model_and_tokens_then_logs() { - let logger = Arc::new(CapturingLogger::default()); - let callbacks: Vec> = vec![logger.clone()]; - let mut streaming = RealTimeStreaming::new( - callbacks, - "call_abc".to_string(), - "gpt-realtime".to_string(), - RequestMetadata { - user_api_key_hash: Some("hash123".to_string()), - user_api_key_user_id: Some("user-1".to_string()), - user_api_key_team_id: Some("team-1".to_string()), - }, - ); - - streaming.observe(&event( - r#"{"type":"session.created","session":{"id":"sess_001","model":"gpt-realtime-2025"}}"#, - )); - streaming.observe(&event( - r#"{"type":"response.done","response":{"usage":{"input_tokens":10,"output_tokens":5,"total_tokens":15}}}"#, - )); - // A second response.done accumulates. - streaming.observe(&event( - r#"{"type":"response.done","response":{"usage":{"input_tokens":3,"output_tokens":2,"total_tokens":5}}}"#, - )); - - let payload = streaming.build_payload(); - assert_eq!(payload.model, "gpt-realtime-2025"); - // Request-id rule: session.created's id becomes BOTH id and - // litellm_call_id (replacing the "call_abc" gateway fallback), so the - // SpendLogs request_id is always the OpenAI session id. - assert_eq!(payload.id, "sess_001"); - assert_eq!(payload.litellm_call_id, "sess_001"); - assert_eq!(payload.prompt_tokens, 13); - assert_eq!(payload.completion_tokens, 7); - assert_eq!(payload.total_tokens, 20); - assert_eq!(payload.response_cost, 0.0); - assert_eq!(payload.call_type, "realtime"); - assert_eq!(payload.custom_llm_provider, "openai"); - assert_eq!( - payload.metadata.user_api_key_hash.as_deref(), - Some("hash123") - ); - - streaming.log_messages(SessionStatus::Success).await; - assert_eq!(logger.calls.load(Ordering::SeqCst), 1); - assert_eq!( - logger.last_model.lock().unwrap().as_deref(), - Some("gpt-realtime-2025") - ); - assert_eq!(logger.last_total_tokens.load(Ordering::SeqCst), 20); - assert_eq!(streaming.dropped(), 0); - } - - #[test] - fn blank_session_id_and_model_keep_the_gateway_fallbacks() { - let mut streaming = RealTimeStreaming::new( - Vec::new(), - "call_fallback".to_string(), - "gpt-realtime".to_string(), - RequestMetadata::default(), - ); - - streaming.observe(&event( - r#"{"type":"session.created","session":{"id":"","model":""}}"#, - )); - let payload = streaming.build_payload(); - assert_eq!(payload.id, "call_fallback"); - assert_eq!(payload.litellm_call_id, "call_fallback"); - assert_eq!(payload.model, "gpt-realtime"); - - streaming.observe(&event( - r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#, - )); - let payload = streaming.build_payload(); - assert_eq!(payload.id, "sess_002"); - assert_eq!(payload.litellm_call_id, "sess_002"); - assert_eq!(payload.model, "gpt-realtime"); - } - - #[test] - fn payload_serializes_with_camelcase_times_and_realtime_call_type() { - let mut streaming = RealTimeStreaming::new( - Vec::new(), - "call_xyz".to_string(), - "gpt-realtime".to_string(), - RequestMetadata::default(), - ); - streaming.observe(&event( - r#"{"type":"response.done","response":{"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}"#, - )); - streaming.set_response_cost(0.0042); - let payload = streaming.build_payload(); - let json = serde_json::to_string(&payload).expect("serialize payload"); - - assert!(json.contains("\"startTime\""), "missing startTime: {json}"); - assert!(json.contains("\"endTime\""), "missing endTime: {json}"); - assert!( - json.contains("\"call_type\":\"realtime\""), - "missing call_type realtime: {json}" - ); - assert!( - json.contains("\"response_cost\""), - "missing response_cost: {json}" - ); - assert_eq!(payload.response_cost, 0.0042); - } - - /// A logger whose enqueue always fails should bump the dropped counter, not - /// panic or propagate. - #[tokio::test] - async fn failing_logger_bumps_dropped_counter() { - struct FailingLogger; - impl CustomLogger for FailingLogger { - fn async_log_success_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Err(LogError::channel_full()) }) - } - - fn async_log_failure_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Err(LogError::channel_closed()) }) - } - } - let callbacks: Vec> = vec![Arc::new(FailingLogger)]; - let mut streaming = RealTimeStreaming::new( - callbacks, - "call_1".to_string(), - "gpt-realtime".to_string(), - RequestMetadata::default(), - ); - streaming.log_messages(SessionStatus::Success).await; - assert_eq!(streaming.dropped(), 1); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md deleted file mode 100644 index c675916f71a..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md +++ /dev/null @@ -1,43 +0,0 @@ -# routes/ — the route template - -Every route follows the **same shape** so the layout is predictable. The rule: - -> **Each route module exposes `pub fn router() -> Router`.** -> `routes/mod.rs::app` merges them all and applies state once. Adding a route is: -> create the module, then add one `.merge(::router())` line. - -## Default: one file -A route is a single file containing `router()` + its handler(s) (handlers stay -private). This is the norm — don't split until it hurts. -``` -pub fn router() -> Router { Router::new().route(PATH, get(handle)) } -async fn handle(...) -> impl IntoResponse { ... } -``` -`health.rs` is the example. - -## Split out `service` when there's real logic -When a route has business logic worth testing without axum, put it in a sibling -`service` (a file, or a folder if the route grows). The route file stays the -**axum surface** (router + handler + any socket/SSE adapter); `service` is plain -Rust with **no axum types**, and its job is to pick the deployment and call the -`core` route entrypoint (see `messages/service.rs` calling -`litellm_core::messages::messages`). Never build a provider request, resolve a -key, or perform the provider call here. `realtime/` is the older example: -``` -realtime/ - mod.rs # axum surface: router() + handler + the WS<->events adapter - service.rs # pure logic: select deployment + call provider (no axum) — testable -``` -Split `service` further (or add `transport`, `repo`, …) only once a single file -genuinely gets hard to read. - -## Invariants -- **Auth is an extractor, not a manual call.** A handler requires auth by adding - `crate::auth::RequireMasterKey` to its arguments; it runs during extraction. - Never re-implement the check per route. -- **Handlers contain no business logic; `service` contains no axum types.** -- **No provider handlers in this crate.** Transforms, auth headers, and the - provider HTTP call live in `core/src//`. -- A route owns its paths in its own `router()`; `mod.rs` only merges. -- Cross-cutting concerns (logging, CORS, timeouts) → Tower layers in `mod.rs`, - not duplicated in handlers. diff --git a/litellm-rust/crates/ai-gateway/src/routes/health.rs b/litellm-rust/crates/ai-gateway/src/routes/health.rs deleted file mode 100644 index c64ca3a7199..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/health.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Health probes. Simple-route template: a `router()` plus its handlers, in one file. - -use axum::Router; -use axum::http::StatusCode; -use axum::routing::get; - -use crate::state::AppState; - -/// This route's contribution to the app router. -pub fn router() -> Router { - Router::new() - .route("/health/liveness", get(liveness)) - .route("/health/readiness", get(readiness)) -} - -/// The process is up. -async fn liveness() -> StatusCode { - StatusCode::OK -} - -/// The server is ready to accept traffic. -async fn readiness() -> StatusCode { - StatusCode::OK -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs deleted file mode 100644 index 3334053a0a4..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ /dev/null @@ -1,532 +0,0 @@ -//! `POST /v1/messages`, the Anthropic Messages HTTP surface. - -mod service; - -use axum::Router; -use axum::body::Body; -use axum::extract::{Json, State}; -use axum::http::StatusCode; -use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; -use axum::response::{IntoResponse, Response}; -use axum::routing::post; -use litellm_core::Error; -use serde_json::{Map, Value}; - -use crate::auth::RequireMasterKey; -use crate::constants::{MESSAGES_HEADERS_NOT_FORWARDED, MESSAGES_ROUTE_PATH}; -use crate::state::AppState; - -/// This route's contribution to the app router. -pub fn router() -> Router { - Router::new().route(MESSAGES_ROUTE_PATH, post(handle)) -} - -#[tracing::instrument( - name = "messages_gateway_route", - target = "litellm::function_trace", - level = "trace", - skip_all -)] -async fn handle( - _auth: RequireMasterKey, - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Result { - let extra_headers = forwarded_headers(&headers)?; - match service::run(&state.router, body, extra_headers) - .await - .map_err(MessagesRouteError::from)? - { - service::MessagesResponse::Json(body) => Ok(Json(body).into_response()), - service::MessagesResponse::Stream(upstream) => stream_response(upstream), - } -} - -fn stream_response(upstream: reqwest::Response) -> Result { - let content_type = upstream - .headers() - .get(CONTENT_TYPE) - .cloned() - .unwrap_or_else(|| HeaderValue::from_static("text/event-stream")); - let mut response = Response::builder() - .status( - StatusCode::from_u16(upstream.status().as_u16()).map_err(|error| { - MessagesRouteError(Error::InvalidResponse(format!( - "invalid upstream response status: {error}" - ))) - })?, - ) - .header(CONTENT_TYPE, content_type); - if let Some(value) = upstream.headers().get(CACHE_CONTROL) { - response = response.header(CACHE_CONTROL, value); - } - response - .body(Body::from_stream(upstream.bytes_stream())) - .map_err(|error| { - MessagesRouteError(Error::InvalidResponse(format!( - "failed to build streaming response: {error}" - ))) - }) -} - -fn forwarded_headers(headers: &HeaderMap) -> Result>, Error> { - let forwarded = headers - .iter() - .filter(|(name, _)| { - !MESSAGES_HEADERS_NOT_FORWARDED - .iter() - .any(|excluded| name.as_str().eq_ignore_ascii_case(excluded)) - }) - .map(|(name, value)| { - let value = value.to_str().map_err(|_| { - Error::InvalidRequest(format!("invalid value for header {}", name.as_str())) - })?; - Ok((name.to_string(), Value::String(value.to_string()))) - }) - .collect::, Error>>()?; - Ok((!forwarded.is_empty()).then_some(forwarded)) -} - -#[derive(Debug)] -struct MessagesRouteError(Error); - -impl From for MessagesRouteError { - fn from(error: Error) -> Self { - Self(error) - } -} - -impl IntoResponse for MessagesRouteError { - fn into_response(self) -> Response { - let (status, message) = match self.0 { - Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), - Error::InvalidProvider(_) | Error::Routing(_) => ( - StatusCode::NOT_FOUND, - "no messages deployment is configured for this model".to_string(), - ), - Error::Auth(_) - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureDocumentIntelligenceCredentials - | Error::MissingReductoApiKey => ( - StatusCode::BAD_GATEWAY, - "messages provider authentication failed".to_string(), - ), - Error::Http { .. } - | Error::Network(_) - | Error::Connect(_) - | Error::InvalidResponse(_) - | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::MissingDocumentUrl => ( - StatusCode::BAD_GATEWAY, - "messages provider request failed".to_string(), - ), - // The gateway has no Python implementation to decline to, so a - // request the core cannot serve is reported to the caller. The - // reason is a fixed internal string, never provider content. - Error::Unsupported(reason) => ( - StatusCode::BAD_REQUEST, - format!("messages request is not supported: {reason}"), - ), - }; - ( - status, - Json(serde_json::json!({"error": {"message": message}})), - ) - .into_response() - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use axum::body::Body; - use axum::http::Request; - use axum::http::StatusCode; - use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE}; - use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter}; - use serde_json::json; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::net::TcpListener; - use tower::ServiceExt; - - use super::super::app; - use crate::io::realtime_pool::RealtimePool; - use crate::state::AppState; - - fn state(model: &str, api_base: String, master_key: Option<&str>) -> AppState { - state_with_provider(model, model, api_base, master_key) - } - - fn state_with_provider( - model_alias: &str, - provider_model: &str, - api_base: String, - master_key: Option<&str>, - ) -> AppState { - AppState { - router: Arc::new(ModelRouter::new(vec![Deployment { - model_name: model_alias.to_string(), - litellm_params: LiteLLMParams { - model: format!("anthropic/{provider_model}"), - api_key: Some("upstream-key".to_string()), - api_base: Some(api_base), - }, - }])), - master_key: master_key.map(Arc::from), - loggers: Arc::new(Vec::new()), - realtime_pool: RealtimePool::disabled(), - } - } - - async fn upstream(listener: TcpListener) -> (String, tokio::task::JoinHandle) { - let address = listener.local_addr().expect("listener has address"); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts request"); - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - let read = socket.read(&mut buffer).await.expect("reads request"); - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - let request = String::from_utf8(request).expect("request is utf8"); - let content_length = request - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - let header_end = request.find("\r\n\r\n").expect("request has headers") + 4; - let mut full_request = request.into_bytes(); - while full_request.len().saturating_sub(header_end) < content_length { - let read = socket.read(&mut buffer).await.expect("reads body"); - full_request.extend_from_slice(&buffer[..read]); - } - let request = String::from_utf8(full_request).expect("request is utf8"); - let body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-test"}"#; - let response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - body.len(), - body - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - request - }); - (format!("http://{address}"), server) - } - - async fn streaming_upstream( - listener: TcpListener, - status: u16, - content_type: &'static str, - body: &'static str, - ) -> (String, tokio::task::JoinHandle) { - let address = listener.local_addr().expect("listener has address"); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts request"); - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - let read = socket.read(&mut buffer).await.expect("reads request"); - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - let request_text = String::from_utf8(request).expect("request is utf8"); - let content_length = request_text - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - let header_end = request_text.find("\r\n\r\n").expect("request has headers") + 4; - let mut full_request = request_text.into_bytes(); - while full_request.len().saturating_sub(header_end) < content_length { - let read = socket.read(&mut buffer).await.expect("reads body"); - full_request.extend_from_slice(&buffer[..read]); - } - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: {content_type}\r\ncache-control: no-cache\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - String::from_utf8(full_request).expect("request is utf8") - }); - (format!("http://{address}"), server) - } - - #[tokio::test] - async fn route_constructs_anthropic_upstream_request() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let (api_base, server) = upstream(listener).await; - let app = app(state("claude-test", api_base, Some("master-key"))); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("x-api-key", "request-upstream-key") - .header("anthropic-beta", "beta-feature") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "claude-test", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body reads"); - assert_eq!( - serde_json::from_slice::(&body).expect("json")["id"], - "msg_1" - ); - let upstream_request = server.await.expect("upstream task completes"); - let (head, body) = upstream_request - .split_once("\r\n\r\n") - .expect("upstream request has body"); - let head = head.to_ascii_lowercase(); - assert!(head.contains("x-api-key: request-upstream-key")); - assert!(head.contains("anthropic-beta: beta-feature")); - assert!(!head.contains("authorization: bearer master-key")); - let body: serde_json::Value = serde_json::from_str(body).expect("upstream body is json"); - assert_eq!(body["model"], "claude-test"); - assert_eq!(body["messages"][0]["content"], "hello"); - } - - #[tokio::test] - async fn route_substitutes_model_alias_with_provider_model_upstream() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let (api_base, server) = upstream(listener).await; - let app = app(state_with_provider( - "production", - "claude-sonnet-4-5", - api_base, - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "production", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::OK); - let upstream_request = server.await.expect("upstream task completes"); - let (_, upstream_body) = upstream_request - .split_once("\r\n\r\n") - .expect("upstream request has body"); - let upstream_body: serde_json::Value = - serde_json::from_str(upstream_body).expect("upstream body is json"); - assert_eq!(upstream_body["model"], "claude-sonnet-4-5"); - assert_ne!(upstream_body["model"], "production"); - } - - #[tokio::test] - async fn route_streams_anthropic_events_without_buffering_or_reordering() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let events = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\"}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; - let (api_base, server) = - streaming_upstream(listener, 200, "text/event-stream", events).await; - let app = app(state("claude-test", api_base, Some("master-key"))); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "claude-test", - "max_tokens": 16, - "stream": true, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response - .headers() - .get(CONTENT_TYPE) - .unwrap() - .to_str() - .unwrap(), - "text/event-stream" - ); - assert_eq!( - response - .headers() - .get(CACHE_CONTROL) - .unwrap() - .to_str() - .unwrap(), - "no-cache" - ); - let response_body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body reads"); - assert_eq!(response_body, events.as_bytes()); - let upstream_request = server.await.expect("upstream task completes"); - let (_, upstream_body) = upstream_request - .split_once("\r\n\r\n") - .expect("upstream request has body"); - assert_eq!( - serde_json::from_str::(upstream_body) - .expect("upstream body is json")["stream"], - true - ); - } - - #[tokio::test] - async fn route_maps_streaming_upstream_errors_before_starting_response() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let (api_base, server) = streaming_upstream( - listener, - 429, - "application/json", - r#"{"error":"rate limited"}"#, - ) - .await; - let app = app(state("claude-test", api_base, Some("master-key"))); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "claude-test", - "max_tokens": 16, - "stream": true, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - let response_body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body reads"); - assert_eq!( - serde_json::from_slice::(&response_body).expect("error is json")["error"] - ["message"], - "messages provider request failed" - ); - server.await.expect("upstream task completes"); - } - - #[tokio::test] - async fn route_rejects_missing_master_key() { - let app = app(state( - "claude-test", - "http://127.0.0.1:1".to_string(), - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("content-type", "application/json") - .body(Body::from("{}")) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn route_rejects_invalid_master_key() { - let app = app(state( - "claude-test", - "http://127.0.0.1:1".to_string(), - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer wrong-key") - .header("content-type", "application/json") - .body(Body::from("{}")) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn route_rejects_malformed_json_without_panicking() { - let app = app(state( - "claude-test", - "http://127.0.0.1:1".to_string(), - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from("{not-json")) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs deleted file mode 100644 index 5434719987b..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::sync::Arc; - -use litellm_core::Error; -use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; -use litellm_core::messages::types::MessagesRequest; -use litellm_core::messages::{messages, messages_stream}; -use litellm_core::router::Router; -use serde_json::{Map, Value}; - -pub(crate) enum MessagesResponse { - Json(Value), - Stream(reqwest::Response), -} - -#[tracing::instrument( - name = "messages_gateway_service", - target = "litellm::function_trace", - level = "trace", - skip_all -)] -pub async fn run( - router: &Arc, - body: Value, - extra_headers: Option>, -) -> Result { - let model = body - .get("model") - .and_then(Value::as_str) - .map(str::trim) - .filter(|model| !model.is_empty()) - .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?; - let deployment = router - .get_available_deployment(model) - .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; - let provider_model = deployment.litellm_params.model.as_str(); - let upstream_model = provider_model - .split_once('/') - .map_or(provider_model, |(_, model)| model); - let custom_llm_provider = if provider_model.contains('/') { - None - } else { - Some(ANTHROPIC_MESSAGES_PROVIDER) - }; - let mut body = body; - body.as_object_mut() - .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))? - .insert( - "model".to_string(), - Value::String(upstream_model.to_string()), - ); - - let request = MessagesRequest { - model: provider_model, - body, - api_key: deployment.litellm_params.api_key.as_deref(), - api_base: deployment.litellm_params.api_base.as_deref(), - custom_llm_provider, - extra_headers, - timeout: None, - }; - if request.body.get("stream").and_then(Value::as_bool) == Some(true) { - return messages_stream(request).await.map(MessagesResponse::Stream); - } - - let response = messages(request).await?; - serde_json::to_value(response) - .map(MessagesResponse::Json) - .map_err(|err| { - Error::InvalidResponse(format!("failed to serialize messages response: {err}")) - }) -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/mod.rs deleted file mode 100644 index 71b05c7d64b..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! HTTP routes. -//! -//! **Template:** every route module exposes `pub fn router() -> Router` -//! that mounts its own paths; [`app`] merges them. A trivial route is a single -//! file (`health.rs`); a non-trivial one is a folder (`realtime/`) with -//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md. - -pub mod health; -pub mod messages; -pub mod realtime; -pub mod responses; - -use axum::Router; - -use crate::state::AppState; - -/// Assemble the application router by merging every route module's `router()`. -pub fn app(state: AppState) -> Router { - Router::new() - .merge(health::router()) - .merge(messages::router()) - .merge(realtime::router()) - .merge(responses::router()) - .with_state(state) -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md b/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md deleted file mode 100644 index 3301576bb85..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Realtime route (`GET /v1/realtime`) - -Proxies OpenAI's realtime WebSocket. `mod.rs` is the axum surface (handler + -socket↔events adapter); `service.rs` is the pure logic (select a deployment, then -splice client ↔ upstream). The pool itself lives in -`crates/providers/src/realtime_pool.rs`. - -## Connection pooling - -### The problem - -The gateway's realtime overhead lives **entirely in session establishment**. On each -client connect it dials a *fresh* upstream WS to OpenAI and waits for -`session.created` before it can serve. Measured at 5000 calls / 500 concurrency, the -fresh-dial session phase is **~360 ms** vs **~7 ms** direct; dial, first-audio, and -streaming add ~0. So the one lever is removing that per-connect handshake from the -critical path. - -### The idea - -Keep a few upstream OpenAI sockets **already connected and already past -`session.created`** (buffered). On a client connect, hand off a warm socket — relay -its buffered `session.created` instantly (a local `Vec::pop`, sub-millisecond) and -splice exactly as a fresh dial would. A background task keeps the pool topped up. On -a miss or dead socket we fall back to fresh-dial: the pool is a latency optimization, -never a correctness dependency. - -``` - ┌───────────────────────────────────────┐ - client connect ──────► │ routes/realtime → service::run │ - │ pool.take(key) │ - │ hit → relay buffered │ - │ session.created, then splice │ - │ miss → fresh dial (original path) │ - └───────────────┬───────────────────────┘ - │ replenish (async, concurrent) - ┌───────────────▼───────────────────────┐ - background task ─────► │ RealtimePool: per-key warm sockets │ - │ each = { ws, buffered session.created}│ - │ liveness-checked before handoff │ - └─────────────────────────────────────────┘ -``` - -A warm session is indistinguishable from a fresh one: OpenAI sends `session.created` -unprompted on connect, we pre-read exactly that one frame and relay it on handoff, -and we send nothing else on the socket before a client exists — so the client's first -`session.update` behaves identically either way. - -### Sizing - -Each warm socket serves **exactly one** session (realtime isn't multiplexed), so the -pool is sized to the **peak concurrent connects per instance**, not total live -connections: - -``` -REALTIME_POOL_SIZE ≈ peak_concurrency / instance_count -``` - -e.g. 500 concurrency over 10 instances → ~50–64 per instance. The replenisher dials -the missing sockets **concurrently**, so a drained pool refills in ~one handshake -window and keeps supply close to the connect rate. Over-provisioning just burns idle -upstream sockets, which is why warm sockets are short-lived -(`REALTIME_POOL_MAX_IDLE_SECS`). - -### Config - -| env | default | meaning | -| ----------------------------- | ------- | --------------------------------------------------------------- | -| `REALTIME_POOL_SIZE` | `4` | target warm sockets per key. `0` disables pooling (fresh-dial). | -| `REALTIME_POOL_MAX_IDLE_SECS` | `30` | max time a warm socket sits before it's closed and replaced. | - -### Notes - -- **Miss / dead socket → fresh dial.** Burst beyond warm supply, or a socket that - died, never blocks or fails — it falls back to the original path. The pool can only - make a connect faster, never slower or more fragile. -- **Auth scope.** The pool key includes `api_key`, so a warm socket is only handed to - a request resolving to the same key — no cross-tenant reuse. -- **Idle billing.** Warm sockets are liveness-checked at handoff and capped at - `REALTIME_POOL_MAX_IDLE_SECS` to bound idle billing and dodge OpenAI's idle timeout. -- **Replenish backoff.** If a key's warm-up dials all fail (invalid credentials, an - unreachable upstream), the replenisher puts that key into exponential backoff - (500 ms → 30 s cap) instead of re-dialing it every tick. This bounds connection - attempts against a broken key so it can't exhaust upstream rate limits and degrade - valid cold-path traffic; the backoff resets the moment a dial succeeds. - -Benchmarks and repro: `../../benchmarks/realtime/README.md`. diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs deleted file mode 100644 index f9144ad1fdb..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! `GET /v1/realtime` (WebSocket). -//! -//! This file is the **axum surface**: `router()`, the handler, and the small -//! socket↔events adapter. The pure logic (no axum) lives in [`service`]. Auth is -//! the `RequireMasterKey` extractor, so the handler stays thin. - -mod service; - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::io::realtime_pool::RealtimePool; -use axum::Router; -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::response::Response; -use axum::routing::get; -use futures_util::{SinkExt, StreamExt}; -use litellm_core::realtime::types::RealtimeEvent; -use litellm_core::router::Router as ModelRouter; -use serde::Deserialize; - -use crate::auth::RequireMasterKey; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; -use crate::realtime::streaming::{RealTimeStreaming, SessionStatus}; -use crate::state::AppState; - -/// Process-local monotonic counter, mixed into the per-session call id so two -/// sessions opened in the same nanosecond still get distinct ids. -static CALL_SEQ: AtomicU64 = AtomicU64::new(0); - -/// Generate a per-connection `litellm_call_id`. No external uuid dep: epoch -/// nanos + a process-local sequence is unique enough for log correlation. -fn new_call_id() -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let seq = CALL_SEQ.fetch_add(1, Ordering::Relaxed); - format!("rt-{nanos:x}-{seq:x}") -} - -/// This route's contribution to the app router. -pub fn router() -> Router { - Router::new().route("/v1/realtime", get(handle)) -} - -#[derive(Debug, Deserialize)] -struct RealtimeQuery { - model: String, -} - -/// Auth runs via the `RequireMasterKey` extractor. We validate the model BEFORE -/// the upgrade so failures are clean HTTP (400/404), not a socket that opens then -/// closes, then hand the socket to `bridge`. -async fn handle( - _auth: RequireMasterKey, - ws: WebSocketUpgrade, - State(state): State, - Query(query): Query, -) -> Result { - if query.model.trim().is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "missing 'model' query param".to_string(), - )); - } - if !state.router.has_deployment(&query.model) { - return Err(( - StatusCode::NOT_FOUND, - format!("no deployment for model '{}'", query.model), - )); - } - - let router = state.router.clone(); - let pool = state.realtime_pool.clone(); - let loggers = state.loggers.clone(); - let master_key = state.master_key.clone(); - let model = query.model; - Ok(ws.on_upgrade(move |socket| bridge(socket, router, pool, loggers, master_key, model))) -} - -/// Adapt the axum socket (text frames) to the typed-event `Stream`/`Sink` the -/// service wants, keeping axum types out of `service`. -/// -/// This is also the realtime-logging seam: every upstream→client event (the -/// direction carrying `session.created` and `response.done` with usage) is fed -/// to a [`RealTimeStreaming`] collector via the splice's `observe` callback. The -/// observe is O(1) and never buffers frames. When the splice returns (any of the -/// three break paths — client disconnect, upstream close, idle timeout), we flush -/// one logging payload to the registered callbacks. -async fn bridge( - socket: WebSocket, - router: Arc, - pool: Arc, - loggers: Arc>>, - master_key: Option>, - model: String, -) { - let (ws_sink, ws_stream) = socket.split(); - - // Attribute the spend log to the key that authenticated this session (the - // master key — the gateway is master-key auth). A non-null user_api_key_hash - // is required for the Python spend logger to write a SpendLogs row. - // - // SECURITY: hash the key — never send the raw credential. This field fans out - // to spend logs and every callback integration; the SHA-256 (matching the - // proxy's hash_token) keeps the plaintext master key out of all of them while - // still matching the key's hash in LiteLLM_SpendLogs. - let metadata = RequestMetadata { - user_api_key_hash: master_key.as_deref().map(crate::auth::hash_token), - ..RequestMetadata::default() - }; - - // Owned by THIS task only. The splice observes it via a synchronous `&mut` - // callback (below), so there is no Arc/Mutex/atomic on the per-frame hot - // path — just a monomorphized FnMut mutating stack-local fields. This is - // what lets observe scale: 10K concurrent sessions = 10K independent - // collectors, zero cross-task synchronization. - let mut collector = RealTimeStreaming::new( - loggers.as_ref().clone(), - new_call_id(), - model.clone(), - metadata, - ); - - let client_in = ws_stream.filter_map(|message| async move { - match message { - Ok(Message::Text(text)) => serde_json::from_str::(&text).ok(), - _ => None, - } - }); - // Plain forwarding sink — no observe here anymore. - let client_out = ws_sink.with(|event: RealtimeEvent| async move { - Ok::(Message::Text( - serde_json::to_string(&event).unwrap_or_default(), - )) - }); - - futures_util::pin_mut!(client_in, client_out); - - // The observe closure borrows `&mut collector` for the duration of the - // splice; the borrow ends when `run` returns, freeing the collector for the - // single post-session `log_messages` flush. `run` picks a pooled (warm) or - // fresh upstream — observe fires on the upstream arm either way. - let result = service::run( - &router, - &pool, - &model, - None, - |event: &RealtimeEvent| collector.observe(event), - client_in, - client_out, - ) - .await; - - let status = if result.is_ok() { - SessionStatus::Success - } else { - SessionStatus::Failure - }; - collector.log_messages(status).await; -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs deleted file mode 100644 index f7bbb37dff4..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Business logic: select a deployment with the (pure) core router, then call the -//! provider splice. The seam between `core::router` (selection only) and -//! `io` (the actual WebSocket I/O). -//! -//! On connect we try a pre-warmed upstream from the pool (handshake already paid, -//! `session.created` buffered) and relay it instantly. On a pool miss or dead warm -//! socket we fresh-dial exactly as before — the pool is never on the critical path -//! for correctness, only latency. - -use std::time::Duration; - -use crate::io::realtime_pool::{RealtimePool, upstream_key}; -use futures_util::{Sink, Stream}; -use litellm_core::error::Error; -use litellm_core::realtime::types::RealtimeEvent; -use litellm_core::router::Router; - -/// Select a deployment for `model` and splice the client stream to the provider. -/// -/// `pool` supplies a pre-warmed upstream when one is available; otherwise we -/// fresh-dial. A disabled pool always misses, so this collapses to the original -/// fresh-dial behavior. -pub async fn run( - router: &Router, - pool: &RealtimePool, - model: &str, - idle_timeout: Option, - observe: impl FnMut(&RealtimeEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - let deployment = router - .get_available_deployment(model) - .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; - let params = &deployment.litellm_params; - // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. - let provider_model = params - .model - .strip_prefix("openai/") - .unwrap_or(¶ms.model); - - // Warm path: take a pooled upstream (handshake already paid) and relay its - // buffered session.created immediately. On miss/dead socket fall through. - if let Some(key) = upstream_key( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - ) && let Some(handoff) = pool.take(&key) - { - return crate::io::realtime::realtime_warm( - provider_model, - handoff, - idle_timeout, - observe, - client_in, - client_out, - ) - .await; - } - - // Cold path: fresh dial (the original behavior). - crate::io::realtime::realtime( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs deleted file mode 100644 index a94853e106d..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs +++ /dev/null @@ -1,348 +0,0 @@ -mod service; - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use axum::Router; -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::response::Response; -use axum::routing::get; -use futures_util::{Sink, SinkExt, StreamExt}; -use litellm_core::responses::types::{ResponsesErrorFrame, ResponsesWsEvent, ResponsesWsEventType}; -use litellm_core::router::Router as ModelRouter; -use serde::Deserialize; - -use crate::auth::RequireMasterKey; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; -use crate::state::AppState; - -static CALL_SEQ: AtomicU64 = AtomicU64::new(0); - -fn new_call_id() -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let sequence = CALL_SEQ.fetch_add(1, Ordering::Relaxed); - format!("respws-{nanos:x}-{sequence:x}") -} - -pub fn router() -> Router { - Router::new() - .route("/v1/responses", get(handle)) - .route("/responses", get(handle)) -} - -#[derive(Debug, Deserialize)] -struct ResponsesQuery { - model: Option, -} - -async fn handle( - _auth: RequireMasterKey, - ws: WebSocketUpgrade, - State(state): State, - Query(query): Query, -) -> Result { - if let Some(model) = query.model.as_deref() { - validate_model(&state.router, model)?; - } - let router = state.router.clone(); - let loggers = state.loggers.clone(); - let master_key = state.master_key.clone(); - Ok(ws.on_upgrade(move |socket| bridge(socket, router, loggers, master_key, query.model))) -} - -fn validate_model(router: &ModelRouter, model: &str) -> Result<(), (StatusCode, String)> { - if model.trim().is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "missing 'model' query param".to_string(), - )); - } - let Some(deployment) = router.get_available_deployment(model) else { - return Err(( - StatusCode::NOT_FOUND, - format!("no deployment for model '{model}'"), - )); - }; - if deployment.litellm_params.model.contains('/') - && !deployment.litellm_params.model.starts_with("openai/") - { - return Err(( - StatusCode::BAD_REQUEST, - "Responses WebSocket route supports OpenAI deployments only".to_string(), - )); - } - Ok(()) -} - -async fn send_error_and_close(sink: &mut S, message: String) -where - S: futures_util::Sink + Unpin, - S::Error: std::fmt::Display, -{ - if let Ok(payload) = serde_json::to_string(&ResponsesErrorFrame::invalid_request(message)) { - let _ = sink.send(Message::Text(payload)).await; - } - let _ = sink - .send(Message::Close(Some(axum::extract::ws::CloseFrame { - code: 1008, - reason: "Pre-call error".into(), - }))) - .await; - let _ = sink.close().await; -} - -struct ResponseClientSink { - sink: futures_util::stream::SplitSink, -} - -impl Sink for ResponseClientSink { - type Error = axum::Error; - - fn poll_ready( - mut self: std::pin::Pin<&mut Self>, - context: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.sink).poll_ready(context) - } - - fn start_send( - mut self: std::pin::Pin<&mut Self>, - item: ResponsesWsEvent, - ) -> Result<(), Self::Error> { - let payload = serde_json::to_string(&item).map_err(axum::Error::new)?; - std::pin::Pin::new(&mut self.sink).start_send(Message::Text(payload)) - } - - fn poll_flush( - mut self: std::pin::Pin<&mut Self>, - context: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.sink).poll_flush(context) - } - - fn poll_close( - mut self: std::pin::Pin<&mut Self>, - context: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.sink).poll_close(context) - } -} - -impl ResponseClientSink { - async fn close_with_code(&mut self, code: u16, reason: &'static str) { - let _ = self - .sink - .send(Message::Close(Some(axum::extract::ws::CloseFrame { - code, - reason: reason.into(), - }))) - .await; - let _ = self.sink.close().await; - } -} - -async fn bridge( - socket: WebSocket, - router: Arc, - loggers: Arc>>, - master_key: Option>, - requested_model: Option, -) { - let (mut ws_sink, ws_stream) = socket.split(); - let (model, first_frame, stream) = if let Some(model) = requested_model { - (model, None, ws_stream) - } else { - let mut stream = ws_stream; - let first = match stream.next().await { - Some(Ok(Message::Text(text))) => { - match serde_json::from_str::(&text) { - Ok(event) => event, - Err(_) => { - send_error_and_close( - &mut ws_sink, - "Invalid JSON in response.create event".to_string(), - ) - .await; - return; - } - } - } - _ => { - send_error_and_close(&mut ws_sink, "Missing response.create event".to_string()) - .await; - return; - } - }; - let Some(model) = first.model().filter(|value| !value.trim().is_empty()) else { - send_error_and_close( - &mut ws_sink, - "Missing model in response.create event".to_string(), - ) - .await; - return; - }; - if first.event_type != ResponsesWsEventType::ResponseCreate { - send_error_and_close( - &mut ws_sink, - "First frame must be a response.create event".to_string(), - ) - .await; - return; - } - (model.to_string(), Some(first), stream) - }; - if let Err((status, message)) = validate_model(&router, &model) { - let _ = status; - let _ = message; - send_error_and_close(&mut ws_sink, "Unknown model deployment".to_string()).await; - return; - } - - let call_id = new_call_id(); - let metadata = RequestMetadata { - user_api_key_hash: master_key.as_deref().map(crate::auth::hash_token), - ..RequestMetadata::default() - }; - let client_in = Box::pin(stream.filter_map(|message| async move { - match message { - Ok(Message::Text(text)) => serde_json::from_str::(&text).ok(), - _ => None, - } - })); - let mut client_out = ResponseClientSink { sink: ws_sink }; - let result = service::run( - &router, - &model, - first_frame, - None, - loggers, - call_id, - metadata, - client_in, - &mut client_out, - ) - .await; - if result.is_err() { - client_out - .close_with_code(1011, "Internal server error") - .await; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::io::realtime_pool::RealtimePool; - use crate::state::AppState; - use axum::body::Body; - use axum::http::Request; - use litellm_core::router::Router as ModelRouter; - use serde_json::json; - use std::pin::Pin; - use std::sync::Arc; - use std::task::{Context, Poll}; - use tower::ServiceExt; - - struct RecordingSink { - messages: Vec, - } - - impl Sink for RecordingSink { - type Error = std::convert::Infallible; - - fn poll_ready( - self: Pin<&mut Self>, - _context: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - - fn start_send(mut self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> { - self.messages.push(item); - Ok(()) - } - - fn poll_flush( - self: Pin<&mut Self>, - _context: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_close( - self: Pin<&mut Self>, - _context: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - } - - #[tokio::test] - async fn pre_call_error_matches_python_frame_and_close() { - let mut sink = RecordingSink { - messages: Vec::new(), - }; - send_error_and_close(&mut sink, "missing model".to_string()).await; - let Message::Text(payload) = &sink.messages[0] else { - panic!("expected error text frame"); - }; - assert_eq!( - serde_json::from_str::(payload).expect("error json"), - json!({ - "type": "error", - "error": { - "type": "invalid_request_error", - "message": "missing model" - } - }) - ); - assert_eq!( - sink.messages[1], - Message::Close(Some(axum::extract::ws::CloseFrame { - code: 1008, - reason: "Pre-call error".into(), - })) - ); - } - - fn state() -> AppState { - AppState { - router: Arc::new(ModelRouter::default()), - master_key: Some(Arc::from("master-key")), - loggers: Arc::new(Vec::new()), - realtime_pool: RealtimePool::disabled(), - } - } - - #[tokio::test] - async fn auth_rejects_responses_upgrade_before_handler() { - let request = Request::builder() - .uri("/responses?model=known") - .body(Body::empty()) - .expect("request"); - let response = router() - .with_state(state()) - .oneshot(request) - .await - .expect("response"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn unknown_query_model_is_rejected_before_upgrade() { - assert_eq!( - validate_model(&ModelRouter::default(), "unknown").expect_err("unknown model"), - ( - StatusCode::NOT_FOUND, - "no deployment for model 'unknown'".to_string() - ) - ); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs deleted file mode 100644 index e8f840c0c8e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs +++ /dev/null @@ -1,156 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use futures_util::{Sink, Stream}; -use litellm_core::Error; -use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext}; -use litellm_core::responses::instrumentation::{ - ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome, - ResponsesWsMetadata, -}; -use litellm_core::responses::types::ResponsesWsEvent; - -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::RequestMetadata; - -#[allow(clippy::too_many_arguments)] -pub async fn run( - router: &litellm_core::router::Router, - model: &str, - first_frame: Option, - idle_timeout: Option, - loggers: Arc>>, - call_id: String, - metadata: RequestMetadata, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - let deployment = router - .get_available_deployment(model) - .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; - let params = &deployment.litellm_params; - let provider_model = params - .model - .strip_prefix("openai/") - .unwrap_or(¶ms.model); - if params.model.contains('/') && !params.model.starts_with("openai/") { - return Err(Error::InvalidProvider( - "Responses WebSocket route supports OpenAI deployments only".to_string(), - )); - } - let instrumentation = Arc::new(ResponsesWsInstrumentation::new( - call_id.clone(), - model, - ResponsesWsMetadata { - user_api_key_hash: metadata.user_api_key_hash, - user_api_key_user_id: metadata.user_api_key_user_id, - user_api_key_team_id: metadata.user_api_key_team_id, - }, - )); - let observer_instrumentation = Arc::clone(&instrumentation); - let context = CallLifecycleContext::new("responses_websocket", model, "openai", call_id); - let result = CallLifecycle::default() - .run(context, (), instrumentation.as_ref(), |_| async move { - crate::io::responses_ws::async_responses_websocket( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - first_frame, - idle_timeout, - move |event| { - observer_instrumentation.observe(event); - }, - client_in, - client_out, - ) - .await - }) - .await; - let outcome = instrumentation.take_or_build_outcome(result.is_ok()); - dispatch_outcome(loggers, outcome).await; - result -} - -async fn dispatch_outcome( - loggers: Arc>>, - outcome: ResponsesWsLogOutcome, -) { - let runner = CustomLoggerRunner::new(loggers.as_ref().clone()); - match outcome { - ResponsesWsLogOutcome::Success { payload, callback } => { - let (details, response, start_time, end_time) = logging_values(payload, callback, None); - let _ = runner - .async_log_success_event( - &details, - &response, - CallbackTiming::new(start_time, end_time), - ) - .await; - } - ResponsesWsLogOutcome::Failure { - payload, - callback, - error_message, - error_kind, - } => { - let error = LoggingError { - message: error_message, - kind: error_kind, - }; - let (details, response, start_time, end_time) = - logging_values(payload, callback, Some(error)); - let _ = runner - .async_log_failure_event( - &details, - Some(&response), - CallbackTiming::new(start_time, end_time), - ) - .await; - } - } -} - -fn logging_values( - payload: litellm_core::responses::instrumentation::ResponsesWsLogPayload, - callback: ResponsesWsCallbackPayload, - error: Option, -) -> (ModelCallDetails, CallbackValue, f64, f64) { - let start_time = payload.start_time; - let end_time = payload.end_time; - let callback = CallbackValue::new(callback.object, callback.value); - let details = ModelCallDetails::from_standard_logging_payload( - crate::integrations::types::StandardLoggingPayload { - id: payload.id, - litellm_call_id: payload.litellm_call_id, - call_type: payload.call_type, - model: payload.model, - custom_llm_provider: payload.custom_llm_provider, - response_cost: payload.response_cost, - prompt_tokens: payload.usage.prompt_tokens, - completion_tokens: payload.usage.completion_tokens, - total_tokens: payload.usage.total_tokens, - start_time: payload.start_time, - end_time: payload.end_time, - stream: payload.stream, - metadata: crate::integrations::types::StandardLoggingMetadata { - user_api_key_hash: payload.metadata.user_api_key_hash, - user_api_key_user_id: payload.metadata.user_api_key_user_id, - user_api_key_team_id: payload.metadata.user_api_key_team_id, - ..Default::default() - }, - messages: None, - }, - ); - let details = match error { - Some(error) => details.with_failure_error(error), - None => details, - }; - (details, callback, start_time, end_time) -} diff --git a/litellm-rust/crates/ai-gateway/src/state.rs b/litellm-rust/crates/ai-gateway/src/state.rs deleted file mode 100644 index 3b61d8309ea..00000000000 --- a/litellm-rust/crates/ai-gateway/src/state.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::sync::Arc; - -use crate::io::realtime_pool::RealtimePool; -use litellm_core::router::Router; - -use crate::integrations::custom_logger::CustomLogger; - -/// Shared application state handed to every route handler. -#[derive(Clone)] -pub struct AppState { - pub router: Arc, - /// The gateway master key. Any caller presenting it as a bearer token may - /// invoke the gateway. `None` → auth not configured (routes fail closed). - pub master_key: Option>, - /// Logging callbacks fanned out at the end of each realtime session. - pub loggers: Arc>>, - /// Pre-warmed upstream realtime connection pool. Disabled - /// (`RealtimePool::disabled()`) when `REALTIME_POOL_SIZE=0`, in which case - /// every realtime connect fresh-dials exactly as before. - pub realtime_pool: Arc, -} diff --git a/litellm-rust/crates/ai-gateway/src/trace_parity.rs b/litellm-rust/crates/ai-gateway/src/trace_parity.rs deleted file mode 100644 index 7540a71fb12..00000000000 --- a/litellm-rust/crates/ai-gateway/src/trace_parity.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Harness-only in-process adapters. Never mounted as production routes. - -use std::sync::Arc; - -use axum::body::{Body, to_bytes}; -use axum::http::header::{AUTHORIZATION, CONTENT_TYPE}; -use axum::http::{Request, StatusCode}; -use litellm_core::Error; -use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter}; -use serde::Serialize; -use serde_json::Value; -use tower::ServiceExt; -use tracing::instrument::WithSubscriber; - -use crate::io::realtime_pool::RealtimePool; -use crate::routes; -use crate::state::AppState; - -#[derive(Debug, Serialize)] -pub struct GatewayResponse { - pub status: u16, - pub body: Value, -} - -#[derive(Debug, Serialize)] -pub struct TracedGatewayResponse { - pub response: Option, - pub error: Option, - pub trace: Vec, -} - -pub async fn traced_request( - path: String, - model_alias: String, - provider_model: String, - api_base: String, - body: Value, -) -> TracedGatewayResponse { - let trace = litellm_core::observability::FunctionTrace::default(); - let result = request(path, model_alias, provider_model, api_base, body) - .with_subscriber(trace.dispatcher()) - .await; - let events = trace.events(); - match result { - Ok(response) => TracedGatewayResponse { - response: Some(response), - error: None, - trace: events, - }, - Err(error) => TracedGatewayResponse { - response: None, - error: Some(error.to_string()), - trace: events, - }, - } -} - -pub async fn request( - path: String, - model_alias: String, - provider_model: String, - api_base: String, - body: Value, -) -> Result { - let state = AppState { - router: Arc::new(ModelRouter::new(vec![Deployment { - model_name: model_alias, - litellm_params: LiteLLMParams { - model: provider_model, - api_key: Some("trace-provider-key".to_string()), - api_base: Some(api_base), - }, - }])), - master_key: Some(Arc::from("trace-master-key")), - loggers: Arc::new(Vec::new()), - realtime_pool: RealtimePool::disabled(), - }; - let request = Request::builder() - .method("POST") - .uri(path) - .header(AUTHORIZATION, "Bearer trace-master-key") - .header(CONTENT_TYPE, "application/json") - .body(Body::from(body.to_string())) - .map_err(|error| Error::InvalidRequest(error.to_string()))?; - let response = match routes::app(state).oneshot(request).await { - Ok(response) => response, - Err(error) => match error {}, - }; - let status: StatusCode = response.status(); - let bytes = to_bytes(response.into_body(), usize::MAX) - .await - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - let body = serde_json::from_slice(&bytes).map_err(|error| { - Error::InvalidResponse(format!("gateway returned invalid JSON: {error}")) - })?; - Ok(GatewayResponse { - status: status.as_u16(), - body, - }) -} diff --git a/litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs b/litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs deleted file mode 100644 index ac37440d682..00000000000 --- a/litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Guards the wiring, not just the helper: a `wss://` dial through the public -//! API has to resolve its own crypto provider, in a test binary where nothing -//! has installed a process-wide one, and has to leave it uninstalled. - -use std::time::Duration; - -use futures_util::{sink, stream}; -use litellm_ai_gateway::io::responses_ws::async_responses_websocket; -use tokio::net::TcpListener; - -async fn dead_tls_server() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind a loopback port"); - let port = listener - .local_addr() - .expect("read the bound address") - .port(); - - tokio::spawn(async move { - while let Ok((stream, _peer)) = listener.accept().await { - drop(stream); - } - }); - - port -} - -#[tokio::test] -async fn dialing_wss_returns_an_error_instead_of_panicking() { - let port = dead_tls_server().await; - - let result = async_responses_websocket( - "gpt-5", - Some("test-key"), - Some(&format!("wss://127.0.0.1:{port}/")), - None, - Some(Duration::from_secs(10)), - |_| {}, - stream::empty(), - sink::drain(), - ) - .await; - - assert!( - result.is_err(), - "a plain TCP server cannot finish a TLS handshake" - ); - assert!( - rustls::crypto::CryptoProvider::get_default().is_none(), - "the dial settles its provider on its own connector, not process-wide" - ); -} diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml new file mode 100644 index 00000000000..d998b647960 --- /dev/null +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "litellm-auth-aws" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-auth.workspace = true + +moka = { workspace = true, features = ["sync"] } +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true + +aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] } +aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } +aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] } +aws-sigv4 = "1.5.1" +aws-types = "1.4.0" +aws-smithy-runtime-api = "1.13.0" + +[dev-dependencies] +reqwest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs new file mode 100644 index 00000000000..3b6b73bc6a9 --- /dev/null +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -0,0 +1,949 @@ +use std::collections::BTreeMap; +use std::sync::OnceLock; +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; + +use moka::sync::Cache; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use aws_credential_types::Credentials; +use aws_credential_types::provider::ProvideCredentials; +use aws_sigv4::http_request::{ + SignableBody, SignableRequest, SigningParams, SigningSettings, sign, +}; +use aws_sigv4::sign::v4; +use aws_smithy_runtime_api::client::identity::Identity; + +use super::Error; +use super::constants::{ + AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME, + AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, + AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, + BEDROCK_SERVICE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, + SIGV4_COMPUTED_HEADER_NAMES, +}; + +const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); +const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600); + +static STATIC_CREDENTIALS_CACHE: OnceLock> = OnceLock::new(); +static AMBIENT_CREDENTIALS_CACHE: OnceLock> = OnceLock::new(); + +fn credential_cache_ttl(flow: &AwsAuthFlow) -> Option { + match flow { + AwsAuthFlow::StaticKeys { .. } => Some(STATIC_CREDENTIALS_TTL), + AwsAuthFlow::DefaultChain => Some(AMBIENT_CREDENTIALS_TTL), + AwsAuthFlow::WebIdentity { .. } + | AwsAuthFlow::AssumeRole { .. } + | AwsAuthFlow::Profile { .. } + | AwsAuthFlow::SessionToken { .. } => None, + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AwsAuthConfig { + pub access_key_id: Option, + pub secret_access_key: Option, + pub session_token: Option, + pub region_name: Option, + pub session_name: Option, + pub profile_name: Option, + pub role_name: Option, + pub web_identity_token: Option, + pub sts_endpoint: Option, + pub external_id: Option, +} + +impl AwsAuthConfig { + fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Self { + Self { + access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)), + secret_access_key: self + .secret_access_key + .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)), + session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)), + region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)), + session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)), + profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)), + role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)), + web_identity_token: self + .web_identity_token + .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)), + sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)), + external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AwsAuthFlow { + WebIdentity { + token: String, + role: String, + session_name: String, + }, + AssumeRole { + role: String, + session_name: Option, + }, + Profile { + name: String, + }, + SessionToken { + access_key_id: String, + secret_access_key: String, + session_token: String, + }, + StaticKeys { + access_key_id: String, + secret_access_key: String, + region_name: String, + }, + DefaultChain, +} + +fn cache_key(config: &AwsAuthConfig, flow: &AwsAuthFlow) -> String { + let mut hasher = Sha256::new(); + hasher.update(format!("{config:?}:{flow:?}")); + format!("{:x}", hasher.finalize()) +} + +fn static_credentials_cache() -> &'static Cache { + STATIC_CREDENTIALS_CACHE.get_or_init(|| { + Cache::builder() + .max_capacity(200) + .time_to_live(STATIC_CREDENTIALS_TTL) + .build() + }) +} + +fn ambient_credentials_cache() -> &'static Cache { + AMBIENT_CREDENTIALS_CACHE.get_or_init(|| { + Cache::builder() + .max_capacity(200) + .time_to_live(AMBIENT_CREDENTIALS_TTL) + .build() + }) +} + +fn get_cached_credentials(key: &str) -> Option { + static_credentials_cache() + .get(key) + .or_else(|| ambient_credentials_cache().get(key)) +} + +fn set_cached_credentials(key: String, credentials: Credentials, ttl: Duration) { + if ttl == STATIC_CREDENTIALS_TTL { + static_credentials_cache().insert(key, credentials); + } else { + ambient_credentials_cache().insert(key, credentials); + } +} + +fn role_identity(arn: &str) -> Option<(&str, &str, &str)> { + let mut parts = arn.splitn(6, ':'); + let ("arn", partition, _, _, account, resource) = ( + parts.next()?, + parts.next()?, + parts.next()?, + parts.next()?, + parts.next()?, + parts.next()?, + ) else { + return None; + }; + let role = if let Some(role) = resource.strip_prefix("role/") { + role.rsplit('/').next()? + } else { + resource.strip_prefix("assumed-role/")?.split('/').next()? + }; + Some((partition, account, role)) +} + +fn same_role_arns(target: &str, caller: &str) -> bool { + role_identity(target) == role_identity(caller) +} + +pub fn classify_auth( + config: AwsAuthConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> AwsAuthFlow { + let config = config.with_environment(env_lookup); + if let (Some(token), Some(role), Some(session_name)) = ( + config.web_identity_token.clone(), + config.role_name.clone(), + config.session_name.clone(), + ) { + return AwsAuthFlow::WebIdentity { + token, + role, + session_name, + }; + } + if let Some(role) = config.role_name.clone() { + return AwsAuthFlow::AssumeRole { + role, + session_name: config.session_name.clone(), + }; + } + if let Some(name) = config.profile_name { + return AwsAuthFlow::Profile { name }; + } + if let (Some(access_key_id), Some(secret_access_key), Some(session_token)) = ( + config.access_key_id.clone(), + config.secret_access_key.clone(), + config.session_token, + ) { + return AwsAuthFlow::SessionToken { + access_key_id, + secret_access_key, + session_token, + }; + } + if let (Some(access_key_id), Some(secret_access_key), Some(region_name)) = ( + config.access_key_id, + config.secret_access_key, + config.region_name, + ) { + return AwsAuthFlow::StaticKeys { + access_key_id, + secret_access_key, + region_name, + }; + } + AwsAuthFlow::DefaultChain +} + +pub async fn resolve_credentials( + config: AwsAuthConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result { + let resolved = config.clone().with_environment(env_lookup); + let flow = classify_auth(config, env_lookup); + match flow { + AwsAuthFlow::SessionToken { + access_key_id, + secret_access_key, + session_token, + } => Ok(Credentials::new( + access_key_id, + secret_access_key, + Some(session_token), + None, + "litellm-static-session", + )), + AwsAuthFlow::StaticKeys { + access_key_id, + secret_access_key, + region_name, + } => { + let flow = AwsAuthFlow::StaticKeys { + access_key_id: access_key_id.clone(), + secret_access_key: secret_access_key.clone(), + region_name, + }; + let key = cache_key(&resolved, &flow); + if let Some(credentials) = get_cached_credentials(&key) { + return Ok(credentials); + } + let credentials = Credentials::new( + access_key_id, + secret_access_key, + None, + None, + "litellm-static", + ); + set_cached_credentials( + key, + credentials.clone(), + credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL), + ); + Ok(credentials) + } + AwsAuthFlow::Profile { name } => { + let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() + .profile_name(name) + .build(); + provider + .provide_credentials() + .await + .map_err(|error| Error::AwsProfile(error.to_string())) + } + AwsAuthFlow::AssumeRole { role, session_name } => { + if is_already_running_as_role(&role, &resolved).await? { + let ambient_flow = AwsAuthFlow::DefaultChain; + let key = cache_key(&resolved, &ambient_flow); + if let Some(credentials) = get_cached_credentials(&key) { + return Ok(credentials); + } + let provider = + aws_config::default_provider::credentials::DefaultCredentialsChain::builder() + .build() + .await; + let credentials = provider + .provide_credentials() + .await + .map_err(|error| Error::AwsDefaultChain(error.to_string()))?; + set_cached_credentials( + key, + credentials.clone(), + credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL), + ); + return Ok(credentials); + } + let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + if let Some(region) = resolved.region_name.clone() { + loader = loader.region(aws_types::region::Region::new(region)); + } + if let Some(endpoint) = resolved.sts_endpoint.clone() { + loader = loader.endpoint_url(endpoint); + } + if let (Some(access_key_id), Some(secret_access_key)) = + (resolved.access_key_id, resolved.secret_access_key) + { + loader = loader.credentials_provider(Credentials::new( + access_key_id, + secret_access_key, + resolved.session_token, + None, + "litellm-role-source", + )); + } + let sdk_config = loader.load().await; + let builder = aws_config::sts::AssumeRoleProvider::builder(role); + let builder = match session_name { + Some(name) => builder.session_name(name), + None => builder.session_name(default_session_name()), + }; + let builder = match resolved.external_id { + Some(id) => builder.external_id(id), + None => builder, + }; + let provider = builder.configure(&sdk_config).build().await; + provider + .provide_credentials() + .await + .map_err(|error| Error::AwsAssumeRole(error.to_string())) + } + AwsAuthFlow::WebIdentity { + token, + role, + session_name, + } => { + let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + if let Some(region) = resolved.region_name { + loader = loader.region(aws_types::region::Region::new(region)); + } + if let Some(endpoint) = resolved.sts_endpoint { + loader = loader.endpoint_url(endpoint); + } + let sdk_config = loader.load().await; + let client = aws_sdk_sts::Client::new(&sdk_config); + let response = client + .assume_role_with_web_identity() + .role_arn(role) + .role_session_name(session_name) + .web_identity_token(token) + .send() + .await + .map_err(|error| Error::AwsWebIdentity(error.to_string()))?; + let credentials = response + .credentials() + .ok_or(Error::AwsMissingWebIdentityCredentials)?; + let expiration = SystemTime::try_from(*credentials.expiration()) + .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?; + Ok(Credentials::new( + credentials.access_key_id(), + credentials.secret_access_key(), + Some(credentials.session_token().to_string()), + Some(expiration), + "litellm-web-identity", + )) + } + AwsAuthFlow::DefaultChain => { + let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain); + if let Some(credentials) = get_cached_credentials(&key) { + return Ok(credentials); + } + let provider = + aws_config::default_provider::credentials::DefaultCredentialsChain::builder() + .build() + .await; + let credentials = provider + .provide_credentials() + .await + .map_err(|error| Error::AwsDefaultChain(error.to_string()))?; + set_cached_credentials( + key, + credentials.clone(), + credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL), + ); + Ok(credentials) + } + } +} + +async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { + if role_identity(role).is_none() { + return Ok(false); + } + if let (Ok(current_role), Ok(token_file)) = ( + std::env::var(AWS_ROLE_ARN), + std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE), + ) && !token_file.is_empty() + { + return Ok(same_role_arns(role, ¤t_role)); + } + + let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + if let Some(region) = config.region_name.clone() { + loader = loader.region(aws_types::region::Region::new(region)); + } + if let Some(endpoint) = config.sts_endpoint.clone() { + loader = loader.endpoint_url(endpoint); + } + let sdk_config = loader.load().await; + let response = match aws_sdk_sts::Client::new(&sdk_config) + .get_caller_identity() + .send() + .await + { + Ok(response) => response, + Err(_) => return Ok(false), + }; + Ok(response + .arn() + .is_some_and(|caller| same_role_arns(role, caller))) +} + +fn default_session_name() -> String { + let seconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()); + format!("{DEFAULT_SESSION_NAME_PREFIX}-{seconds}") +} + +/// The subset of `headers` SigV4 should cover. +/// +/// Python signs only these and reattaches the rest afterwards, so a forwarded +/// client header cannot change the canonical request and invalidate the +/// signature. Signing everything instead makes the request 403 on a header the +/// caller supplied, on a deployment that works on the Python path. +pub fn aws_signature_headers(headers: &BTreeMap) -> BTreeMap { + headers + .iter() + .filter(|(name, _)| { + let name = name.to_ascii_lowercase(); + AWS_SIGNED_HEADER_NAMES.contains(&name.as_str()) + || name.starts_with("x-amz-") + || name.starts_with("x-amzn-") + }) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +/// Whether the signer produces `name` itself. +/// +/// Python's reattach loop skips these, so a caller-supplied copy never reaches +/// the wire next to the computed one. +pub fn is_sigv4_computed_header(name: &str) -> bool { + SIGV4_COMPUTED_HEADER_NAMES.contains(&name.to_ascii_lowercase().as_str()) +} + +pub fn sign_bedrock_post( + url: &str, + body: &[u8], + headers: &BTreeMap, + region: &str, + credentials: &Credentials, + signing_time: SystemTime, +) -> Result, Error> { + let identity: Identity = credentials.clone().into(); + let params = v4::SigningParams::builder() + .identity(&identity) + .region(region) + .name(BEDROCK_SERVICE) + .time(signing_time) + .settings(SigningSettings::default()) + .build() + .map(SigningParams::from) + .map_err(|error| Error::AwsSigningParameters(error.to_string()))?; + let header_refs = headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_str())); + let request = SignableRequest::new("POST", url, header_refs, SignableBody::Bytes(body)) + .map_err(|error| Error::AwsSignableRequest(error.to_string()))?; + let (instructions, _) = sign(request, ¶ms) + .map_err(|error| Error::AwsSigning(error.to_string()))? + .into_parts(); + Ok(instructions + .headers() + .map(|(name, value)| { + let normalized_name = match name { + "authorization" => "Authorization", + "x-amz-date" => "X-Amz-Date", + "x-amz-security-token" => "X-Amz-Security-Token", + _ => name, + }; + (normalized_name.to_string(), value.to_string()) + }) + .collect()) +} + +/// Model-id and region parsing shared by every Bedrock route. +pub fn bedrock_model_id_and_region(model: &str) -> (String, Option) { + let mut stripped = model; + for prefix in ["bedrock/converse/", "bedrock/", "converse/"] { + if let Some(value) = stripped.strip_prefix(prefix) { + stripped = value; + break; + } + } + let mut region = None; + if let Some((candidate, remainder)) = stripped.split_once('/') + && is_bedrock_region(candidate) + { + region = Some(candidate.to_string()); + stripped = remainder; + } + for prefix in ["nova-2/", "nova/"] { + if let Some(value) = stripped.strip_prefix(prefix) { + stripped = value; + break; + } + } + if region.is_none() { + // Python splits the whole ARN and takes field 3, the region. Stripping + // `arn:` first shifts every field down one, so the region is field 2 + // here; field 3 is the account id. + region = stripped + .strip_prefix("arn:") + .and_then(|value| value.split(':').nth(2)) + .filter(|value| !value.is_empty()) + .map(str::to_string); + } + (stripped.to_string(), region) +} + +fn is_bedrock_region(value: &str) -> bool { + value.len() > 3 + && value.contains('-') + && value + .chars() + .all(|char| char.is_ascii_alphanumeric() || char == '-') +} + +pub fn resolve_bedrock_region( + model_region: Option<&str>, + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, +) -> String { + if let Some(region) = optional_params + .get("aws_region_name") + .and_then(Value::as_str) + { + return region.to_string(); + } + if let Some(region) = model_region { + return region.to_string(); + } + env_lookup(AWS_REGION_NAME) + .or_else(|| env_lookup(AWS_REGION)) + .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string()) +} + +pub fn aws_auth_config( + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, +) -> AwsAuthConfig { + let value = |key: &str| { + optional_params + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + }; + let env = |key: &str| env_lookup(key); + AwsAuthConfig { + access_key_id: value("aws_access_key_id").or_else(|| env("AWS_ACCESS_KEY_ID")), + secret_access_key: value("aws_secret_access_key").or_else(|| env("AWS_SECRET_ACCESS_KEY")), + session_token: value("aws_session_token").or_else(|| env("AWS_SESSION_TOKEN")), + region_name: value("aws_region_name").or_else(|| env(AWS_REGION_NAME)), + session_name: value("aws_session_name").or_else(|| env("AWS_SESSION_NAME")), + profile_name: value("aws_profile_name").or_else(|| env("AWS_PROFILE_NAME")), + role_name: value("aws_role_name").or_else(|| env("AWS_ROLE_NAME")), + web_identity_token: value("aws_web_identity_token") + .or_else(|| env("AWS_WEB_IDENTITY_TOKEN")), + sts_endpoint: value("aws_sts_endpoint").or_else(|| env("AWS_STS_ENDPOINT")), + external_id: value("aws_external_id").or_else(|| env("AWS_EXTERNAL_ID")), + } +} + +/// Credentials a host resolved through its own chain and handed down verbatim. +/// +/// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads +/// profiles, STS and boto sessions) passes the result here so the core signs +/// with exactly those. Without this the core would re-derive from ambient +/// state, where an unrelated `AWS_ROLE_NAME` or `AWS_PROFILE_NAME` in the +/// environment outranks explicit keys in [`classify_auth`] and the two sides +/// would sign as different principals. +pub fn host_supplied_credentials(optional_params: &Map) -> Option { + let value = |key: &str| { + optional_params + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + }; + let access_key_id = value("aws_access_key_id")?; + let secret_access_key = value("aws_secret_access_key")?; + Some(Credentials::new( + access_key_id, + secret_access_key, + value("aws_session_token").map(str::to_string), + None, + "litellm-host-supplied", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn no_env(_: &str) -> Option { + None + } + + fn parity_inputs() -> (String, Vec, BTreeMap) { + ( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke" + .to_string(), + br#"{"input":"hello"}"#.to_vec(), + BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())]), + ) + } + + #[test] + fn reads_the_region_field_of_a_model_arn_not_the_account_id() { + // Python's `_get_aws_region_from_model_arn` splits the whole ARN and + // takes field 3. Stripping `arn:` first shifts every field down one, so + // the region is field 2 here. Taking field 3 after the strip returns + // the account id, which is not a region at all. + let (_, region) = bedrock_model_id_and_region( + "bedrock/arn:aws:bedrock:us-west-2:123456789012:foundation-model/anthropic.claude-v2", + ); + assert_eq!(region.as_deref(), Some("us-west-2")); + } + + #[test] + fn classification_preserves_python_precedence() { + let config = AwsAuthConfig { + access_key_id: Some("ak".into()), + secret_access_key: Some("sk".into()), + session_token: Some("token".into()), + region_name: Some("us-east-1".into()), + session_name: Some("session".into()), + profile_name: Some("profile".into()), + role_name: Some("role".into()), + web_identity_token: Some("oidc".into()), + ..Default::default() + }; + assert!(matches!( + classify_auth(config, &no_env), + AwsAuthFlow::WebIdentity { .. } + )); + } + + #[test] + fn classification_covers_fallthroughs() { + let env = |key: &str| match key { + AWS_PROFILE_NAME => Some("profile".into()), + _ => None, + }; + assert!(matches!( + classify_auth(AwsAuthConfig::default(), &env), + AwsAuthFlow::Profile { .. } + )); + assert!(matches!( + classify_auth( + AwsAuthConfig { + access_key_id: Some("ak".into()), + secret_access_key: Some("sk".into()), + session_token: Some("token".into()), + ..Default::default() + }, + &no_env + ), + AwsAuthFlow::SessionToken { .. } + )); + assert!(matches!( + classify_auth( + AwsAuthConfig { + access_key_id: Some("ak".into()), + secret_access_key: Some("sk".into()), + region_name: Some("us-east-1".into()), + ..Default::default() + }, + &no_env + ), + AwsAuthFlow::StaticKeys { .. } + )); + assert_eq!( + classify_auth(AwsAuthConfig::default(), &no_env), + AwsAuthFlow::DefaultChain + ); + } + + #[tokio::test] + async fn static_credentials_do_not_use_network() { + let credentials = resolve_credentials( + AwsAuthConfig { + access_key_id: Some("ak".into()), + secret_access_key: Some("sk".into()), + region_name: Some("us-east-1".into()), + ..Default::default() + }, + &no_env, + ) + .await + .expect("static credentials"); + assert_eq!(credentials.access_key_id(), "ak"); + assert_eq!(credentials.session_token(), None); + } + + #[test] + fn cache_policy_matches_python_flows() { + assert_eq!( + credential_cache_ttl(&AwsAuthFlow::StaticKeys { + access_key_id: "ak".into(), + secret_access_key: "sk".into(), + region_name: "us-east-1".into(), + }), + Some(STATIC_CREDENTIALS_TTL) + ); + assert_eq!( + credential_cache_ttl(&AwsAuthFlow::DefaultChain), + Some(AMBIENT_CREDENTIALS_TTL) + ); + assert_eq!( + credential_cache_ttl(&AwsAuthFlow::SessionToken { + access_key_id: "ak".into(), + secret_access_key: "sk".into(), + session_token: "token".into(), + }), + None + ); + assert_eq!( + credential_cache_ttl(&AwsAuthFlow::Profile { + name: "profile".into() + }), + None + ); + assert_eq!( + credential_cache_ttl(&AwsAuthFlow::AssumeRole { + role: "arn:aws:iam::123456789012:role/demo".into(), + session_name: None, + }), + None + ); + assert_eq!( + credential_cache_ttl(&AwsAuthFlow::WebIdentity { + token: "token".into(), + role: "arn:aws:iam::123456789012:role/demo".into(), + session_name: "session".into(), + }), + None + ); + } + + #[test] + fn cache_round_trip_preserves_credentials() { + let key = format!("cache-test-{}", std::process::id()); + let credentials = Credentials::new("cache-ak", "cache-sk", None, None, "test"); + set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL); + assert_eq!( + get_cached_credentials(&key).map(|value| value.access_key_id().to_string()), + Some("cache-ak".to_string()) + ); + } + + #[test] + fn same_role_comparison_matches_partition_account_and_role() { + assert!(same_role_arns( + "arn:aws:iam::123456789012:role/path/demo", + "arn:aws:sts::123456789012:assumed-role/demo/session" + )); + assert!(!same_role_arns( + "arn:aws:iam::123456789012:role/demo", + "arn:aws:iam::999999999999:role/demo" + )); + assert!(!same_role_arns( + "arn:aws:iam::123456789012:role/demo", + "arn:aws-cn:iam::123456789012:role/demo" + )); + assert!(!same_role_arns( + "arn:aws:iam::123456789012:user/demo", + "arn:aws:iam::123456789012:role/demo" + )); + } + + #[test] + fn a_forwarded_client_header_is_not_folded_into_the_signature() { + // Python signs only the AWS header set, so a header a caller forwarded + // cannot change the canonical request. Signing it instead makes the + // request 403 the moment anything on the wire rewrites or drops it. + let (url, body, mut headers) = parity_inputs(); + headers.insert("x-request-id".to_string(), "abc-123".to_string()); + headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); + headers.insert("x-amzn-trace-id".to_string(), "Root=1-abc".to_string()); + let signable = aws_signature_headers(&headers); + + assert!(!signable.contains_key("x-request-id")); + assert!(!signable.contains_key("Accept-Encoding")); + // The AWS-prefixed one is genuinely part of the signature. + assert!(signable.contains_key("x-amzn-trace-id")); + assert!(signable.contains_key("Content-Type")); + + let credentials = Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None, + None, + "test", + ); + let signed = sign_bedrock_post( + &url, + &body, + &signable, + "us-east-1", + &credentials, + SystemTime::UNIX_EPOCH, + ) + .expect("signs"); + let authorization = signed + .get("Authorization") + .expect("carries an authorization header"); + assert!( + !authorization.contains("x-request-id"), + "forwarded header reached SignedHeaders: {authorization}" + ); + assert!( + !authorization.contains("accept-encoding"), + "forwarded header reached SignedHeaders: {authorization}" + ); + } + + #[test] + fn signing_matches_botocore_golden_vector() { + let (url, body, headers) = parity_inputs(); + let credentials = Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Some("session-token".to_string()), + None, + "test", + ); + let signed = sign_bedrock_post( + &url, + &body, + &headers, + "us-east-1", + &credentials, + UNIX_EPOCH + std::time::Duration::from_secs(1_704_164_645), + ) + .expect("golden signature"); + assert_eq!( + signed.get("X-Amz-Date").map(String::as_str), + Some("20240102T030405Z") + ); + assert_eq!( + signed.get("X-Amz-Security-Token").map(String::as_str), + Some("session-token") + ); + assert_eq!( + signed.get("Authorization").map(String::as_str), + Some( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20240102/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-security-token, Signature=55c027ef47527d3ad63f1735f9d099efdbc99f296ff914bd94e727e24ec0e464" + ) + ); + } + + #[test] + fn signing_without_session_token_omits_security_header() { + let (url, body, headers) = parity_inputs(); + let credentials = Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None, + None, + "test", + ); + let signed = sign_bedrock_post( + &url, + &body, + &headers, + "us-east-1", + &credentials, + UNIX_EPOCH + std::time::Duration::from_secs(1_704_164_645), + ) + .expect("signature"); + assert!(!signed.contains_key("X-Amz-Security-Token")); + } + + #[ignore] + #[tokio::test] + async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box> { + let access_key_id = std::env::var("AWS_BEDROCK_TEST_ACCESS_KEY_ID")?; + let secret_access_key = std::env::var("AWS_BEDROCK_TEST_SECRET_ACCESS_KEY")?; + let body = br#"{"anthropic_version":"bedrock-2023-05-31","max_tokens":1,"messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}"#.to_vec(); + let headers = + BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())]); + let credentials = resolve_credentials( + AwsAuthConfig { + access_key_id: Some(access_key_id), + secret_access_key: Some(secret_access_key), + region_name: Some("us-west-2".to_string()), + ..Default::default() + }, + &no_env, + ) + .await?; + let client = reqwest::Client::new(); + let mut failures = Vec::new(); + + for region in ["us-west-2", "us-east-1"] { + let url = format!( + "https://bedrock-runtime.{region}.amazonaws.com/model/us.anthropic.claude-opus-4-8/invoke" + ); + let signed_headers = sign_bedrock_post( + &url, + &body, + &headers, + region, + &credentials, + SystemTime::now(), + )?; + let mut request = client.post(&url).body(body.clone()); + for (name, value) in &headers { + request = request.header(name, value); + } + for (name, value) in signed_headers { + request = request.header(name, value); + } + let response = request.send().await?; + let status = response.status(); + let response_body = response.text().await?; + let snippet: String = response_body.chars().take(240).collect(); + println!("region={region} status={status} response={snippet}"); + if status == reqwest::StatusCode::OK { + return Ok(()); + } + failures.push(format!("{region}: {status} {snippet}")); + } + + panic!( + "no Bedrock region returned HTTP 200: {}", + failures.join("; ") + ); + } +} diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs new file mode 100644 index 00000000000..be215cc9016 --- /dev/null +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -0,0 +1,43 @@ +pub const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; +pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; +pub const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; +pub const AWS_REGION_NAME: &str = "AWS_REGION_NAME"; +pub const AWS_REGION: &str = "AWS_REGION"; +pub const AWS_SESSION_NAME: &str = "AWS_SESSION_NAME"; +pub const AWS_PROFILE_NAME: &str = "AWS_PROFILE_NAME"; +pub const AWS_ROLE_NAME: &str = "AWS_ROLE_NAME"; +pub const AWS_WEB_IDENTITY_TOKEN: &str = "AWS_WEB_IDENTITY_TOKEN"; +pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN"; +pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; +pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT"; +pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID"; +pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; + +/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors +/// Python's `_filter_headers_for_aws_signature` allowlist. +pub const AWS_SIGNED_HEADER_NAMES: &[&str] = &[ + "host", + "content-type", + "date", + "x-amz-date", + "x-amz-security-token", + "x-amz-content-sha256", + "x-amz-algorithm", + "x-amz-credential", + "x-amz-signedheaders", + "x-amz-signature", +]; +/// Headers the signer emits itself. Mirrors Python's `SIGV4_COMPUTED_HEADERS`, +/// which the reattach loop skips so a caller's copy cannot ride alongside the +/// computed one. +pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[ + "authorization", + "x-amz-date", + "x-amz-security-token", + "date", +]; +pub const BEDROCK_SERVICE: &str = "bedrock"; +pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session"; +pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2"; +pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str = + "https://bedrock-runtime.{region}.amazonaws.com"; diff --git a/litellm-rust/crates/auth-aws/src/error.rs b/litellm-rust/crates/auth-aws/src/error.rs new file mode 100644 index 00000000000..f80fbce456e --- /dev/null +++ b/litellm-rust/crates/auth-aws/src/error.rs @@ -0,0 +1,46 @@ +use thiserror::Error as ThisError; + +#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +pub enum Error { + #[error("AWS profile credentials failed: {0}")] + AwsProfile(String), + #[error("AWS default credentials failed: {0}")] + AwsDefaultChain(String), + #[error("AWS role credentials failed: {0}")] + AwsAssumeRole(String), + #[error("AWS web identity credentials failed: {0}")] + AwsWebIdentity(String), + #[error("AWS web identity expiration was invalid: {0}")] + AwsWebIdentityExpiration(String), + #[error("AWS signing parameters failed: {0}")] + AwsSigningParameters(String), + #[error("AWS signable request failed: {0}")] + AwsSignableRequest(String), + #[error("AWS request signing failed: {0}")] + AwsSigning(String), + #[error("AWS web identity response had no credentials")] + AwsMissingWebIdentityCredentials, +} + +impl From for litellm_auth::Error { + fn from(error: Error) -> Self { + Self::ProviderAuthentication(error.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::Error; + + #[test] + fn converts_to_shared_auth_error_without_losing_context() { + let error = litellm_auth::Error::from(Error::AwsProfile("profile not found".into())); + + assert_eq!( + error, + litellm_auth::Error::ProviderAuthentication( + "AWS profile credentials failed: profile not found".into() + ) + ); + } +} diff --git a/litellm-rust/crates/auth-aws/src/lib.rs b/litellm-rust/crates/auth-aws/src/lib.rs new file mode 100644 index 00000000000..264592ccb2e --- /dev/null +++ b/litellm-rust/crates/auth-aws/src/lib.rs @@ -0,0 +1,6 @@ +mod aws; +pub mod constants; +mod error; + +pub use aws::*; +pub use error::Error; diff --git a/litellm-rust/crates/auth-azure/Cargo.toml b/litellm-rust/crates/auth-azure/Cargo.toml new file mode 100644 index 00000000000..9f8260c7b3f --- /dev/null +++ b/litellm-rust/crates/auth-azure/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "litellm-auth-azure" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-auth.workspace = true + +moka.workspace = true +serde_json.workspace = true +sha2.workspace = true +strum.workspace = true +url.workspace = true + +azure_core = "1.0.0" +azure_identity = { version = "1.0.0", features = ["tokio"] } + +[dev-dependencies] +tokio.workspace = true diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/credential_provider_cache.rs b/litellm-rust/crates/auth-azure/src/credential_provider_cache.rs similarity index 87% rename from litellm-rust/crates/core/src/providers/azure_ai/auth/credential_provider_cache.rs rename to litellm-rust/crates/auth-azure/src/credential_provider_cache.rs index 297e4cc6502..ab9ffc719df 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/auth/credential_provider_cache.rs +++ b/litellm-rust/crates/auth-azure/src/credential_provider_cache.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use azure_core::credentials::TokenCredential; use moka::future::Cache; -use crate::AuthError; +use litellm_auth::Error; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(crate) struct AzureCredentialProviderCacheKey { @@ -31,9 +31,9 @@ impl AzureCredentialProviderCache { &self, key: AzureCredentialProviderCacheKey, create: F, - ) -> Result, AuthError> + ) -> Result, Error> where - F: Future, AuthError>>, + F: Future, Error>>, { self.entries .try_get_with(key, create) diff --git a/litellm-rust/crates/auth-azure/src/lib.rs b/litellm-rust/crates/auth-azure/src/lib.rs new file mode 100644 index 00000000000..e76227d6aa2 --- /dev/null +++ b/litellm-rust/crates/auth-azure/src/lib.rs @@ -0,0 +1,7 @@ +mod credential_provider_cache; +mod native; +mod resolve; +mod types; + +pub use resolve::AzureAuthService; +pub use types::AzureAuthInputs; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/native.rs b/litellm-rust/crates/auth-azure/src/native.rs similarity index 94% rename from litellm-rust/crates/core/src/providers/azure_ai/auth/native.rs rename to litellm-rust/crates/auth-azure/src/native.rs index b8f19818d16..5f913a8ad01 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/auth/native.rs +++ b/litellm-rust/crates/auth-azure/src/native.rs @@ -1,4 +1,3 @@ -use crate::auth::error::AuthConfigurationError; use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; @@ -13,8 +12,8 @@ use azure_identity::{ }; use sha2::{Digest, Sha256}; -use crate::AuthError; -use crate::auth::{InputSource, ResolvedCredential, SecretValue, Sourced}; +use litellm_auth::Error; +use litellm_auth::{InputSource, ResolvedCredential, SecretValue, Sourced}; use super::credential_provider_cache::{ AzureCredentialProviderCache, AzureCredentialProviderCacheKey, @@ -62,7 +61,7 @@ pub(crate) struct ValidatedAzureRequest { } impl ValidatedAzureRequest { - pub(crate) fn new(request: NativeAzureRequest) -> Result { + pub(crate) fn new(request: NativeAzureRequest) -> Result { validate_authority(&request)?; let credential_source = validate_sources(&request)?; Ok(Self { @@ -120,7 +119,7 @@ impl NativeAzureTokenAcquirer { pub(crate) async fn acquire( &self, request: ValidatedAzureRequest, - ) -> Result { + ) -> Result { let scope = request.request.scope().to_string(); let key = request.request.cache_key(); let transport = self.transport.clone(); @@ -134,7 +133,7 @@ impl NativeAzureTokenAcquirer { let token = credential .get_token(&[scope.as_str()], None) .await - .map_err(|error| AuthError::AzureTokenAcquisition(error.to_string()))?; + .map_err(|error| Error::AzureTokenAcquisition(error.to_string()))?; let expires_on = u64::try_from(token.expires_on.unix_timestamp()) .ok() .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds)); @@ -239,7 +238,7 @@ impl NativeAzureRequest { } } -fn validate_authority(request: &NativeAzureRequest) -> Result<(), AuthError> { +fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> { let authority = match request { NativeAzureRequest::ClientSecret { authority, .. } | NativeAzureRequest::ClientAssertion { authority, .. } @@ -251,8 +250,7 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), AuthError> { let Some(authority) = authority else { return Ok(()); }; - let url = url::Url::parse(authority.value()) - .map_err(|_| AuthError::Configuration(AuthConfigurationError::InvalidAzureAuthority))?; + let url = url::Url::parse(authority.value()).map_err(|_| Error::InvalidAzureAuthority)?; if url.scheme() != "https" || url.host_str().is_none() || !url.username().is_empty() @@ -261,14 +259,12 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), AuthError> { || url.fragment().is_some() || !matches!(url.path(), "" | "/") { - return Err(AuthError::Configuration( - AuthConfigurationError::InvalidAzureAuthority, - )); + return Err(Error::InvalidAzureAuthority); } Ok(()) } -fn validate_sources(request: &NativeAzureRequest) -> Result { +fn validate_sources(request: &NativeAzureRequest) -> Result { match request { NativeAzureRequest::ClientSecret { tenant_id, @@ -356,7 +352,7 @@ fn is_request_controlled(value: &Sourced, optional: Option<&Sourced Result { +fn trusted_only(sources: &[InputSource]) -> Result { if sources.contains(&InputSource::Request) { return mixed_sources(); } @@ -371,16 +367,14 @@ fn trusted_source(sources: &[InputSource]) -> InputSource { } } -fn mixed_sources() -> Result { - Err(AuthError::Configuration( - AuthConfigurationError::MixedAzureCredentialSources, - )) +fn mixed_sources() -> Result { + Err(Error::MixedAzureCredentialSources) } fn build_credential( request: NativeAzureRequest, transport: Option, -) -> Result, AuthError> { +) -> Result, Error> { match request { NativeAzureRequest::ClientSecret { tenant_id, @@ -439,11 +433,7 @@ fn build_credential( NativeAzureRequest::DeveloperTools { .. } => DeveloperToolsCredential::new(None) .map(|credential| credential as Arc), } - .map_err(|error| { - AuthError::Configuration(AuthConfigurationError::AzureCredentialInitialization( - error.to_string(), - )) - }) + .map_err(|error| Error::AzureCredentialInitialization(error.to_string())) } fn client_options( @@ -494,7 +484,7 @@ mod tests { use azure_core::{Bytes, Result}; use super::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest}; - use crate::auth::{InputSource, SecretValue, Sourced}; + use litellm_auth::{InputSource, SecretValue, Sourced}; fn deployment(value: T) -> Sourced { Sourced::new(value, InputSource::Deployment) @@ -659,9 +649,7 @@ mod tests { assert!(matches!( error, - crate::AuthError::Configuration( - crate::auth::error::AuthConfigurationError::MixedAzureCredentialSources - ) + litellm_auth::Error::MixedAzureCredentialSources )); } @@ -691,12 +679,7 @@ mod tests { authority, )) .unwrap_err(); - assert!(matches!( - error, - crate::AuthError::Configuration( - crate::auth::error::AuthConfigurationError::InvalidAzureAuthority - ) - )); + assert!(matches!(error, litellm_auth::Error::InvalidAzureAuthority)); } } } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs similarity index 89% rename from litellm-rust/crates/core/src/providers/azure_ai/auth/resolve.rs rename to litellm-rust/crates/auth-azure/src/resolve.rs index 025dd4f8740..660a95b79d8 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/auth/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -1,6 +1,5 @@ -use crate::AuthError; -use crate::auth::error::AuthConfigurationError; -use crate::auth::{ +use litellm_auth::Error; +use litellm_auth::{ CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, SecretValue, Sourced, TokenProviderHandle, }; @@ -37,7 +36,7 @@ pub(crate) enum AzureCredentialPlan { } /// Rust counterpart to Python's `get_azure_ad_token`, not `BaseAzureLLM`. -pub(crate) struct AzureAuthService { +pub struct AzureAuthService { native: Arc, } @@ -45,14 +44,14 @@ trait AzureTokenAcquirer: Send + Sync { fn acquire( &self, request: ValidatedAzureRequest, - ) -> Pin> + Send + '_>>; + ) -> Pin> + Send + '_>>; } impl AzureTokenAcquirer for NativeAzureTokenAcquirer { fn acquire( &self, request: ValidatedAzureRequest, - ) -> Pin> + Send + '_>> { + ) -> Pin> + Send + '_>> { Box::pin(NativeAzureTokenAcquirer::acquire(self, request)) } } @@ -71,17 +70,17 @@ impl AzureAuthService { Self { native } } - pub(crate) async fn get_azure_ad_token( + pub async fn get_azure_ad_token( &self, inputs: &AzureAuthInputs, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result>, AuthError> { + ) -> Result>, Error> { match select_auth_plan(inputs, env_lookup)? { AzureCredentialPlan::Supplied(credential) => Ok(Some(credential)), AzureCredentialPlan::Caller(caller) => { let credential = caller.acquire().await?; if credential.secret().expose().is_empty() { - return Err(AuthError::EmptyAzureToken); + return Err(Error::EmptyAzureToken); } Ok(Some(Sourced::new(credential, InputSource::Deployment))) } @@ -94,7 +93,7 @@ impl AzureAuthService { } => { let assertion = resolve_reference(inputs, env_lookup, reference.value()) .await? - .ok_or(AuthError::UnresolvedOidcReference)?; + .ok_or(Error::UnresolvedOidcReference)?; let request = ValidatedAzureRequest::new(NativeAzureRequest::ClientAssertion { tenant_id, client_id, @@ -126,7 +125,7 @@ impl AzureAuthService { Err(error) => failures.push(error), } } - Err(AuthError::CredentialChain(failures)) + Err(Error::CredentialChain(failures)) } AzureCredentialPlan::Missing => Ok(None), } @@ -136,7 +135,7 @@ impl AzureAuthService { pub(crate) fn select_auth_plan( inputs: &AzureAuthInputs, env_lookup: &dyn Fn(&str) -> Option, -) -> Result { +) -> Result { let token = configured_secret(&inputs.azure_ad_token, AZURE_AD_TOKEN_ENV, env_lookup); let tenant_id = configured_string(&inputs.tenant_id, AZURE_TENANT_ID_ENV, env_lookup); let client_id = configured_string(&inputs.client_id, AZURE_CLIENT_ID_ENV, env_lookup); @@ -157,7 +156,7 @@ pub(crate) fn select_auth_plan( .map(|selector| Sourced::new(selector, value.source())) }) .transpose() - .map_err(|_| AuthError::Configuration(AuthConfigurationError::InvalidAzureSelector))?; + .map_err(|_| Error::InvalidAzureSelector)?; let federated_token_file = configured_string( &inputs.federated_token_file, AZURE_FEDERATED_TOKEN_FILE_ENV, @@ -229,7 +228,7 @@ fn select_native_plan( scope: Sourced, authority: Option>, refresh_source: InputSource, -) -> Result { +) -> Result { let selected = selector.unwrap_or_else(|| { Sourced::new( { @@ -247,9 +246,7 @@ fn select_native_plan( let selection_source = selected.source(); match selected.into_value() { - AzureCredentialType::ClientSecretCredential => Err(AuthError::Configuration( - AuthConfigurationError::MissingClientSecretFields, - )), + AzureCredentialType::ClientSecretCredential => Err(Error::MissingClientSecretFields), AzureCredentialType::WorkloadIdentityCredential => { Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( workload_request(tenant_id, client_id, federated_token_file, scope, authority)?, @@ -331,17 +328,11 @@ fn workload_request( token_file_path: Option>, scope: Sourced, authority: Option>, -) -> Result { +) -> Result { Ok(NativeAzureRequest::WorkloadIdentity { - tenant_id: tenant_id.ok_or(AuthError::Configuration( - AuthConfigurationError::MissingWorkloadTenant, - ))?, - client_id: client_id.ok_or(AuthError::Configuration( - AuthConfigurationError::MissingWorkloadClient, - ))?, - token_file_path: token_file_path.ok_or(AuthError::Configuration( - AuthConfigurationError::MissingWorkloadTokenFile, - ))?, + tenant_id: tenant_id.ok_or(Error::MissingWorkloadTenant)?, + client_id: client_id.ok_or(Error::MissingWorkloadClient)?, + token_file_path: token_file_path.ok_or(Error::MissingWorkloadTokenFile)?, scope, authority, }) @@ -383,7 +374,7 @@ async fn resolve_reference( inputs: &AzureAuthInputs, env_lookup: &(dyn Fn(&str) -> Option + Sync), reference: &CredentialRef, -) -> Result, AuthError> { +) -> Result, Error> { let lookup = match reference { CredentialRef::Explicit(secret) => return Ok(Some(secret.clone())), CredentialRef::Env(name) => env_lookup(name) @@ -395,9 +386,7 @@ async fn resolve_reference( let resolver = inputs .credential_resolver .as_ref() - .ok_or(AuthError::Configuration( - AuthConfigurationError::MissingHostResolver, - ))?; + .ok_or(Error::MissingHostResolver)?; resolver.resolve(reference).await? } }; @@ -409,15 +398,13 @@ async fn resolve_reference( fn oidc_reference( token: &Option>, -) -> Result>, AuthError> { +) -> Result>, Error> { let Some(token) = token.as_ref() else { return Ok(None); }; let value = token.value().expose(); if token.source() == InputSource::Request && value.starts_with("oidc/") { - return Err(AuthError::Configuration( - AuthConfigurationError::RequestAzureCredentialReference, - )); + return Err(Error::RequestAzureCredentialReference); } if let Some(name) = value.strip_prefix("oidc/env/") { return non_empty_reference(name, "OIDC environment reference") @@ -439,18 +426,14 @@ fn oidc_reference( ))); } if value.starts_with("oidc/") { - return Err(AuthError::Configuration( - AuthConfigurationError::UnsupportedOidcReference, - )); + return Err(Error::UnsupportedOidcReference); } Ok(None) } -fn non_empty_reference(value: &str, kind: &str) -> Result { +fn non_empty_reference(value: &str, kind: &str) -> Result { if value.is_empty() { - return Err(AuthError::Configuration( - AuthConfigurationError::EmptyReference(kind.to_string()), - )); + return Err(Error::EmptyReference(kind.to_string())); } Ok(value.to_string()) } @@ -466,14 +449,14 @@ mod tests { AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference, resolve_reference, select_auth_plan, }; - use crate::AuthError; - use crate::auth::ResolvedCredential; - use crate::auth::{ + use crate::native::ValidatedAzureRequest; + use crate::types::AzureAuthInputs; + use litellm_auth::Error; + use litellm_auth::ResolvedCredential; + use litellm_auth::{ CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialRef, CredentialResolver, CredentialResolverHandle, InputSource, SecretValue, Sourced, }; - use crate::providers::azure_ai::auth::native::ValidatedAzureRequest; - use crate::providers::azure_ai::auth::types::AzureAuthInputs; #[derive(Debug)] struct FileResolver; @@ -487,9 +470,8 @@ mod tests { fn acquire( &self, request: ValidatedAzureRequest, - ) -> std::pin::Pin< - Box> + Send + '_>, - > { + ) -> std::pin::Pin> + Send + '_>> + { let kind = request.kind(); self.requests.lock().unwrap().push(kind); Box::pin(async move { @@ -499,7 +481,7 @@ mod tests { expires_on: None, }) } else { - Err(AuthError::AzureTokenAcquisition(format!("{kind} failed"))) + Err(Error::AzureTokenAcquisition(format!("{kind} failed"))) } }) } @@ -612,12 +594,7 @@ mod tests { }) .unwrap_err(); - assert!(matches!( - error, - AuthError::Configuration( - crate::auth::error::AuthConfigurationError::RequestAzureCredentialReference - ) - )); + assert!(matches!(error, Error::RequestAzureCredentialReference)); } #[tokio::test] @@ -678,6 +655,6 @@ mod tests { .await .unwrap_err(); - assert!(matches!(error, AuthError::CredentialChain(errors) if errors.len() == 2)); + assert!(matches!(error, Error::CredentialChain(errors) if errors.len() == 2)); } } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/types.rs b/litellm-rust/crates/auth-azure/src/types.rs similarity index 93% rename from litellm-rust/crates/core/src/providers/azure_ai/auth/types.rs rename to litellm-rust/crates/auth-azure/src/types.rs index f15d526d945..2a510de1f43 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/auth/types.rs +++ b/litellm-rust/crates/auth-azure/src/types.rs @@ -1,10 +1,9 @@ -use crate::auth::error::AuthConfigurationError; use serde_json::{Map, Value}; use std::collections::BTreeMap; use strum::EnumString; -use crate::AuthError; -use crate::auth::{ +use litellm_auth::Error; +use litellm_auth::{ CredentialResolverHandle, InputSource, SecretValue, Sourced, TokenProviderHandle, }; @@ -54,14 +53,14 @@ pub struct AzureAuthInputs { impl AzureAuthInputs { #[cfg(test)] - pub fn from_optional_params(params: &Map) -> Result { + pub fn from_optional_params(params: &Map) -> Result { Self::from_sourced_optional_params(params, &BTreeMap::new()) } pub fn from_sourced_optional_params( params: &Map, sources: &BTreeMap, - ) -> Result { + ) -> Result { Ok(Self { azure_ad_token: secret_config(params, sources, "azure_ad_token")?, azure_ad_token_provider: None, @@ -88,15 +87,13 @@ fn string_config( params: &Map, sources: &BTreeMap, name: &str, -) -> Result, AuthError> { +) -> Result, Error> { let source = source_for(sources, name); match params.get(name) { None => Ok(ConfigValue::Absent), Some(Value::Null) => Ok(ConfigValue::ExplicitNone(source)), Some(Value::String(value)) => Ok(ConfigValue::Value(Sourced::new(value.clone(), source))), - Some(_) => Err(AuthError::Configuration( - AuthConfigurationError::InvalidFieldType(name.to_string()), - )), + Some(_) => Err(Error::InvalidFieldType(name.to_string())), } } @@ -104,7 +101,7 @@ fn secret_config( params: &Map, sources: &BTreeMap, name: &str, -) -> Result, AuthError> { +) -> Result, Error> { Ok(match string_config(params, sources, name)? { ConfigValue::Absent => ConfigValue::Absent, ConfigValue::ExplicitNone(source) => ConfigValue::ExplicitNone(source), @@ -123,7 +120,7 @@ mod tests { use std::collections::BTreeMap; use super::{AzureAuthInputs, AzureCredentialType, ConfigValue}; - use crate::auth::{InputSource, Sourced}; + use litellm_auth::{InputSource, Sourced}; #[test] fn selector_parsing_is_exact() { diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml new file mode 100644 index 00000000000..f24582db13e --- /dev/null +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "litellm-auth-gcp" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-auth.workspace = true + +moka.workspace = true +serde_json.workspace = true +sha2.workspace = true +tokio.workspace = true + +gcp_auth = "0.12.7" diff --git a/litellm-rust/crates/core/src/auth/vertex.rs b/litellm-rust/crates/auth-gcp/src/lib.rs similarity index 89% rename from litellm-rust/crates/core/src/auth/vertex.rs rename to litellm-rust/crates/auth-gcp/src/lib.rs index 00a0a7ea7ee..f8402624edc 100644 --- a/litellm-rust/crates/core/src/auth/vertex.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -9,9 +9,8 @@ use moka::future::Cache; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; -use crate::auth::error::AuthConfigurationError; -use crate::auth::http::apply_credential; -use crate::auth::{AuthError, CredentialPlacement, InputSource, SecretValue, Sourced}; +use litellm_auth::http::apply_credential; +use litellm_auth::{CredentialPlacement, Error, InputSource, SecretValue, Sourced}; const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform"; const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token"; @@ -24,17 +23,17 @@ const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION"; const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION"; #[derive(Clone, Debug, Default)] -pub(crate) struct VertexConfig { +pub struct VertexConfig { credentials: Option>, project_id: Option, location: Option, } impl VertexConfig { - pub(crate) fn from_sourced_optional_params( + pub fn from_sourced_optional_params( params: &Map, sources: &BTreeMap, - ) -> Result { + ) -> Result { Ok(Self { credentials: optional_credentials( params, @@ -46,16 +45,16 @@ impl VertexConfig { }) } - pub(crate) fn project_id(&self) -> Option<&str> { + pub fn project_id(&self) -> Option<&str> { self.project_id.as_deref() } - pub(crate) fn location(&self) -> Option<&str> { + pub fn location(&self) -> Option<&str> { self.location.as_deref() } } -pub(crate) struct VertexEnvironment { +pub struct VertexEnvironment { pub headers: Vec<(String, String)>, pub project_id: String, } @@ -65,7 +64,7 @@ struct VertexAccessToken { project_id: String, } -pub(crate) fn get_vertex_ai_project( +pub fn get_vertex_ai_project( config: &VertexConfig, env_lookup: &dyn Fn(&str) -> Option, ) -> Option { @@ -75,7 +74,7 @@ pub(crate) fn get_vertex_ai_project( .or_else(|| non_empty_env(env_lookup, VERTEXAI_PROJECT_ENV)) } -pub(crate) fn get_vertex_ai_location( +pub fn get_vertex_ai_location( config: &VertexConfig, env_lookup: &dyn Fn(&str) -> Option, ) -> Option { @@ -87,7 +86,7 @@ pub(crate) fn get_vertex_ai_location( } #[derive(Clone)] -pub(crate) struct VertexAuth { +pub struct VertexAuth { providers: Cache>, loader: Arc, } @@ -106,14 +105,13 @@ impl VertexAuth { } } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - pub(crate) async fn validate_environment( + pub async fn validate_environment( &self, headers: Vec<(String, String)>, api_key: Option<&str>, config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result { + ) -> Result { let has_authorization = headers .iter() .any(|(name, _)| name.eq_ignore_ascii_case("Authorization")); @@ -161,7 +159,7 @@ impl VertexAuth { &self, config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result { + ) -> Result { let provider = self.load_provider(config, env_lookup).await?; let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?; Ok(VertexAccessToken { token, project_id }) @@ -171,7 +169,7 @@ impl VertexAuth { &self, config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, AuthError> { + ) -> Result, Error> { let source = credential_source(config, env_lookup); let key = source.cache_key(); self.providers @@ -190,7 +188,7 @@ trait VertexProviderLoader: Send + Sync { fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc>; } -type VertexAuthFuture<'a, T> = Pin> + Send + 'a>>; +type VertexAuthFuture<'a, T> = Pin> + Send + 'a>>; struct GcpTokenSource(Arc); @@ -250,7 +248,7 @@ impl VertexProviderLoader for GcpProviderLoader { } } -fn validate_request_credentials(configured: &str) -> Result<&str, AuthError> { +fn validate_request_credentials(configured: &str) -> Result<&str, Error> { let token_uri = serde_json::from_str::(configured) .ok() .and_then(|credentials| { @@ -260,7 +258,7 @@ fn validate_request_credentials(configured: &str) -> Result<&str, AuthError> { .map(str::to_string) }); if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { - return Err(AuthConfigurationError::RequestVertexTokenEndpoint.into()); + return Err(Error::RequestVertexTokenEndpoint); } Ok(configured) } @@ -322,7 +320,7 @@ fn optional_credentials( params: &Map, sources: &BTreeMap, names: &[&str], -) -> Result>, AuthError> { +) -> Result>, Error> { for name in names { let source = source_for(sources, name); match params.get(*name) { @@ -337,17 +335,10 @@ fn optional_credentials( .map(SecretValue::new) .map(|value| Sourced::new(value, source)) .map(Some) - .map_err(|error| { - AuthError::Configuration(AuthConfigurationError::InvalidFieldType(format!( - "{}: {error}", - names[0] - ))) - }); + .map_err(|error| Error::InvalidFieldType(format!("{}: {error}", names[0]))); } Some(_) => { - return Err(AuthError::Configuration( - AuthConfigurationError::InvalidFieldType(names[0].to_string()), - )); + return Err(Error::InvalidFieldType(names[0].to_string())); } } } @@ -358,19 +349,14 @@ fn source_for(sources: &BTreeMap, name: &str) -> InputSourc sources.get(name).copied().unwrap_or_default() } -fn optional_string( - params: &Map, - names: &[&str], -) -> Result, AuthError> { +fn optional_string(params: &Map, names: &[&str]) -> Result, Error> { for name in names { match params.get(*name) { None | Some(Value::Null) => continue, Some(Value::String(value)) if value.trim().is_empty() => continue, Some(Value::String(value)) => return Ok(Some(value.clone())), Some(_) => { - return Err(AuthError::Configuration( - AuthConfigurationError::InvalidFieldType(names[0].to_string()), - )); + return Err(Error::InvalidFieldType(names[0].to_string())); } } } @@ -383,8 +369,8 @@ fn non_empty_env(env_lookup: &dyn Fn(&str) -> Option, name: &str) -> Opt .filter(|value| !value.is_empty()) } -fn auth_acquisition_error(error: gcp_auth::Error) -> AuthError { - AuthError::VertexTokenAcquisition(error.to_string()) +fn auth_acquisition_error(error: gcp_auth::Error) -> Error { + Error::VertexTokenAcquisition(error.to_string()) } #[cfg(test)] @@ -538,15 +524,11 @@ mod tests { ); assert!(matches!( validate_request_credentials(r#"{"token_uri":"http://127.0.0.1/token"}"#), - Err(AuthError::Configuration( - AuthConfigurationError::RequestVertexTokenEndpoint - )) + Err(Error::RequestVertexTokenEndpoint) )); assert!(matches!( validate_request_credentials("{}"), - Err(AuthError::Configuration( - AuthConfigurationError::RequestVertexTokenEndpoint - )) + Err(Error::RequestVertexTokenEndpoint) )); } diff --git a/litellm-rust/crates/auth/Cargo.toml b/litellm-rust/crates/auth/Cargo.toml new file mode 100644 index 00000000000..128a05c1a25 --- /dev/null +++ b/litellm-rust/crates/auth/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-auth" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +subtle.workspace = true +thiserror.workspace = true +veil.workspace = true + +[dev-dependencies] +tokio.workspace = true diff --git a/litellm-rust/crates/core/src/auth/credential.rs b/litellm-rust/crates/auth/src/credential.rs similarity index 91% rename from litellm-rust/crates/core/src/auth/credential.rs rename to litellm-rust/crates/auth/src/credential.rs index c64d331b877..6721eb67a35 100644 --- a/litellm-rust/crates/core/src/auth/credential.rs +++ b/litellm-rust/crates/auth/src/credential.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use veil::Redact; -use crate::AuthError; +use crate::Error; use super::{ResolvedCredential, SecretValue, TokenProviderHandle}; @@ -48,7 +48,7 @@ pub enum CredentialLookup { } pub type CredentialLookupFuture<'a> = - Pin> + Send + 'a>>; + Pin> + Send + 'a>>; pub trait CredentialResolver: std::fmt::Debug + Send + Sync { fn resolve<'a>(&'a self, reference: &'a CredentialRef) -> CredentialLookupFuture<'a>; @@ -62,7 +62,7 @@ impl CredentialResolverHandle { Self(resolver) } - pub async fn resolve(&self, reference: &CredentialRef) -> Result { + pub async fn resolve(&self, reference: &CredentialRef) -> Result { self.0.resolve(reference).await } } @@ -84,7 +84,7 @@ impl CredentialPlan { pub async fn resolve( &self, resolver: &CredentialResolverHandle, - ) -> Result { + ) -> Result { match self { Self::Static(CredentialRef::Explicit(secret)) => Ok( CredentialPlanResolution::Resolved(ResolvedCredential::Static(secret.clone())), @@ -103,7 +103,7 @@ impl CredentialPlan { Self::Caller(caller) => { let credential = caller.acquire().await?; if credential.secret().expose().is_empty() { - return Err(AuthError::EmptyCallerCredential); + return Err(Error::EmptyCallerCredential); } Ok(CredentialPlanResolution::Resolved(credential)) } @@ -119,8 +119,8 @@ mod tests { CredentialLookup, CredentialLookupFuture, CredentialPlan, CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, }; - use crate::AuthError; - use crate::auth::SecretValue; + use crate::Error; + use crate::SecretValue; #[derive(Debug)] struct HostResolver; @@ -164,7 +164,7 @@ mod tests { impl CredentialResolver for FailingResolver { fn resolve<'a>(&'a self, _reference: &'a CredentialRef) -> CredentialLookupFuture<'a> { - Box::pin(async { Err(AuthError::UnresolvedOidcReference) }) + Box::pin(async { Err(Error::UnresolvedOidcReference) }) } } @@ -178,6 +178,6 @@ mod tests { .await .expect_err("acquisition errors cannot become fallback"); - assert_eq!(error, AuthError::UnresolvedOidcReference); + assert_eq!(error, Error::UnresolvedOidcReference); } } diff --git a/litellm-rust/crates/auth/src/error.rs b/litellm-rust/crates/auth/src/error.rs new file mode 100644 index 00000000000..914265ffb32 --- /dev/null +++ b/litellm-rust/crates/auth/src/error.rs @@ -0,0 +1,120 @@ +use thiserror::Error as ThisError; + +#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +pub enum Error { + #[error("invalid authentication configuration: credential header already exists")] + ExistingCredentialHeader, + #[error( + "invalid authentication configuration: credential plan is not allowed by the provider auth policy" + )] + DisallowedCredentialPlan, + #[error("invalid authentication configuration: credential cannot be empty")] + EmptyCredential, + #[error("invalid authentication configuration: invalid Azure credential selector")] + InvalidAzureSelector, + #[error( + "invalid authentication configuration: ClientSecretCredential requires tenant_id, client_id, and client_secret" + )] + MissingClientSecretFields, + #[error("invalid authentication configuration: WorkloadIdentityCredential requires tenant_id")] + MissingWorkloadTenant, + #[error("invalid authentication configuration: WorkloadIdentityCredential requires client_id")] + MissingWorkloadClient, + #[error( + "invalid authentication configuration: WorkloadIdentityCredential requires azure_federated_token_file" + )] + MissingWorkloadTokenFile, + #[error( + "invalid authentication configuration: credential reference requires a host credential resolver" + )] + MissingHostResolver, + #[error( + "invalid authentication configuration: caller credential plan requires provider-specific inputs" + )] + MissingCallerInputs, + #[error("invalid authentication configuration: credential header {0} already exists")] + DuplicateHeader(&'static str), + #[error("invalid authentication configuration: {0} must be a string or null")] + InvalidFieldType(String), + #[error("invalid authentication configuration: unsupported OIDC reference")] + UnsupportedOidcReference, + #[error("invalid authentication configuration: {0} cannot be empty")] + EmptyReference(String), + #[error("invalid authentication configuration: Azure credential initialization failed: {0}")] + AzureCredentialInitialization(String), + #[error( + "invalid authentication configuration: Azure authority must be an HTTPS origin without credentials, query, or fragment" + )] + InvalidAzureAuthority, + #[error( + "invalid authentication configuration: request-controlled Azure auth inputs cannot be combined with host credentials" + )] + MixedAzureCredentialSources, + #[error( + "invalid authentication configuration: request-controlled Azure credential references are not allowed" + )] + RequestAzureCredentialReference, + #[error( + "invalid authentication configuration: host credentials cannot be sent to a request-controlled Azure endpoint" + )] + RequestAzureCredentialDestination, + #[error( + "invalid authentication configuration: credentials cannot be sent to a request-controlled Vertex AI endpoint" + )] + RequestVertexCredentialDestination, + #[error( + "invalid authentication configuration: request-controlled Vertex credentials must use the canonical Google OAuth token endpoint" + )] + RequestVertexTokenEndpoint, + #[error("credential acquisition failed: {0}")] + AzureTokenAcquisition(String), + #[error("credential acquisition failed: Vertex AI credentials: {0}")] + VertexTokenAcquisition(String), + #[error("{0}")] + ProviderAuthentication(String), + #[error("credential acquisition failed: {}", .0.iter().map(ToString::to_string).collect::>().join("; "))] + CredentialChain(Vec), + #[error("credential caller failed: credential caller returned an empty credential")] + EmptyCallerCredential, + #[error("credential caller failed: Azure AD token provider returned an empty token")] + EmptyAzureToken, + #[error("credential acquisition failed: Azure OIDC reference did not resolve to a value")] + UnresolvedOidcReference, + #[error( + "Missing {provider} API Key - Set `api_key` or the {environment_variable} environment variable" + )] + MissingApiKey { + provider: &'static str, + environment_variable: &'static str, + }, + #[error( + "Missing {provider} API Base - Set {environment_variable} environment variable or pass api_base parameter" + )] + MissingApiBase { + provider: &'static str, + environment_variable: &'static str, + }, + #[error( + "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. Expected format: https://.services.ai.azure.com/anthropic" + )] + MissingAzureApiBase, + #[error("invalid authentication header")] + InvalidHeader, +} + +#[cfg(test)] +mod tests { + use super::Error; + + #[test] + fn missing_api_key_names_provider_and_environment_variable() { + assert_eq!( + Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + } + .to_string(), + "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable" + ); + } +} diff --git a/litellm-rust/crates/core/src/auth/http.rs b/litellm-rust/crates/auth/src/http.rs similarity index 84% rename from litellm-rust/crates/core/src/auth/http.rs rename to litellm-rust/crates/auth/src/http.rs index 83931311550..7d20991d838 100644 --- a/litellm-rust/crates/core/src/auth/http.rs +++ b/litellm-rust/crates/auth/src/http.rs @@ -1,5 +1,4 @@ -use crate::AuthError; -use crate::auth::error::AuthConfigurationError; +use crate::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CredentialPlacement { @@ -16,23 +15,19 @@ impl CredentialPlacement { } } -pub(crate) fn apply_credential( +pub fn apply_credential( headers: Vec<(String, String)>, credential: &str, placement: CredentialPlacement, -) -> Result, AuthError> { +) -> Result, Error> { if credential.trim().is_empty() { - return Err(AuthError::Configuration( - AuthConfigurationError::EmptyCredential, - )); + return Err(Error::EmptyCredential); } if headers .iter() .any(|(name, _)| name.eq_ignore_ascii_case(placement.header_name())) { - return Err(AuthError::Configuration( - AuthConfigurationError::DuplicateHeader(placement.header_name()), - )); + return Err(Error::DuplicateHeader(placement.header_name())); } let value = match placement { CredentialPlacement::Bearer => format!("Bearer {credential}"), diff --git a/litellm-rust/crates/core/src/auth/mod.rs b/litellm-rust/crates/auth/src/lib.rs similarity index 94% rename from litellm-rust/crates/core/src/auth/mod.rs rename to litellm-rust/crates/auth/src/lib.rs index 2940a983fb9..7a24d2acf70 100644 --- a/litellm-rust/crates/core/src/auth/mod.rs +++ b/litellm-rust/crates/auth/src/lib.rs @@ -1,8 +1,6 @@ mod credential; -pub mod error; -pub(crate) mod vertex; -pub use error::AuthError; -pub(crate) mod http; +mod error; +pub mod http; mod policy; mod secret; mod token; @@ -51,6 +49,7 @@ pub use credential::{ CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, credential_default_fields, credential_index, }; +pub use error::Error; pub use http::{CredentialPlacement, RequestAuth}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use secret::SecretValue; diff --git a/litellm-rust/crates/core/src/auth/policy.rs b/litellm-rust/crates/auth/src/policy.rs similarity index 82% rename from litellm-rust/crates/core/src/auth/policy.rs rename to litellm-rust/crates/auth/src/policy.rs index b796dedf0d8..4a1f5eeecf9 100644 --- a/litellm-rust/crates/core/src/auth/policy.rs +++ b/litellm-rust/crates/auth/src/policy.rs @@ -1,5 +1,4 @@ -use crate::AuthError; -use crate::auth::error::AuthConfigurationError; +use crate::Error; use super::http::apply_credential; use super::{CredentialPlacement, ResolvedCredential}; @@ -46,22 +45,18 @@ impl ProviderAuthPolicy { headers: Vec<(String, String)>, kind: CredentialPlanKind, credential: &ResolvedCredential, - ) -> Result, AuthError> { + ) -> Result, Error> { if self.has_existing_credential(&headers) { return match self.existing_header_behavior { ExistingHeaderBehavior::Preserve => Ok(headers), - ExistingHeaderBehavior::Reject => Err(AuthError::Configuration( - AuthConfigurationError::ExistingCredentialHeader, - )), + ExistingHeaderBehavior::Reject => Err(Error::ExistingCredentialHeader), }; } - let rule = - self.rules - .iter() - .find(|rule| rule.kind == kind) - .ok_or(AuthError::Configuration( - AuthConfigurationError::DisallowedCredentialPlan, - ))?; + let rule = self + .rules + .iter() + .find(|rule| rule.kind == kind) + .ok_or(Error::DisallowedCredentialPlan)?; apply_credential(headers, credential.secret().expose(), rule.placement) } } @@ -69,7 +64,7 @@ impl ProviderAuthPolicy { #[cfg(test)] mod tests { use super::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; - use crate::auth::{CredentialPlacement, ResolvedCredential, SecretValue}; + use crate::{CredentialPlacement, ResolvedCredential, SecretValue}; const RULES: &[CredentialRule] = &[CredentialRule { kind: CredentialPlanKind::Static, diff --git a/litellm-rust/crates/core/src/auth/secret.rs b/litellm-rust/crates/auth/src/secret.rs similarity index 100% rename from litellm-rust/crates/core/src/auth/secret.rs rename to litellm-rust/crates/auth/src/secret.rs diff --git a/litellm-rust/crates/core/src/auth/token.rs b/litellm-rust/crates/auth/src/token.rs similarity index 83% rename from litellm-rust/crates/core/src/auth/token.rs rename to litellm-rust/crates/auth/src/token.rs index cfc6b8f0d6b..94da5f259fb 100644 --- a/litellm-rust/crates/core/src/auth/token.rs +++ b/litellm-rust/crates/auth/src/token.rs @@ -5,7 +5,7 @@ use std::time::SystemTime; use veil::Redact; -use crate::AuthError; +use crate::Error; use super::secret::SecretValue; @@ -27,7 +27,7 @@ impl ResolvedCredential { } pub type TokenFuture<'a> = - Pin> + Send + 'a>>; + Pin> + Send + 'a>>; pub trait TokenProvider: std::fmt::Debug + Send + Sync { fn acquire(&self) -> TokenFuture<'_>; @@ -41,7 +41,7 @@ impl TokenProviderHandle { Self(caller) } - pub async fn acquire(&self) -> Result { + pub async fn acquire(&self) -> Result { self.0.acquire().await } } diff --git a/litellm-rust/crates/cache-memory/Cargo.toml b/litellm-rust/crates/cache-memory/Cargo.toml new file mode 100644 index 00000000000..d4487573a9a --- /dev/null +++ b/litellm-rust/crates/cache-memory/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-cache-memory" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +serde_json.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/cache-memory/src/cache.rs b/litellm-rust/crates/cache-memory/src/cache.rs new file mode 100644 index 00000000000..1908ff44a81 --- /dev/null +++ b/litellm-rust/crates/cache-memory/src/cache.rs @@ -0,0 +1,254 @@ +use std::cmp::Reverse; +use std::collections::{BinaryHeap, HashMap}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use litellm_cache::{ + BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs, + Error, +}; + +const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200; +const DEFAULT_TTL: Duration = Duration::from_secs(600); + +type ValueMeasure = Arc Result + Send + Sync>; +type ValueValidator = Arc Result<(), Error> + Send + Sync>; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CacheWrite { + Stored, + Disabled, + TooLarge, +} + +struct CacheState { + values: HashMap, + expirations: HashMap, + expiration_heap: BinaryHeap>, +} + +pub struct InMemoryCache { + state: Mutex>, + max_size_in_memory: usize, + default_ttl: Duration, + max_entry_bytes: Option, + measure_value: Option>, + validate_value: Option>, + now: Arc Duration + Send + Sync>, +} + +impl Default for InMemoryCache { + fn default() -> Self { + Self::new(None, None) + } +} + +impl InMemoryCache { + pub fn new(max_size_in_memory: Option, default_ttl: Option) -> Self { + Self::with_clock(max_size_in_memory, default_ttl, || { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + }) + } + + pub fn with_clock( + max_size_in_memory: Option, + default_ttl: Option, + now: impl Fn() -> Duration + Send + Sync + 'static, + ) -> Self { + Self::with_clock_and_size_measurement(max_size_in_memory, default_ttl, None, None, now) + } + + pub fn with_clock_and_size_measurement( + max_size_in_memory: Option, + default_ttl: Option, + max_entry_bytes: Option, + measure_value: Option>, + now: impl Fn() -> Duration + Send + Sync + 'static, + ) -> Self { + Self { + state: Mutex::new(CacheState { + values: HashMap::new(), + expirations: HashMap::new(), + expiration_heap: BinaryHeap::new(), + }), + max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), + default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), + max_entry_bytes, + measure_value, + validate_value: None, + now: Arc::new(now), + } + } + + pub fn set_cache( + &self, + key: impl Into, + value: V, + ttl: Option, + ) -> Result { + if self.max_size_in_memory == 0 { + return Ok(CacheWrite::Disabled); + } + if let Some(validate) = &self.validate_value { + validate(&value)?; + } + if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value) + && measure(&value)? > limit + { + return Ok(CacheWrite::TooLarge); + } + let now = (self.now)(); + let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + Self::evict(&mut state, self.max_size_in_memory, now); + let key = key.into(); + state.values.insert(key.clone(), value); + let expiration = state.expirations.get(&key).copied(); + if expiration.is_none_or(|expiration| expiration < now) { + let expiration = now + ttl.unwrap_or(self.default_ttl); + state.expirations.insert(key.clone(), expiration); + state.expiration_heap.push(Reverse((expiration, key))); + } + Ok(CacheWrite::Stored) + } + + pub fn get_cache(&self, key: &str) -> Result, Error> { + let now = (self.now)(); + let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + if state + .expirations + .get(key) + .is_some_and(|expiration| *expiration < now) + { + Self::remove(&mut state, key); + } + Ok(state.values.get(key).cloned()) + } + + pub fn expires_at(&self, key: &str) -> Result, Error> { + Ok(self + .state + .lock() + .map_err(|_| Error::Unavailable)? + .expirations + .get(key) + .copied()) + } + + pub fn delete_cache(&self, key: &str) -> Result<(), Error> { + let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + Self::remove(&mut state, key); + Ok(()) + } + + pub fn flush_cache(&self) -> Result<(), Error> { + let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + state.values.clear(); + state.expirations.clear(); + state.expiration_heap.clear(); + Ok(()) + } + + fn evict(state: &mut CacheState, capacity: usize, now: Duration) { + while let Some(Reverse((expiration, key))) = state.expiration_heap.peek().cloned() { + if state.expirations.get(&key).copied() != Some(expiration) { + state.expiration_heap.pop(); + } else if expiration <= now { + state.expiration_heap.pop(); + Self::remove(state, &key); + } else { + break; + } + } + while state.values.len() >= capacity { + let Some(Reverse((expiration, key))) = state.expiration_heap.pop() else { + break; + }; + if state.expirations.get(&key).copied() == Some(expiration) { + Self::remove(state, &key); + } + } + } + + fn remove(state: &mut CacheState, key: &str) { + state.values.remove(key); + state.expirations.remove(key); + } +} + +impl InMemoryCache { + pub fn response_cache(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self { + Self::response_cache_with_clock(capacity, ttl, max_entry_bytes, || { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + }) + } + + pub fn response_cache_with_clock( + capacity: usize, + ttl: Duration, + max_entry_bytes: usize, + now: impl Fn() -> Duration + Send + Sync + 'static, + ) -> Self { + let mut cache = Self::with_clock_and_size_measurement( + Some(capacity), + Some(ttl), + Some(max_entry_bytes), + Some(Arc::new(|entry: &CacheEntry| { + serde_json::to_vec(entry) + .map(|bytes| bytes.len()) + .map_err(|_| Error::InvalidEntry) + })), + now, + ); + cache.validate_value = Some(Arc::new(|entry: &CacheEntry| { + entry + .timestamp + .is_finite() + .then_some(()) + .ok_or(Error::InvalidEntry) + })); + cache + } +} + +impl BaseCache for InMemoryCache { + type Value = CacheEntry; + + fn default_ttl(&self) -> Duration { + self.default_ttl + } + + fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> { + let ttl = self.get_ttl(&kwargs); + self.set_cache(key, value, Some(ttl)).map(|_| ()) + } + + fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result, Error> { + self.get_cache(key) + } + + fn delete_cache(&self, key: &str) -> Result<(), Error> { + self.delete_cache(key) + } + + fn flush_cache(&self) -> Result<(), Error> { + self.flush_cache() + } + + fn disconnect(&self) -> CacheFuture<'_, ()> { + Box::pin(async { Ok(()) }) + } + + fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> { + Box::pin(async { + Ok(CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: "In-memory cache connection test successful".into(), + error: None, + }) + }) + } +} diff --git a/litellm-rust/crates/cache-memory/src/lib.rs b/litellm-rust/crates/cache-memory/src/lib.rs new file mode 100644 index 00000000000..c5b7fb6cb54 --- /dev/null +++ b/litellm-rust/crates/cache-memory/src/lib.rs @@ -0,0 +1,3 @@ +mod cache; + +pub use cache::{CacheWrite, InMemoryCache}; diff --git a/litellm-rust/crates/cache-memory/tests/cache.rs b/litellm-rust/crates/cache-memory/tests/cache.rs new file mode 100644 index 00000000000..aaf82641db7 --- /dev/null +++ b/litellm-rust/crates/cache-memory/tests/cache.rs @@ -0,0 +1,158 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use litellm_cache::{BaseCache, CacheConnectionStatus, CacheEntry, Error}; +use litellm_cache_memory::{CacheWrite, InMemoryCache}; +use rstest::{fixture, rstest}; + +#[fixture] +fn clock() -> Arc { + Arc::new(AtomicU64::new(100)) +} + +fn cache(clock: Arc, capacity: usize) -> InMemoryCache { + InMemoryCache::with_clock(Some(capacity), Some(Duration::from_secs(60)), move || { + Duration::from_secs(clock.load(Ordering::SeqCst)) + }) +} + +#[rstest] +fn default_explicit_and_override_ttls_follow_python_rules(clock: Arc) { + let cache = cache(clock.clone(), 4); + cache.set_cache("key", "first".into(), None).unwrap(); + assert_eq!( + cache.expires_at("key").unwrap(), + Some(Duration::from_secs(160)) + ); + cache + .set_cache("key", "second".into(), Some(Duration::from_secs(10))) + .unwrap(); + assert_eq!( + cache.expires_at("key").unwrap(), + Some(Duration::from_secs(160)) + ); + clock.store(160, Ordering::SeqCst); + assert_eq!(cache.get_cache("key").unwrap(), Some("second".into())); + clock.store(161, Ordering::SeqCst); + assert_eq!(cache.get_cache("key").unwrap(), None); + cache + .set_cache("key", "third".into(), Some(Duration::from_secs(10))) + .unwrap(); + assert_eq!( + cache.expires_at("key").unwrap(), + Some(Duration::from_secs(171)) + ); +} + +#[rstest] +fn write_at_expiry_boundary_refreshes_ttl(clock: Arc) { + let cache = cache(clock.clone(), 4); + cache + .set_cache("key", "first".into(), Some(Duration::from_secs(10))) + .unwrap(); + clock.store(110, Ordering::SeqCst); + cache + .set_cache("key", "second".into(), Some(Duration::from_secs(10))) + .unwrap(); + assert_eq!( + cache.expires_at("key").unwrap(), + Some(Duration::from_secs(120)) + ); + clock.store(115, Ordering::SeqCst); + assert_eq!(cache.get_cache("key").unwrap(), Some("second".into())); +} + +#[rstest] +fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Arc) { + let cache = cache(clock, 2); + cache + .set_cache("early", "a".into(), Some(Duration::from_secs(10))) + .unwrap(); + cache + .set_cache("late", "b".into(), Some(Duration::from_secs(20))) + .unwrap(); + cache.delete_cache("early").unwrap(); + cache + .set_cache("new", "c".into(), Some(Duration::from_secs(30))) + .unwrap(); + assert_eq!(cache.get_cache("late").unwrap(), Some("b".into())); + cache + .set_cache("last", "d".into(), Some(Duration::from_secs(40))) + .unwrap(); + assert_eq!(cache.get_cache("late").unwrap(), None); +} + +#[test] +fn disabled_size_limited_and_synchronized_response_writes_are_observable() { + let disabled = InMemoryCache::::response_cache(0, Duration::from_secs(60), 80); + assert_eq!( + disabled + .set_cache( + "a", + CacheEntry { + timestamp: 1.0, + response: serde_json::json!("x") + }, + None + ) + .unwrap(), + CacheWrite::Disabled + ); + let cache = InMemoryCache::::response_cache(2, Duration::from_secs(60), 80); + assert_eq!( + cache + .set_cache( + "large", + CacheEntry { + timestamp: 1.0, + response: serde_json::json!("x".repeat(100)) + }, + None + ) + .unwrap(), + CacheWrite::TooLarge + ); + cache + .set_cache( + "small", + CacheEntry { + timestamp: 1.0, + response: serde_json::json!("ok"), + }, + None, + ) + .unwrap(); + assert!(cache.get_cache("small").unwrap().is_some()); + assert_eq!( + cache + .set_cache( + "invalid", + CacheEntry { + timestamp: f64::NAN, + response: serde_json::json!("bad"), + }, + None, + ) + .unwrap_err(), + Error::InvalidEntry + ); + cache.delete_cache("small").unwrap(); + cache.flush_cache().unwrap(); +} + +#[tokio::test] +async fn connection_test_matches_python_result_contract() { + let cache = InMemoryCache::::default(); + let result = BaseCache::test_connection(&cache).await.unwrap(); + assert_eq!(result.status, CacheConnectionStatus::Success); + assert_eq!(result.message, "In-memory cache connection test successful"); + assert_eq!(result.error, None); + assert_eq!( + serde_json::to_value(result).unwrap(), + serde_json::json!({ + "status": "success", + "message": "In-memory cache connection test successful" + }) + ); +} diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml similarity index 50% rename from litellm-rust/crates/config/Cargo.toml rename to litellm-rust/crates/cache/Cargo.toml index ae9710266a3..a14c4294aa0 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -1,16 +1,15 @@ [package] -name = "litellm-config" +name = "litellm-cache" version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] -litellm-core.workspace = true -pyo3 = { workspace = true, features = ["auto-initialize"], optional = true } +serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true -[features] -default = [] -python = ["dep:pyo3"] +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/cache/src/base_cache.rs b/litellm-rust/crates/cache/src/base_cache.rs new file mode 100644 index 00000000000..2ba8ff92ebd --- /dev/null +++ b/litellm-rust/crates/cache/src/base_cache.rs @@ -0,0 +1,98 @@ +use std::future::Future; +use std::pin::Pin; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::Error; + +pub type CacheFuture<'a, T> = Pin> + Send + 'a>>; + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CacheKwargs { + pub ttl: Option, + pub extras: Map, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CacheConnectionStatus { + Success, + Failed, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct CacheConnectionResult { + pub status: CacheConnectionStatus, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +pub trait BaseCache: Send + Sync { + type Value: Clone + Send + Sync + 'static; + + fn default_ttl(&self) -> Duration { + Duration::from_secs(60) + } + + fn get_ttl(&self, kwargs: &CacheKwargs) -> Duration { + kwargs.ttl.unwrap_or_else(|| self.default_ttl()) + } + + fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error>; + + fn get_cache(&self, key: &str, kwargs: &CacheKwargs) -> Result, Error>; + + fn async_set_cache<'a>( + &'a self, + key: &'a str, + value: Self::Value, + kwargs: CacheKwargs, + ) -> CacheFuture<'a, ()> { + Box::pin(async move { self.set_cache(key, value, kwargs) }) + } + + fn async_get_cache<'a>( + &'a self, + key: &'a str, + kwargs: &'a CacheKwargs, + ) -> CacheFuture<'a, Option> { + Box::pin(async move { self.get_cache(key, kwargs) }) + } + + fn async_set_cache_pipeline<'a>( + &'a self, + cache_list: Vec<(String, Self::Value)>, + kwargs: CacheKwargs, + ) -> CacheFuture<'a, ()> { + Box::pin(async move { + for (key, value) in cache_list { + self.set_cache(&key, value, kwargs.clone())?; + } + Ok(()) + }) + } + + fn batch_cache_write<'a>( + &'a self, + key: &'a str, + value: Self::Value, + kwargs: CacheKwargs, + ) -> CacheFuture<'a, ()> { + self.async_set_cache(key, value, kwargs) + } + + fn delete_cache(&self, key: &str) -> Result<(), Error>; + + fn async_delete_cache<'a>(&'a self, key: &'a str) -> CacheFuture<'a, ()> { + Box::pin(async move { self.delete_cache(key) }) + } + + fn flush_cache(&self) -> Result<(), Error>; + + fn disconnect(&self) -> CacheFuture<'_, ()>; + + fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult>; +} diff --git a/litellm-rust/crates/cache/src/caching.rs b/litellm-rust/crates/cache/src/caching.rs new file mode 100644 index 00000000000..1aab6ee8e91 --- /dev/null +++ b/litellm-rust/crates/cache/src/caching.rs @@ -0,0 +1,166 @@ +use std::sync::Arc; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::{BaseCache, CacheKwargs, Error}; + +pub use crate::BaseCache as Cache; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +pub enum CacheMode { + #[default] + #[serde(rename = "default_on")] + DefaultOn, + #[serde(rename = "default_off")] + DefaultOff, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CacheKeyField { + pub name: String, + pub value: Option, + pub api_parameter: bool, + pub internal_parameter: bool, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct CacheKeyInput { + pub fields: Vec, + pub preset: Option, + pub namespace: Option, + pub include_provider_parameters: bool, +} + +#[derive(Default)] +pub struct CacheKeyContext { + pub model_group: Option, + pub caching_groups: Vec<(Vec, String)>, + pub file_checksum: Option, + pub file_object_name: Option, + pub metadata_file_name: Option, + pub parameters_file_name: Option, +} + +impl CacheKeyContext { + pub fn apply(self, input: &mut CacheKeyInput) { + let group = self.model_group.as_ref().and_then(|model| { + self.caching_groups + .iter() + .find(|(models, _)| models.contains(model)) + }); + for field in &mut input.fields { + match field.name.as_str() { + "model" => { + field.value = group + .map(|(_, formatted)| formatted.clone()) + .or_else(|| self.model_group.clone()) + .or_else(|| field.value.take()) + } + "file" => { + field.value = self + .file_checksum + .clone() + .or_else(|| self.file_object_name.clone()) + .or_else(|| self.metadata_file_name.clone()) + .or_else(|| self.parameters_file_name.clone()) + } + _ => {} + } + } + } +} + +pub fn get_cache_key(input: &CacheKeyInput) -> String { + cache_key(input) +} + +pub fn cache_key(input: &CacheKeyInput) -> String { + if let Some(preset) = &input.preset { + return preset.clone(); + } + let mut digest = Sha256::new(); + for field in &input.fields { + if (field.api_parameter || (input.include_provider_parameters && !field.internal_parameter)) + && let Some(value) = &field.value + { + digest.update(field.name.as_bytes()); + digest.update(b": "); + digest.update(value.as_bytes()); + } + } + let hash = format!("{:x}", digest.finalize()); + input + .namespace + .as_deref() + .filter(|namespace| !namespace.is_empty()) + .map_or(hash.clone(), |namespace| format!("{namespace}:{hash}")) +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)] +pub struct CacheControls { + pub supported_call_type: bool, + pub configured: bool, + pub native_backend: bool, + pub default_on: bool, + pub caching: Option, + pub no_cache: bool, + pub no_store: bool, + #[serde(default)] + pub use_cache: bool, +} + +impl CacheControls { + pub fn reads(self) -> bool { + self.supported_call_type + && self.configured + && self.caching.unwrap_or(true) + && !self.no_cache + && (self.default_on || self.use_cache) + } + + pub fn writes(self) -> bool { + self.supported_call_type + && self.configured + && !self.no_store + && (self.default_on || self.use_cache) + } +} + +pub fn should_use_cache(controls: CacheControls) -> bool { + controls.reads() || controls.writes() +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CacheEntry { + pub timestamp: f64, + pub response: Value, +} + +impl CacheEntry { + pub fn fresh(&self, now: Duration, max_age: Option) -> bool { + self.timestamp.is_finite() + && max_age.is_none_or(|age| now.as_secs_f64() - self.timestamp <= age.as_secs_f64()) + } +} + +pub fn get_cache( + cache: &dyn BaseCache, + key: &str, + kwargs: &CacheKwargs, +) -> Result, Error> { + cache.get_cache(key, kwargs) +} + +pub fn set_cache( + cache: &dyn BaseCache, + key: &str, + entry: CacheEntry, + kwargs: CacheKwargs, +) -> Result<(), Error> { + cache.set_cache(key, entry, kwargs) +} + +pub type CacheBackend = Arc>; diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs new file mode 100644 index 00000000000..d447c80f62d --- /dev/null +++ b/litellm-rust/crates/cache/src/error.rs @@ -0,0 +1,7 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("cache is unavailable")] + Unavailable, + #[error("invalid cache entry")] + InvalidEntry, +} diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs new file mode 100644 index 00000000000..d0fe3de15cd --- /dev/null +++ b/litellm-rust/crates/cache/src/lib.rs @@ -0,0 +1,12 @@ +mod base_cache; +mod caching; +mod error; + +pub use base_cache::{ + BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheFuture, CacheKwargs, +}; +pub use caching::{ + Cache, CacheBackend, CacheControls, CacheEntry, CacheKeyContext, CacheKeyField, CacheKeyInput, + CacheMode, cache_key, get_cache, get_cache_key, set_cache, should_use_cache, +}; +pub use error::Error; diff --git a/litellm-rust/crates/cache/tests/caching.rs b/litellm-rust/crates/cache/tests/caching.rs new file mode 100644 index 00000000000..1192fc9a2b0 --- /dev/null +++ b/litellm-rust/crates/cache/tests/caching.rs @@ -0,0 +1,139 @@ +use litellm_cache::{ + BaseCache, CacheConnectionResult, CacheControls, CacheEntry, CacheFuture, CacheKeyContext, + CacheKeyField, CacheKeyInput, CacheKwargs, Error, cache_key, get_cache_key, +}; +use sha2::{Digest, Sha256}; +use std::time::Duration; + +struct TestCache { + default_ttl: Duration, +} + +impl BaseCache for TestCache { + type Value = CacheEntry; + + fn default_ttl(&self) -> Duration { + self.default_ttl + } + + fn set_cache(&self, _: &str, _: Self::Value, _: CacheKwargs) -> Result<(), Error> { + Ok(()) + } + + fn get_cache(&self, _: &str, _: &CacheKwargs) -> Result, Error> { + Ok(None) + } + + fn delete_cache(&self, _: &str) -> Result<(), Error> { + Ok(()) + } + + fn flush_cache(&self) -> Result<(), Error> { + Ok(()) + } + + fn disconnect(&self) -> CacheFuture<'_, ()> { + Box::pin(async { Ok(()) }) + } + + fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> { + unreachable!() + } +} + +#[test] +fn ttl_uses_default_and_allows_per_call_override() { + let cache = TestCache { + default_ttl: Duration::from_secs(60), + }; + assert_eq!( + cache.get_ttl(&CacheKwargs::default()), + Duration::from_secs(60) + ); + assert_eq!( + cache.get_ttl(&CacheKwargs { + ttl: Some(Duration::from_secs(5)), + ..Default::default() + }), + Duration::from_secs(5) + ); +} + +#[test] +fn keys_match_python_order_groups_files_presets_and_namespaces() { + let mut input = CacheKeyInput { + fields: vec![ + CacheKeyField { + name: "model".into(), + value: Some("deployment".into()), + api_parameter: true, + internal_parameter: false, + }, + CacheKeyField { + name: "file".into(), + value: None, + api_parameter: true, + internal_parameter: false, + }, + ], + namespace: Some("team".into()), + ..Default::default() + }; + CacheKeyContext { + model_group: Some("group".into()), + caching_groups: vec![(vec!["group".into()], "['group']".into())], + file_checksum: Some("checksum".into()), + ..Default::default() + } + .apply(&mut input); + assert_eq!( + cache_key(&input), + format!( + "team:{:x}", + Sha256::digest(b"model: ['group']file: checksum") + ) + ); + input.preset = Some("preset".into()); + assert_eq!(get_cache_key(&input), "preset"); +} + +#[test] +fn cache_controls_honor_default_modes_and_directives() { + let enabled = CacheControls { + supported_call_type: true, + configured: true, + default_on: true, + ..Default::default() + }; + assert!(enabled.reads()); + assert!(enabled.writes()); + assert!( + !CacheControls { + default_on: false, + ..enabled + } + .reads() + ); + assert!( + CacheControls { + default_on: false, + use_cache: true, + ..enabled + } + .reads() + ); + assert!( + !CacheControls { + no_cache: true, + ..enabled + } + .reads() + ); + assert!( + !CacheControls { + no_store: true, + ..enabled + } + .writes() + ); +} diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs deleted file mode 100644 index cec7bc5c110..00000000000 --- a/litellm-rust/crates/config/src/error.rs +++ /dev/null @@ -1,11 +0,0 @@ -use thiserror::Error as ThisError; - -#[derive(Debug, ThisError)] -pub enum Error { - #[error("read_model_list failed: {0}")] - PythonLoading(String), - #[error("serializing model_list failed: {0}")] - Serialization(String), - #[error("parsing model_list failed: {0}")] - ModelListParsing(#[source] serde_json::Error), -} diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs deleted file mode 100644 index 655affbb0b7..00000000000 --- a/litellm-rust/crates/config/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod error; -#[cfg(feature = "python")] -mod python; - -pub use error::Error; -#[cfg(feature = "python")] -pub use python::load_model_list; diff --git a/litellm-rust/crates/config/src/python.rs b/litellm-rust/crates/config/src/python.rs deleted file mode 100644 index fdad5027baa..00000000000 --- a/litellm-rust/crates/config/src/python.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::path::Path; - -use litellm_core::router::Deployment; -use pyo3::prelude::*; - -use crate::Error; - -pub fn load_model_list(config_path: &Path) -> Result, Error> { - Python::attach(|python| { - let model_list = python - .import("litellm.proxy.read_model_list") - .and_then(|module| module.getattr("read_model_list")) - .and_then(|reader| reader.call1((config_path.to_string_lossy().as_ref(),))) - .map_err(|error| Error::PythonLoading(error.to_string()))?; - - let model_list_json = python - .import("json") - .and_then(|json| json.getattr("dumps")) - .and_then(|dumps| dumps.call1((model_list,))) - .and_then(|encoded| encoded.extract::()) - .map_err(|error| Error::Serialization(error.to_string()))?; - - parse_model_list(&model_list_json) - }) -} - -fn parse_model_list(model_list_json: &str) -> Result, Error> { - serde_json::from_str(model_list_json).map_err(Error::ModelListParsing) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_resolved_model_list() { - let deployments = parse_model_list( - r#"[ - { - "model_name": "realtime", - "litellm_params": { - "model": "openai/gpt-realtime", - "api_key": "resolved-secret", - "api_base": "https://api.example.test/v1" - } - }, - { - "model_name": "without-optional-values", - "litellm_params": {"model": "openai/gpt-4.1"} - } - ]"#, - ) - .expect("resolved model list should parse"); - - assert_eq!(deployments.len(), 2); - assert_eq!(deployments[0].model_name, "realtime"); - assert_eq!( - deployments[0].litellm_params.api_key.as_deref(), - Some("resolved-secret") - ); - assert_eq!( - deployments[0].litellm_params.api_base.as_deref(), - Some("https://api.example.test/v1") - ); - assert_eq!(deployments[1].litellm_params.api_key, None); - assert_eq!(deployments[1].litellm_params.api_base, None); - } - - #[test] - fn malformed_model_list_returns_parsing_error() { - let error = parse_model_list(r#"[{"model_name":"missing-params"}]"#) - .expect_err("missing litellm_params should fail"); - - assert!(matches!(error, Error::ModelListParsing(_))); - } -} diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 09c526f73cf..ededfeef8af 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -10,10 +10,11 @@ autotests = false bytes.workspace = true futures-util.workspace = true base64.workspace = true -azure_core.workspace = true -azure_identity.workspace = true data-url = "0.3.2" -gcp_auth.workspace = true +litellm-auth.workspace = true +litellm-auth-aws.workspace = true +litellm-auth-azure.workspace = true +litellm-auth-gcp.workspace = true moka.workspace = true mime_guess = "2.0.5" rand.workspace = true @@ -28,30 +29,9 @@ subtle.workspace = true tokio = { workspace = true, features = ["sync"] } tokio-tungstenite.workspace = true thiserror.workspace = true -tracing.workspace = true -tracing-subscriber = { workspace = true, optional = true } sha2.workspace = true url.workspace = true veil.workspace = true -aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } -aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } -aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } -aws-sigv4 = { version = "1.5.1", optional = true } -aws-types = { version = "1.4.0", optional = true } -aws-smithy-runtime-api = { version = "1.13.0", optional = true } - -[features] -default = [] -bedrock-auth = [ - "dep:aws-config", - "dep:aws-credential-types", - "dep:aws-sdk-sts", - "dep:aws-sigv4", - "dep:aws-types", - "dep:aws-smithy-runtime-api", -] -observability = ["dep:tracing-subscriber"] [dev-dependencies] rstest.workspace = true -tracing-subscriber.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/error.rs b/litellm-rust/crates/core/src/audio_transcription/error.rs new file mode 100644 index 00000000000..f9ffb12d349 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/error.rs @@ -0,0 +1,26 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid provider: {0}")] + InvalidProvider(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("unsupported by the rust path: {0}")] + Unsupported(&'static str), + #[error(transparent)] + Auth(#[from] litellm_auth::Error), + #[error(transparent)] + Transport(#[from] crate::transport::Error), + #[error(transparent)] + Headers(#[from] crate::http_utils::HeaderError), + #[error(transparent)] + Aws(#[from] litellm_auth_aws::Error), +} diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index 9a96b9d1140..bd1740a8b93 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -1,12 +1,11 @@ use serde_json::Value; -use crate::error::Error; +use super::Error; use crate::http_utils::{http_request, truncate_error_body}; use super::client::http_client; use super::types::ProviderAudioTranscriptionRequest; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, ) -> Result { @@ -22,17 +21,17 @@ pub async fn execute_audio_transcription_provider_call( } let response = http_request(request_builder) .await - .map_err(|error| Error::Network(error.to_string()))?; + .map_err(|error| Error::Transport(crate::transport::Error::Network(error.to_string())))?; let status = response.status(); let text = response .text() .await - .map_err(|error| Error::Network(error.to_string()))?; + .map_err(|error| Error::Transport(crate::transport::Error::Network(error.to_string())))?; if !status.is_success() { - return Err(Error::Http { + return Err(Error::Transport(crate::transport::Error::Http { status: status.as_u16(), body: truncate_error_body(&text), - }); + })); } let response_json = serde_json::from_str(&text) .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; @@ -42,7 +41,6 @@ pub async fn execute_audio_transcription_provider_call( .into_json()) } -#[cfg(feature = "bedrock-auth")] async fn signed_headers( request: &ProviderAudioTranscriptionRequest, body: &[u8], @@ -74,18 +72,3 @@ async fn signed_headers( )?; Ok(unsigned.into_iter().chain(signature).collect()) } - -#[cfg(not(feature = "bedrock-auth"))] -async fn signed_headers( - request: &ProviderAudioTranscriptionRequest, - _body: &[u8], -) -> Result, Error> { - use crate::audio_transcription::transformation::AudioTranscriptionAuth; - - match request.auth { - AudioTranscriptionAuth::AwsSigV4 { .. } => Err(Error::Unsupported( - "AWS SigV4 requires the bedrock-auth feature", - )), - AudioTranscriptionAuth::Bearer => Ok(request.upstream_headers.clone()), - } -} diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index 31b6de4b3e4..87f6c41d80f 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -1,4 +1,5 @@ -use crate::Error; +mod error; +pub use error::Error; mod client; mod handler; mod prepare; @@ -11,7 +12,6 @@ pub use handler::execute_audio_transcription_provider_call; pub use prepare::prepare_audio_transcription_provider_call; pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) .await diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index bbef97341a9..82f85ba85ce 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -1,15 +1,12 @@ -use crate::error::Error; +use super::Error; use crate::http_utils::{has_header, string_headers}; -#[cfg(feature = "bedrock-auth")] use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProviderConfig> { - #[cfg(feature = "bedrock-auth")] if provider == "bedrock" { return Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG); } @@ -17,7 +14,6 @@ fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProv None } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub fn prepare_audio_transcription_provider_call( request: AudioTranscriptionRequest<'_>, ) -> Result { @@ -67,7 +63,6 @@ pub fn prepare_audio_transcription_provider_call( body: transformed.body, upstream_headers: headers, auth, - #[cfg(feature = "bedrock-auth")] optional_params: request.optional_params, timeout: request.timeout, }) diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index aa9846427dc..a849f052e12 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -1,4 +1,4 @@ -use crate::Error; +use super::Error; use serde_json::{Map, Value}; use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData}; @@ -15,7 +15,6 @@ pub enum AudioTranscriptionAuth { pub trait AudioTranscriptionProviderConfig: Sync { fn supported_transcription_params(&self) -> &'static [&'static str]; - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_transcription_params(&self, params: &Map) -> Map { params .iter() diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs index 559d7837027..1f90f61c0da 100644 --- a/litellm-rust/crates/core/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -25,7 +25,6 @@ pub struct ProviderAudioTranscriptionRequest { pub(super) body: Value, pub(super) upstream_headers: Vec<(String, String)>, pub(super) auth: AudioTranscriptionAuth, - #[cfg(feature = "bedrock-auth")] pub(super) optional_params: Map, pub(super) timeout: Option, } diff --git a/litellm-rust/crates/core/src/auth/error.rs b/litellm-rust/crates/core/src/auth/error.rs deleted file mode 100644 index e7027c0df10..00000000000 --- a/litellm-rust/crates/core/src/auth/error.rs +++ /dev/null @@ -1,128 +0,0 @@ -use thiserror::Error; - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum AuthError { - #[error("invalid authentication configuration: {0}")] - Configuration(#[from] AuthConfigurationError), - #[error("credential acquisition failed: {0}")] - AzureTokenAcquisition(String), - #[error("credential acquisition failed: Vertex AI credentials: {0}")] - VertexTokenAcquisition(String), - #[error("credential acquisition failed: {}", .0.iter().map(ToString::to_string).collect::>().join("; "))] - CredentialChain(Vec), - #[error("credential caller failed: credential caller returned an empty credential")] - EmptyCallerCredential, - #[error("credential caller failed: Azure AD token provider returned an empty token")] - EmptyAzureToken, - #[error("credential acquisition failed: Azure OIDC reference did not resolve to a value")] - UnresolvedOidcReference, - #[error( - "Missing {provider} API Key - A call is being made to {provider} but no key is set either in the environment variables or via params" - )] - MissingApiKey { provider: &'static str }, - #[error( - "Missing {provider} API Base - Set {environment_variable} environment variable or pass api_base parameter" - )] - MissingApiBase { - provider: &'static str, - environment_variable: &'static str, - }, - #[error("{0}")] - MissingCredential(#[from] MissingCredential), - #[error("{0}")] - Aws(#[from] AwsAuthError), - #[error("invalid authentication header")] - InvalidHeader, -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum AuthConfigurationError { - #[error("credential header already exists")] - ExistingCredentialHeader, - #[error("credential plan is not allowed by the provider auth policy")] - DisallowedCredentialPlan, - #[error("credential cannot be empty")] - EmptyCredential, - #[error("invalid Azure credential selector")] - InvalidAzureSelector, - #[error("ClientSecretCredential requires tenant_id, client_id, and client_secret")] - MissingClientSecretFields, - #[error("WorkloadIdentityCredential requires tenant_id")] - MissingWorkloadTenant, - #[error("WorkloadIdentityCredential requires client_id")] - MissingWorkloadClient, - #[error("WorkloadIdentityCredential requires azure_federated_token_file")] - MissingWorkloadTokenFile, - #[error("credential reference requires a host credential resolver")] - MissingHostResolver, - #[error("caller credential plan requires provider-specific inputs")] - MissingCallerInputs, - #[error("credential header {0} already exists")] - DuplicateHeader(&'static str), - #[error("{0} must be a string or null")] - InvalidFieldType(String), - #[error("unsupported OIDC reference")] - UnsupportedOidcReference, - #[error("{0} cannot be empty")] - EmptyReference(String), - #[error("Azure credential initialization failed: {0}")] - AzureCredentialInitialization(String), - #[error("Azure authority must be an HTTPS origin without credentials, query, or fragment")] - InvalidAzureAuthority, - #[error("request-controlled Azure auth inputs cannot be combined with host credentials")] - MixedAzureCredentialSources, - #[error("request-controlled Azure credential references are not allowed")] - RequestAzureCredentialReference, - #[error("host credentials cannot be sent to a request-controlled Azure endpoint")] - RequestAzureCredentialDestination, - #[error("credentials cannot be sent to a request-controlled Vertex AI endpoint")] - RequestVertexCredentialDestination, - #[error( - "request-controlled Vertex credentials must use the canonical Google OAuth token endpoint" - )] - RequestVertexTokenEndpoint, -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum MissingCredential { - #[error( - "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable" - )] - AnthropicApiKey, - #[error("Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable")] - AzureApiKey, - #[error( - "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. Expected format: https://.services.ai.azure.com/anthropic" - )] - AzureApiBase, - #[error( - "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable" - )] - OpenAiRealtimeApiKey, - #[error( - "Missing OpenAI API Key - a Responses WebSocket call is being made but no key was passed via params or the OPENAI_API_KEY environment variable" - )] - OpenAiResponsesApiKey, -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum AwsAuthError { - #[error("AWS profile credentials failed: {0}")] - Profile(String), - #[error("AWS default credentials failed: {0}")] - DefaultChain(String), - #[error("AWS role credentials failed: {0}")] - AssumeRole(String), - #[error("AWS web identity credentials failed: {0}")] - WebIdentity(String), - #[error("AWS web identity expiration was invalid: {0}")] - WebIdentityExpiration(String), - #[error("AWS signing parameters failed: {0}")] - SigningParameters(String), - #[error("AWS signable request failed: {0}")] - SignableRequest(String), - #[error("AWS request signing failed: {0}")] - Signing(String), - #[error("AWS web identity response had no credentials")] - MissingWebIdentityCredentials, -} diff --git a/litellm-rust/crates/core/src/caching/in_memory_cache.rs b/litellm-rust/crates/core/src/caching/in_memory_cache.rs deleted file mode 100644 index 45d4bd69b79..00000000000 --- a/litellm-rust/crates/core/src/caching/in_memory_cache.rs +++ /dev/null @@ -1,258 +0,0 @@ -use std::cmp::Reverse; -use std::collections::{BinaryHeap, HashMap}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200; -const DEFAULT_TTL: Duration = Duration::from_secs(600); - -pub struct InMemoryCache { - pub cache_dict: HashMap, - pub ttl_dict: HashMap, - pub expiration_heap: BinaryHeap>, - pub max_size_in_memory: usize, - pub default_ttl: Duration, - now: Box Duration + Send + Sync>, -} - -impl Default for InMemoryCache { - fn default() -> Self { - Self::new(None, None) - } -} - -impl InMemoryCache { - pub fn new(max_size_in_memory: Option, default_ttl: Option) -> Self { - Self::with_clock(max_size_in_memory, default_ttl, || { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - }) - } - - pub fn with_clock( - max_size_in_memory: Option, - default_ttl: Option, - now: impl Fn() -> Duration + Send + Sync + 'static, - ) -> Self { - Self { - cache_dict: HashMap::new(), - ttl_dict: HashMap::new(), - expiration_heap: BinaryHeap::new(), - max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), - default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), - now: Box::new(now), - } - } - - pub fn evict_cache(&mut self) { - if self.max_size_in_memory == 0 { - return; - } - - let current_time = (self.now)(); - while let Some(Reverse((expiration_time, key))) = self.expiration_heap.peek().cloned() { - if self.ttl_dict.get(&key).copied() != Some(expiration_time) { - self.expiration_heap.pop(); - } else if expiration_time <= current_time { - self.expiration_heap.pop(); - self.remove_key(&key); - } else { - break; - } - } - - while self.cache_dict.len() >= self.max_size_in_memory { - let Some(Reverse((expiration_time, key))) = self.expiration_heap.pop() else { - break; - }; - if self.ttl_dict.get(&key).copied() == Some(expiration_time) { - self.remove_key(&key); - } - } - } - - pub fn allow_ttl_override(&self, key: &str) -> bool { - match self.ttl_dict.get(key).copied() { - None => true, - Some(expiration_time) => expiration_time < (self.now)(), - } - } - - pub fn set_cache(&mut self, key: impl Into, value: V, ttl: Option) { - if self.max_size_in_memory == 0 { - return; - } - - self.evict_cache(); - let key = key.into(); - self.cache_dict.insert(key.clone(), value); - if self.allow_ttl_override(&key) { - let expiration_time = (self.now)() + ttl.unwrap_or(self.default_ttl); - self.ttl_dict.insert(key.clone(), expiration_time); - self.expiration_heap.push(Reverse((expiration_time, key))); - } - } - - // Generic values intentionally omit Python's per-item size check. - pub fn get_cache(&mut self, key: &str) -> Option { - if self.cache_dict.contains_key(key) { - if self.is_key_expired(key) { - self.remove_key(key); - return None; - } - return self.cache_dict.get(key).cloned(); - } - None - } - - pub fn get_ttl(&self, key: &str) -> Option { - self.ttl_dict.get(key).copied() - } - - pub fn delete_cache(&mut self, key: &str) { - self.remove_key(key); - } - - pub fn flush_cache(&mut self) { - self.cache_dict.clear(); - self.ttl_dict.clear(); - self.expiration_heap.clear(); - } - - fn is_key_expired(&self, key: &str) -> bool { - self.ttl_dict - .get(key) - .is_some_and(|expiration_time| *expiration_time < (self.now)()) - } - - fn remove_key(&mut self, key: &str) { - self.cache_dict.remove(key); - self.ttl_dict.remove(key); - } -} - -#[cfg(test)] -mod tests { - use std::sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }; - - use super::InMemoryCache; - use std::time::Duration; - - fn cache(now: Arc, max_size: usize, default_ttl: Duration) -> InMemoryCache { - InMemoryCache::with_clock(Some(max_size), Some(default_ttl), move || { - Duration::from_secs(now.load(Ordering::Relaxed)) - }) - } - - #[test] - fn ttl_expiry_is_deterministic() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now.clone(), 10, Duration::from_secs(60)); - cache.set_cache("key", "value".to_string(), None); - assert_eq!(cache.get_cache("key"), Some("value".to_string())); - now.store(161, Ordering::Relaxed); - assert_eq!(cache.get_cache("key"), None); - assert_eq!(cache.get_ttl("key"), None); - } - - #[test] - fn default_and_per_set_ttl_are_applied() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now.clone(), 10, Duration::from_secs(60)); - cache.set_cache("default", "value".to_string(), None); - cache.set_cache("custom", "value".to_string(), Some(Duration::from_secs(20))); - assert_eq!(cache.get_ttl("default"), Some(Duration::from_secs(160))); - assert_eq!(cache.get_ttl("custom"), Some(Duration::from_secs(120))); - } - - #[test] - fn unexpired_entries_do_not_allow_ttl_override() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now.clone(), 10, Duration::from_secs(60)); - cache.set_cache("key", "first".to_string(), Some(Duration::from_secs(20))); - cache.set_cache("key", "second".to_string(), Some(Duration::from_secs(80))); - assert_eq!(cache.get_cache("key"), Some("second".to_string())); - assert_eq!(cache.get_ttl("key"), Some(Duration::from_secs(120))); - now.store(121, Ordering::Relaxed); - cache.set_cache("key", "third".to_string(), Some(Duration::from_secs(80))); - assert_eq!(cache.get_ttl("key"), Some(Duration::from_secs(201))); - } - - #[test] - fn max_size_evicts_earliest_expiration() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now, 2, Duration::from_secs(60)); - cache.set_cache("early", "value".to_string(), Some(Duration::from_secs(10))); - cache.set_cache("late", "value".to_string(), Some(Duration::from_secs(20))); - cache.set_cache("new", "value".to_string(), Some(Duration::from_secs(30))); - assert_eq!(cache.get_cache("early"), None); - assert!(cache.get_cache("late").is_some()); - assert!(cache.get_cache("new").is_some()); - } - - #[test] - fn expired_entries_are_evicted_before_live_entries() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now.clone(), 3, Duration::from_secs(60)); - cache.set_cache( - "expired-one", - "value".to_string(), - Some(Duration::from_secs(10)), - ); - cache.set_cache( - "expired-two", - "value".to_string(), - Some(Duration::from_secs(20)), - ); - cache.set_cache("live", "value".to_string(), Some(Duration::from_secs(100))); - now.store(121, Ordering::Relaxed); - cache.set_cache("new", "value".to_string(), Some(Duration::from_secs(100))); - assert_eq!(cache.get_cache("expired-one"), None); - assert_eq!(cache.get_cache("expired-two"), None); - assert!(cache.get_cache("live").is_some()); - assert!(cache.get_cache("new").is_some()); - } - - #[test] - fn stale_heap_entries_are_skipped() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now, 1, Duration::from_secs(60)); - cache.set_cache( - "removed", - "value".to_string(), - Some(Duration::from_secs(10)), - ); - cache.delete_cache("removed"); - cache.set_cache("kept", "value".to_string(), Some(Duration::from_secs(20))); - cache.set_cache("new", "value".to_string(), Some(Duration::from_secs(30))); - assert_eq!(cache.get_cache("removed"), None); - assert_eq!(cache.get_cache("kept"), None); - assert!(cache.get_cache("new").is_some()); - } - - #[test] - fn delete_and_flush_remove_values_and_ttls() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now, 10, Duration::from_secs(60)); - cache.set_cache("one", "value".to_string(), None); - cache.set_cache("two", "value".to_string(), None); - cache.delete_cache("one"); - assert_eq!(cache.get_cache("one"), None); - cache.flush_cache(); - assert!(cache.cache_dict.is_empty()); - assert!(cache.ttl_dict.is_empty()); - assert!(cache.expiration_heap.is_empty()); - } - - #[test] - fn zero_max_size_does_not_cache() { - let now = Arc::new(AtomicU64::new(100)); - let mut cache = cache(now, 0, Duration::from_secs(60)); - cache.set_cache("key", "value".to_string(), None); - assert_eq!(cache.get_cache("key"), None); - assert!(cache.cache_dict.is_empty()); - } -} diff --git a/litellm-rust/crates/core/src/caching/mod.rs b/litellm-rust/crates/core/src/caching/mod.rs deleted file mode 100644 index 5fb8a0e5174..00000000000 --- a/litellm-rust/crates/core/src/caching/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod in_memory_cache; diff --git a/litellm-rust/crates/core/src/call_lifecycle/host.rs b/litellm-rust/crates/core/src/call_lifecycle/host.rs index ac6ddf99b9e..97eb9c4c650 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/host.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/host.rs @@ -6,10 +6,11 @@ pub enum HostCallStep { Complete(C), } -pub type HostCallFuture<'a, O, C> = - Pin, crate::Error>> + Send + 'a>>; +pub type HostCallFuture<'a, O, C, E> = + Pin, E>> + Send + 'a>>; pub trait HostCall: Send + Sync { + type Error: Send + Sync + 'static; type Operation: Send + 'static; type Result: Send + 'static; type Complete: Send + 'static; @@ -17,12 +18,12 @@ pub trait HostCall: Send + Sync { fn resume( &mut self, result: Option, - ) -> HostCallFuture<'_, Self::Operation, Self::Complete>; + ) -> HostCallFuture<'_, Self::Operation, Self::Complete, Self::Error>; fn interrupt( &mut self, - failure: HostFailure, - ) -> HostCallFuture<'_, Self::Operation, Self::Complete>; + failure: HostFailure, + ) -> HostCallFuture<'_, Self::Operation, Self::Complete, Self::Error>; } pub enum HostStep { @@ -48,9 +49,9 @@ pub enum HostPhase { } #[derive(Clone, Debug)] -pub enum HostFailure { - Error(crate::Error), - Cancelled(crate::Error), +pub enum HostFailure { + Error(E), + Cancelled(E), } pub struct HostLifecycle { @@ -70,7 +71,7 @@ impl HostLifecycle { self.phase } - pub fn accept(&mut self, result: Result<(), HostFailure>) -> Option { + pub fn accept(&mut self, result: Result<(), HostFailure>) -> Option { if let Err(failure) = result { if self.phase == HostPhase::DeploymentFailure { self.phase = HostPhase::Failure; diff --git a/litellm-rust/crates/core/src/call_lifecycle/mod.rs b/litellm-rust/crates/core/src/call_lifecycle/mod.rs index 5c752a73899..dce240c3d2b 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/mod.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/mod.rs @@ -1,8 +1,6 @@ use std::future::Future; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::Error; - pub mod host; #[cfg(test)] #[path = "../../tests/host_lifecycle.rs"] @@ -15,14 +13,15 @@ pub use types::{ }; pub trait CallLifecycleHooks: Send + Sync { - type PreCallFuture<'a>: Future> + Send + 'a + type Error: Send + Sync; + type PreCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, ProviderReq: 'a, Resp: 'a; - type DuringCallFuture<'a>: Future> + Send + 'a + type DuringCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, @@ -60,7 +59,7 @@ pub trait CallLifecycleHooks: Send + Sync { fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a Error, + error: &'a Self::Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a>; } @@ -90,12 +89,12 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> Result + ) -> Result where InitialReq: CallLifecycleRequest, Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let context = request.lifecycle_context(); self.run(context, request, hooks, provider_call).await @@ -107,11 +106,11 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> Result + ) -> Result where Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let call_start = epoch_seconds(); let mut phases = Vec::new(); @@ -170,7 +169,7 @@ impl<'a> CallLifecycle<'a> { &self, context: &CallLifecycleContext, hooks: &Hooks, - error: &Error, + error: &Hooks::Error, call_start: f64, phases: &mut Vec, ) where @@ -255,8 +254,9 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, Result>; - type DuringCallFuture<'a> = BoxFuture<'a, Result>; + type Error = crate::messages::Error; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -298,7 +298,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a Error, + _error: &'a crate::messages::Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -308,8 +308,9 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, Result>; - type DuringCallFuture<'a> = BoxFuture<'a, Result>; + type Error = crate::messages::Error; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -349,7 +350,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a Error, + _error: &'a crate::messages::Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -387,13 +388,20 @@ mod tests { "request".to_string(), &hooks, |_request| async move { - Err::(Error::Network("provider down".to_string())) + Err::(crate::messages::Error::Transport( + crate::transport::Error::Network("provider down".to_string()), + )) }, ) .await .expect_err("call fails"); - assert_eq!(error, Error::Network("provider down".to_string())); + assert_eq!( + error, + crate::messages::Error::Transport(crate::transport::Error::Network( + "provider down".to_string() + )) + ); assert_eq!(hooks.events(), vec!["pre_call", "during_call", "failure"]); } diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 69e5f175ad5..9ebc5ae0efa 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,4 +1,4 @@ -use crate::Error; +use super::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG; use serde_json::{Map, Value}; @@ -7,13 +7,11 @@ use super::transformation::ChatCompletionsProviderConfig; const HEADER_CONTEXT: &str = "chat completions"; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn chat_completions_provider_config( provider: &str, ) -> Option<&'static dyn ChatCompletionsProviderConfig> { match provider { "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG), - #[cfg(feature = "bedrock-auth")] "bedrock" => Some( &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, ), @@ -24,5 +22,5 @@ pub(super) fn chat_completions_provider_config( pub(super) fn string_headers( extra_headers: Option>, ) -> Result, Error> { - shared_string_headers(HEADER_CONTEXT, extra_headers) + shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from) } diff --git a/litellm-rust/crates/core/src/chat_completions/error.rs b/litellm-rust/crates/core/src/chat_completions/error.rs new file mode 100644 index 00000000000..f9ffb12d349 --- /dev/null +++ b/litellm-rust/crates/core/src/chat_completions/error.rs @@ -0,0 +1,26 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid provider: {0}")] + InvalidProvider(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("unsupported by the rust path: {0}")] + Unsupported(&'static str), + #[error(transparent)] + Auth(#[from] litellm_auth::Error), + #[error(transparent)] + Transport(#[from] crate::transport::Error), + #[error(transparent)] + Headers(#[from] crate::http_utils::HeaderError), + #[error(transparent)] + Aws(#[from] litellm_auth_aws::Error), +} diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index 96d001e2892..d4527e99a10 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::Error; +use super::Error; use crate::http_utils::{http_request, truncate_error_body}; use super::client::http_client; @@ -11,7 +11,6 @@ use super::types::{ ResolvedChatCompletionsRequest, }; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) async fn execute_chat_completions_provider_call( request: ResolvedChatCompletionsRequest<'_>, ) -> Result { @@ -36,9 +35,9 @@ pub(super) async fn execute_chat_completions_provider_call( // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. if err.is_connect() || err.is_builder() { - Error::Connect(err.to_string()) + Error::Transport(crate::transport::Error::Connect(err.to_string())) } else { - Error::Network(err.to_string()) + Error::Transport(crate::transport::Error::Network(err.to_string())) } })?; @@ -46,13 +45,13 @@ pub(super) async fn execute_chat_completions_provider_call( let text = response .text() .await - .map_err(|err| Error::Network(err.to_string()))?; + .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; if !status.is_success() { - return Err(Error::Http { + return Err(Error::Transport(crate::transport::Error::Http { status: status.as_u16(), body: truncate_error_body(&text), - }); + })); } let body: Value = serde_json::from_str(&text).map_err(|err| { @@ -75,12 +74,12 @@ pub(super) async fn execute_chat_completions_provider_call( /// can only mean the provider was already called. pub(super) fn as_response_error(err: Error) -> Error { match err { - already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, + already @ (Error::InvalidResponse(_) + | Error::Transport(crate::transport::Error::Http { .. })) => already, other => Error::InvalidResponse(other.to_string()), } } -#[cfg(feature = "bedrock-auth")] pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, body: &[u8], @@ -136,16 +135,3 @@ pub(super) async fn signed_headers( // that would collide, so no name appears twice. Ok(unsigned.into_iter().chain(signature).collect()) } - -#[cfg(not(feature = "bedrock-auth"))] -pub(super) async fn signed_headers( - request: &ProviderChatCompletionsRequest, - _body: &[u8], -) -> Result, Error> { - match &request.auth { - ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported( - "AWS SigV4 requires the bedrock-auth feature", - )), - _ => Ok(request.upstream_headers.clone()), - } -} diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 32dea17d202..401eef609f2 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -6,7 +6,8 @@ //! credentials, and it resolves the provider, translates the conversation, //! calls the provider, and returns a typed OpenAI-shaped response. -use crate::Error; +mod error; +pub use error::Error; mod client; mod common_utils; pub mod conversation; @@ -22,7 +23,6 @@ use handler::execute_chat_completions_provider_call; use prepare::{parse_messages, resolve_provider_config, resolve_request}; use types::{ChatCompletionsRequest, ChatCompletionsResponse}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn chat_completions( request: ChatCompletionsRequest<'_>, ) -> Result { diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 3be2ba21de4..e8d8d70f271 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,8 +1,8 @@ use serde_json::Value; -use crate::error::Error; +use super::Error; use crate::http_utils::has_header; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{chat_completions_provider_config, string_headers}; use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig}; @@ -62,7 +62,6 @@ pub(super) fn resolve_request( }) } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn validate_environment( request: &ResolvedChatCompletionsRequest<'_>, model: &str, diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index f8594dee447..39fabe27f44 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value, json}; -use crate::error::Error; +use super::Error; use super::prepare::{prepare_provider_request, resolve_request}; use super::transformation::ChatCompletionsAuth; @@ -264,13 +264,14 @@ fn rejects_non_string_extra_headers() { call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); assert_eq!( decline(call), - Error::InvalidRequest( - "chat completions extra_headers.x-trace must be a string, got number".to_string() - ) + Error::Headers(crate::http_utils::HeaderError { + context: "chat completions", + name: "x-trace".to_string(), + actual: "number", + }) ); } -#[cfg(feature = "bedrock-auth")] #[test] fn prepares_a_bedrock_call_without_resolving_credentials() { let mut call = request( @@ -302,7 +303,6 @@ fn prepares_a_bedrock_call_without_resolving_credentials() { assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16})); } -#[cfg(feature = "bedrock-auth")] #[tokio::test] async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() { // Python signs only the AWS header set and reattaches the rest, so a header @@ -351,7 +351,6 @@ async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() { ); } -#[cfg(feature = "bedrock-auth")] #[tokio::test] async fn a_forwarded_header_the_signer_computes_declines_to_python() { // Reattaching the caller's copy next to the computed one puts the name on @@ -386,7 +385,6 @@ async fn a_forwarded_header_the_signer_computes_declines_to_python() { } } -#[cfg(feature = "bedrock-auth")] #[test] fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() { // `get_request_headers` assigns `headers["Authorization"]` unconditionally @@ -453,7 +451,6 @@ fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() { ); } -#[cfg(feature = "bedrock-auth")] #[test] fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() { // The configured bearer identity has its own account and quota boundary, @@ -769,7 +766,10 @@ mod round_trip { .expect_err("upstream rejects"); handle.await.expect("server task"); assert!( - matches!(err, Error::Http { status: 429, .. }), + matches!( + err, + Error::Transport(crate::transport::Error::Http { status: 429, .. }) + ), "expected a 429, got {err:?}" ); } @@ -793,7 +793,7 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, Error::Connect(_)), + matches!(err, Error::Transport(crate::transport::Error::Connect(_))), "expected a pre-send connect failure, got {err:?}" ); } @@ -806,7 +806,7 @@ mod round_trip { Error::MissingField("usage"), Error::Unsupported("non-text response content block"), Error::InvalidRequest("whatever".to_string()), - Error::Auth("whatever".to_string()), + Error::Auth(litellm_auth::Error::InvalidHeader), ] { let label = format!("{original:?}"); assert!( @@ -816,11 +816,11 @@ mod round_trip { } // An upstream status is already unambiguous, so it survives intact. assert!(matches!( - as_response_error(Error::Http { + as_response_error(Error::Transport(crate::transport::Error::Http { status: 500, body: "boom".to_string() - }), - Error::Http { status: 500, .. } + })), + Error::Transport(crate::transport::Error::Http { status: 500, .. }) )); } } diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index d7b9704c46c..1000dbaa673 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -1,4 +1,4 @@ -use crate::Error; +use super::Error; use serde_json::{Map, Value}; use super::types::{ diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs index 3238d09b6b5..7178d594870 100644 --- a/litellm-rust/crates/core/src/chat_completions/types.rs +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -40,7 +40,6 @@ pub(super) struct ProviderChatCompletionsRequest { pub(super) body: Value, pub(super) upstream_headers: Vec<(String, String)>, pub(super) auth: ChatCompletionsAuth, - #[cfg_attr(not(feature = "bedrock-auth"), allow(dead_code))] pub(super) optional_params: Map, pub(super) timeout: Option, } diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 1babb0078b8..4ff4333c4ac 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -42,8 +42,6 @@ pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion"; pub const EMPTY_TEXT_PLACEHOLDER: &str = "[System: Empty message content sanitised to satisfy protocol]"; -pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; - pub(crate) const MEDIA_CONNECT_TIMEOUT_SECS: u64 = 10; pub(crate) const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 359ad56c336..15d27602052 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -1,220 +1,13 @@ -use thiserror::Error as ThisError; - -#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[error("expected {expected}, got {actual}")] - InvalidType { - expected: &'static str, - actual: &'static str, - }, - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("Document URL is required")] - MissingDocumentUrl, - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error("invalid provider: {0}")] - InvalidProvider(String), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("{0}")] - Auth(String), - #[error( - "Missing {provider} API Key - A call is being made to {provider} but no key is set either in the environment variables or via params" - )] - MissingApiKey { provider: &'static str }, - #[error( - "invalid authentication configuration: Missing Azure AI credentials - set AZURE_AI_API_KEY or configure Entra ID" - )] - MissingAzureAiCredentials, - #[error( - "invalid authentication configuration: Missing Azure Document Intelligence credentials - set AZURE_DOCUMENT_INTELLIGENCE_API_KEY or configure Entra ID" - )] - MissingAzureDocumentIntelligenceCredentials, - #[error( - "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()" - )] - MissingReductoApiKey, - #[error("upstream request failed with status {status}: {body}")] - Http { status: u16, body: String }, - #[error("upstream network error: {0}")] - Network(String), - /// The provider was never reached: DNS, TCP, TLS or proxy setup failed - /// before any byte of the request went out. Nothing was billed, so a host - /// that keeps a reference implementation can serve the request itself. - /// A timeout is deliberately not this, since the provider may have received - /// and answered the request already. - #[error("could not reach the provider: {0}")] - Connect(String), - #[error("routing error: {0}")] - Routing(String), - /// The request is outside the surface this route covers in Rust. Hosts that - /// keep a reference implementation treat this as "fall back", not "fail". - #[error("unsupported by the rust path: {0}")] - Unsupported(&'static str), -} - -impl Error { - pub const fn http_status_code(&self) -> Option { - match self { - Self::InvalidRequest(_) => Some(400), - Self::MissingDocumentUrl => Some(500), - Self::Http { status, .. } => Some(*status), - _ => None, - } - } -} - -#[derive(Debug, ThisError)] -pub(crate) enum MediaError { - #[error("media URL rejected by network policy")] - BlockedUrl, - #[error("media download is disabled")] - DownloadDisabled, - #[error("media download exceeds the maximum size")] - DownloadTooLarge, - #[error("too many redirects while fetching media")] - TooManyRedirects, - #[error("media redirect is missing a Location header")] - MissingRedirectLocation, - #[error("invalid media redirect")] - InvalidRedirect, - #[error("media download failed with status {0}")] - Http(u16), - #[error("media download timed out")] - Timeout, - #[error("{0}")] - Transport(#[from] TransportError), -} - -#[derive(Clone, Debug, ThisError, PartialEq, Eq)] -pub enum TransportError { - #[error("upstream request failed with status {status}: {body}")] - Http { status: u16, body: String }, - #[error("upstream network error: {0}")] - Network(String), - #[error("could not reach the provider: {0}")] - Connect(String), -} - -impl TransportError { - pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self { - let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder()); - let message = error.without_url().to_string(); - if before_dispatch { - Self::Connect(message) - } else { - Self::Network(message) - } - } -} - -impl From for TransportError { - fn from(error: reqwest::Error) -> Self { - Self::Network(error.without_url().to_string()) - } -} - -impl From for Error { - fn from(error: crate::ocr::error::OcrRequestError) -> Self { - match error { - crate::ocr::error::OcrRequestError::MissingField(field) => Self::MissingField(field), - crate::ocr::error::OcrRequestError::MissingDocumentUrl => Self::MissingDocumentUrl, - error => Self::InvalidRequest(error.to_string()), - } - } -} - -impl From for Error { - fn from(error: crate::ocr::error::OcrResponseError) -> Self { - Self::InvalidResponse(error.to_string()) - } -} - -impl From for Error { - fn from(error: TransportError) -> Self { - match error { - TransportError::Http { status, body } => Self::Http { status, body }, - TransportError::Network(message) => Self::Network(message), - TransportError::Connect(message) => Self::Connect(message), - } - } -} - -impl From for Error { - fn from(error: crate::AuthError) -> Self { - match error { - crate::AuthError::MissingApiKey { provider } => Self::MissingApiKey { provider }, - error => Self::Auth(error.to_string()), - } - } -} - -pub fn json_type_name(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", - } -} - -#[cfg(test)] -mod transport_tests { - use super::*; - - #[test] - fn missing_auth_key_preserves_provider_in_public_error() { - assert_eq!( - Error::from(crate::AuthError::MissingApiKey { provider: "Vertex" }), - Error::MissingApiKey { provider: "Vertex" } - ); - } - - #[tokio::test] - async fn transport_errors_remove_urls_and_keep_dispatch_context() { - let error = reqwest::Client::builder() - .no_proxy() - .build() - .expect("client") - .get("http://localhost:invalid/private?api_key=secret") - .send() - .await - .expect_err("invalid port"); - let error = TransportError::from_reqwest_before_dispatch(error); - assert!(matches!(error, TransportError::Connect(_))); - assert!(!error.to_string().contains("secret")); - assert!(!error.to_string().contains("private")); - } - - #[tokio::test] - async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() { - use std::time::Duration; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind"); - let address = listener.local_addr().expect("address"); - let request = reqwest::Client::builder() - .no_proxy() - .build() - .expect("client") - .get(format!("http://{address}")) - .timeout(Duration::from_millis(200)) - .send(); - let (response, accepted) = tokio::join!( - request, - tokio::time::timeout(Duration::from_secs(2), listener.accept()) - ); - let _connection = accepted - .expect("accept deadline") - .expect("accepted connection"); - let error = response.expect_err("server does not respond"); - assert!(error.is_timeout()); - assert!(matches!( - TransportError::from_reqwest_before_dispatch(error), - TransportError::Network(_) - )); - } + #[error(transparent)] + Ocr(#[from] crate::ocr::Error), + #[error(transparent)] + Messages(#[from] crate::messages::Error), + #[error(transparent)] + ChatCompletions(#[from] crate::chat_completions::Error), + #[error(transparent)] + AudioTranscription(#[from] crate::audio_transcription::Error), + #[error(transparent)] + Responses(#[from] crate::responses::Error), } diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index 9299bb77ac8..53d2f961bd5 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -1,7 +1,14 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid request: {context} extra_headers.{name} must be a string, got {actual}")] +pub struct HeaderError { + pub context: &'static str, + pub name: String, + pub actual: &'static str, +} + use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; -use crate::error::{Error, json_type_name}; #[allow( dead_code, @@ -38,13 +45,19 @@ pub(crate) fn with_headers( }) } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn http_request( request: reqwest::RequestBuilder, ) -> Result { request.send().await } +pub async fn execute_http_request( + client: &reqwest::Client, + request: reqwest::Request, +) -> Result { + client.execute(request).await +} + pub fn truncate_error_body(body: &str) -> String { if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS { return body.to_string(); @@ -56,7 +69,7 @@ pub fn truncate_error_body(body: &str) -> String { pub fn string_headers( context: &'static str, extra_headers: Option>, -) -> Result, Error> { +) -> Result, HeaderError> { extra_headers .unwrap_or_default() .into_iter() @@ -64,11 +77,10 @@ pub fn string_headers( value .as_str() .map(|value| (key.clone(), value.to_string())) - .ok_or_else(|| { - Error::InvalidRequest(format!( - "{context} extra_headers.{key} must be a string, got {}", - json_type_name(&value) - )) + .ok_or_else(|| HeaderError { + context, + name: key, + actual: json_type_name(&value), }) }) .collect() @@ -106,6 +118,17 @@ where as serde::Deserialize>::deserialize(deserializer).map(Some) } +pub fn json_type_name(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} + #[cfg(test)] mod tests { use super::*; @@ -185,9 +208,11 @@ mod tests { let err = string_headers("chat completions", Some(headers)).expect_err("non-string value"); assert_eq!( err, - Error::InvalidRequest( - "chat completions extra_headers.x-trace must be a string, got number".to_string() - ) + HeaderError { + context: "chat completions", + name: "x-trace".into(), + actual: "number" + } ); } diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 0b3573deab2..b028b7bc9b1 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,6 +1,4 @@ pub mod audio_transcription; -pub mod auth; -pub mod caching; pub mod call_lifecycle; pub mod chat_completions; pub mod constants; @@ -8,15 +6,10 @@ pub mod error; pub mod http_utils; mod media; pub mod messages; -#[cfg(any(feature = "observability", test))] -pub mod observability; pub mod ocr; pub mod providers; -pub mod realtime; pub mod responses; -pub mod router; -pub mod routing_utils; +pub mod transport; mod url_utils; -pub use auth::AuthError; pub use error::Error; diff --git a/litellm-rust/crates/core/src/media.rs b/litellm-rust/crates/core/src/media.rs index 5f9a43794c2..ba26f431e57 100644 --- a/litellm-rust/crates/core/src/media.rs +++ b/litellm-rust/crates/core/src/media.rs @@ -9,7 +9,28 @@ use reqwest::Url; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use crate::constants::MEDIA_CONNECT_TIMEOUT_SECS; -use crate::error::{MediaError, TransportError}; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("media URL rejected by network policy")] + BlockedUrl, + #[error("media download is disabled")] + DownloadDisabled, + #[error("media download exceeds the maximum size")] + DownloadTooLarge, + #[error("too many redirects while fetching media")] + TooManyRedirects, + #[error("media redirect is missing a Location header")] + MissingRedirectLocation, + #[error("invalid media redirect")] + InvalidRedirect, + #[error("media download failed with status {0}")] + Http(u16), + #[error("media download timed out")] + Timeout, + #[error("{0}")] + Transport(#[from] crate::transport::Error), +} #[derive(Clone)] pub(crate) struct MediaFetcher { @@ -75,20 +96,20 @@ impl MediaFetcher { &self, url: Url, policy: DownloadPolicy, - ) -> Result { + ) -> Result { if policy.max_bytes == 0 { - return Err(MediaError::DownloadDisabled); + return Err(Error::DownloadDisabled); } tokio::time::timeout(policy.timeout, self.fetch_before_deadline(url, policy)) .await - .map_err(|_| MediaError::Timeout)? + .map_err(|_| Error::Timeout)? } async fn fetch_before_deadline( &self, mut url: Url, policy: DownloadPolicy, - ) -> Result { + ) -> Result { let mut redirects_followed = 0; loop { self.validate_url(&url).await?; @@ -97,24 +118,22 @@ impl MediaFetcher { .get(url.clone()) .send() .await - .map_err(TransportError::from)?; + .map_err(crate::transport::Error::from)?; if response.status().is_redirection() { if redirects_followed == policy.max_redirects { - return Err(MediaError::TooManyRedirects); + return Err(Error::TooManyRedirects); } let location = response .headers() .get(reqwest::header::LOCATION) .and_then(|value| value.to_str().ok()) - .ok_or(MediaError::MissingRedirectLocation)?; - url = url - .join(location) - .map_err(|_| MediaError::InvalidRedirect)?; + .ok_or(Error::MissingRedirectLocation)?; + url = url.join(location).map_err(|_| Error::InvalidRedirect)?; redirects_followed += 1; continue; } if !response.status().is_success() { - return Err(MediaError::Http(response.status().as_u16())); + return Err(Error::Http(response.status().as_u16())); } enforce_download_size(response.content_length().unwrap_or(0), policy.max_bytes)?; let content_type = response @@ -127,7 +146,11 @@ impl MediaFetcher { .unwrap_or("application/octet-stream") .to_string(); let mut bytes = Vec::new(); - while let Some(chunk) = response.chunk().await.map_err(TransportError::from)? { + while let Some(chunk) = response + .chunk() + .await + .map_err(crate::transport::Error::from)? + { enforce_download_size(bytes.len() as u64 + chunk.len() as u64, policy.max_bytes)?; bytes.extend_from_slice(&chunk); } @@ -138,42 +161,40 @@ impl MediaFetcher { } } - async fn validate_url(&self, url: &Url) -> Result<(), MediaError> { + async fn validate_url(&self, url: &Url) -> Result<(), Error> { if !matches!(url.scheme(), "http" | "https") || !url.username().is_empty() || url.password().is_some() { - return Err(MediaError::BlockedUrl); + return Err(Error::BlockedUrl); } - let host = url.host_str().ok_or(MediaError::BlockedUrl)?; + let host = url.host_str().ok_or(Error::BlockedUrl)?; if self.allow_private_network { return Ok(()); } if let Ok(ip) = host.parse::() { - return (!is_blocked_ip(ip)) - .then_some(()) - .ok_or(MediaError::BlockedUrl); + return (!is_blocked_ip(ip)).then_some(()).ok_or(Error::BlockedUrl); } - let port = url.port_or_known_default().ok_or(MediaError::BlockedUrl)?; + let port = url.port_or_known_default().ok_or(Error::BlockedUrl)?; let addresses = self .address_resolver .resolve(host, port) .await - .map_err(|error| TransportError::Network(error.to_string()))?; + .map_err(|error| crate::transport::Error::Network(error.to_string()))?; validate_addresses(&addresses) } } -fn enforce_download_size(length: u64, max_bytes: u64) -> Result<(), MediaError> { +fn enforce_download_size(length: u64, max_bytes: u64) -> Result<(), Error> { if length > max_bytes { - return Err(MediaError::DownloadTooLarge); + return Err(Error::DownloadTooLarge); } Ok(()) } -fn validate_addresses(addresses: &[SocketAddr]) -> Result<(), MediaError> { +fn validate_addresses(addresses: &[SocketAddr]) -> Result<(), Error> { if addresses.is_empty() || addresses.iter().any(|address| is_blocked_ip(address.ip())) { - return Err(MediaError::BlockedUrl); + return Err(Error::BlockedUrl); } Ok(()) } @@ -415,7 +436,7 @@ mod tests { .await .expect_err("oversize body is rejected"); server.await.expect("server completes"); - assert!(matches!(error, MediaError::DownloadTooLarge)); + assert!(matches!(error, Error::DownloadTooLarge)); } #[tokio::test] @@ -433,7 +454,7 @@ mod tests { .await .expect_err("stream crossing limit is rejected"); server.await.expect("server completes"); - assert!(matches!(error, MediaError::DownloadTooLarge)); + assert!(matches!(error, Error::DownloadTooLarge)); } #[tokio::test] @@ -469,7 +490,7 @@ mod tests { .expect_err("private redirect is rejected"); let requests = server.await.expect("server completes"); assert_eq!(requests.len(), 1); - assert!(matches!(error, MediaError::BlockedUrl)); + assert!(matches!(error, Error::BlockedUrl)); } #[tokio::test] @@ -496,7 +517,7 @@ mod tests { .await .expect_err("fetch times out"); server.await.expect("server completes"); - assert!(matches!(error, MediaError::Timeout)); + assert!(matches!(error, Error::Timeout)); } #[tokio::test] @@ -522,7 +543,7 @@ mod tests { Url::parse("https://user:password@8.8.8.8/document").expect("credentialed URL parses"); assert!(matches!( fetcher.validate_url(&url).await, - Err(MediaError::BlockedUrl) + Err(Error::BlockedUrl) )); } } diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index 8f0f6652fa4..cbaf92b4986 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,4 +1,4 @@ -use crate::Error; +use super::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; @@ -10,7 +10,6 @@ pub(super) use crate::http_utils::{has_bearer_auth, has_header, truncate_error_b const HEADER_CONTEXT: &str = "messages"; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn messages_provider_config( provider: &str, ) -> Option<&'static dyn AnthropicMessagesProviderConfig> { @@ -24,5 +23,5 @@ pub(super) fn messages_provider_config( pub(super) fn string_headers( extra_headers: Option>, ) -> Result, Error> { - shared_string_headers(HEADER_CONTEXT, extra_headers) + shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from) } diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs new file mode 100644 index 00000000000..8bea035f0b0 --- /dev/null +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -0,0 +1,17 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("invalid provider: {0}")] + InvalidProvider(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("routing error: {0}")] + Routing(String), + #[error(transparent)] + Auth(#[from] litellm_auth::Error), + #[error(transparent)] + Transport(#[from] crate::transport::Error), + #[error(transparent)] + Headers(#[from] crate::http_utils::HeaderError), +} diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 61ff81bcdc8..d7d593f2d57 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,5 +1,5 @@ +use super::Error; use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::error::Error; use crate::http_utils::http_request; use super::client::http_client; @@ -7,7 +7,6 @@ use super::common_utils::truncate_error_body; use super::prepare::prepare_provider_request; use super::types::{AnthropicMessagesResponse, MessagesRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) async fn execute_messages_provider_call( request: MessagesRequest<'_>, ) -> Result { @@ -22,19 +21,19 @@ pub(super) async fn execute_messages_provider_call( let response = http_request(request_builder) .await - .map_err(|err| Error::Network(err.to_string()))?; + .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; let status = response.status(); let text = response .text() .await - .map_err(|err| Error::Network(err.to_string()))?; + .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; if !status.is_success() { - return Err(Error::Http { + return Err(Error::Transport(crate::transport::Error::Http { status: status.as_u16(), body: truncate_error_body(&text), - }); + })); } let response = serde_json::from_str(&text) @@ -62,17 +61,17 @@ pub(super) async fn execute_messages_provider_stream( let response = http_request(request_builder) .await - .map_err(|err| Error::Network(err.to_string()))?; + .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; let status = response.status(); if !status.is_success() { let text = response .text() .await - .map_err(|err| Error::Network(err.to_string()))?; - return Err(Error::Http { + .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; + return Err(Error::Transport(crate::transport::Error::Http { status: status.as_u16(), body: truncate_error_body(&text), - }); + })); } Ok(response) } diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index cfa8bda1104..156f42056f1 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -7,7 +7,8 @@ //! is the streaming variant; it hands the raw upstream response back so a host //! can splice the event stream to its own caller. -use crate::Error; +mod error; +pub use error::Error; mod client; mod common_utils; mod handler; @@ -18,7 +19,6 @@ pub mod types; use handler::{execute_messages_provider_call, execute_messages_provider_stream}; use types::{AnthropicMessagesResponse, MessagesRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(request).await } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index ec83d03f535..b10e03ea9c0 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,5 +1,5 @@ -use crate::error::Error; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use super::Error; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; use super::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; @@ -56,7 +56,6 @@ pub(super) fn prepare_provider_request( }) } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn validate_environment( config: &dyn AnthropicMessagesProviderConfig, extra_headers: Option>, diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index df9f7051011..f454effd7b5 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use crate::error::Error; +use super::Error; use super::common_utils::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, @@ -77,7 +77,14 @@ fn truncate_error_body_caps_long_payloads() { fn string_headers_rejects_non_string_values() { let headers = json!({"x-count": 3}).as_object().unwrap().clone(); let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert!(matches!(err, Error::InvalidRequest(_))); + assert_eq!( + err, + Error::Headers(crate::http_utils::HeaderError { + context: "messages", + name: "x-count".to_string(), + actual: "number", + }) + ); } #[test] @@ -420,7 +427,10 @@ async fn messages_maps_provider_error_status_to_http_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, Error::Http { status: 401, .. })); + assert!(matches!( + err, + Error::Transport(crate::transport::Error::Http { status: 401, .. }) + )); } #[tokio::test] diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index a5904c085a0..2719e62d280 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,5 +1,5 @@ +use super::Error; use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; -use crate::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { @@ -45,7 +45,6 @@ pub trait AnthropicMessagesProviderConfig: Sync { ] } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, request: AnthropicMessagesRequest, @@ -53,7 +52,6 @@ pub trait AnthropicMessagesProviderConfig: Sync { Ok(request) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_response( &self, _model: &str, diff --git a/litellm-rust/crates/core/src/observability/function_trace.rs b/litellm-rust/crates/core/src/observability/function_trace.rs deleted file mode 100644 index 2031e35901c..00000000000 --- a/litellm-rust/crates/core/src/observability/function_trace.rs +++ /dev/null @@ -1,215 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use serde::Serialize; -use tracing::span::{Attributes, Id}; -use tracing::{Dispatch, Subscriber}; -use tracing_subscriber::layer::Context; -use tracing_subscriber::prelude::*; -use tracing_subscriber::registry::LookupSpan; -use tracing_subscriber::{Layer, Registry}; - -use super::function_trace_filter; - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct FunctionTraceEvent { - pub id: usize, - pub parent_id: Option, - pub function: &'static str, - pub module_path: Option<&'static str>, - pub file: Option<&'static str>, - pub line: Option, -} - -#[derive(Clone, Default)] -pub struct FunctionTrace { - events: Arc>>, - span_events: Arc>>, -} - -impl FunctionTrace { - pub fn dispatcher(&self) -> Dispatch { - Dispatch::new( - Registry::default().with( - FunctionTraceLayer { - trace: self.clone(), - } - .with_filter(function_trace_filter()), - ), - ) - } - - pub fn events(&self) -> Vec { - self.events - .lock() - .unwrap_or_else(|error| error.into_inner()) - .clone() - } -} - -struct FunctionTraceLayer { - trace: FunctionTrace, -} - -impl Layer for FunctionTraceLayer -where - S: Subscriber + for<'lookup> LookupSpan<'lookup>, -{ - fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) { - let parent_id = context.span(id).and_then(|span| { - let span_events = self - .trace - .span_events - .lock() - .unwrap_or_else(|error| error.into_inner()); - span.scope() - .skip(1) - .find_map(|ancestor| span_events.get(&ancestor.id()).copied()) - }); - let mut events = self - .trace - .events - .lock() - .unwrap_or_else(|error| error.into_inner()); - let event_id = events.len(); - events.push(FunctionTraceEvent { - id: event_id, - parent_id, - function: attributes.metadata().name(), - module_path: attributes.metadata().module_path(), - file: attributes.metadata().file(), - line: attributes.metadata().line(), - }); - self.trace - .span_events - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(id.clone(), event_id); - } -} - -#[cfg(test)] -mod tests { - use crate::constants::FUNCTION_TRACE_TARGET; - - use super::*; - - fn event( - id: usize, - parent_id: Option, - function: &'static str, - ) -> (usize, Option, &'static str) { - (id, parent_id, function) - } - - fn structural_events( - events: &[FunctionTraceEvent], - ) -> Vec<(usize, Option, &'static str)> { - events - .iter() - .map(|event| (event.id, event.parent_id, event.function)) - .collect() - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn outer() { - tokio::task::yield_now().await; - inner().await; - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn inner() { - tokio::task::yield_now().await; - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn concurrent_parent() { - tokio::join!(inner(), inner()); - } - - #[tokio::test] - async fn concurrent_futures_keep_separate_traces_across_yields() { - use tracing::instrument::WithSubscriber; - - let first = FunctionTrace::default(); - let second = FunctionTrace::default(); - let outside = FunctionTrace::default(); - - async { - tokio::join!( - outer().with_subscriber(first.dispatcher()), - inner().with_subscriber(second.dispatcher()), - ); - inner().await; - } - .with_subscriber(outside.dispatcher()) - .await; - - assert_eq!( - structural_events(&first.events()), - vec![event(0, None, "outer"), event(1, Some(0), "inner")], - ); - assert_eq!( - structural_events(&second.events()), - vec![event(0, None, "inner")], - ); - assert_eq!( - structural_events(&outside.events()), - vec![event(0, None, "inner")], - ); - } - - #[tokio::test] - async fn concurrent_siblings_keep_the_same_parent() { - use tracing::instrument::WithSubscriber; - - let trace = FunctionTrace::default(); - concurrent_parent() - .with_subscriber(trace.dispatcher()) - .await; - - assert_eq!( - structural_events(&trace.events()), - vec![ - event(0, None, "concurrent_parent"), - event(1, Some(0), "inner"), - event(2, Some(0), "inner"), - ] - ); - } - - #[test] - fn records_matching_spans_in_creation_order() { - let trace = FunctionTrace::default(); - let dispatch = trace.dispatcher(); - - tracing::dispatcher::with_default(&dispatch, || { - let _ignored = tracing::trace_span!(target: "other", "ignored"); - let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name"); - let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level"); - let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name"); - }); - - assert_eq!( - structural_events(&trace.events()), - vec![event(0, None, "same_name"), event(1, None, "same_name")] - ); - } - - #[test] - fn records_matching_span_nesting_depth() { - let trace = FunctionTrace::default(); - let dispatch = trace.dispatcher(); - - tracing::dispatcher::with_default(&dispatch, || { - let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer"); - let _outer_guard = outer.enter(); - let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner"); - }); - - assert_eq!( - structural_events(&trace.events()), - vec![event(0, None, "outer"), event(1, Some(0), "inner")] - ); - } -} diff --git a/litellm-rust/crates/core/src/observability/mod.rs b/litellm-rust/crates/core/src/observability/mod.rs deleted file mode 100644 index 3f9da8e2bb4..00000000000 --- a/litellm-rust/crates/core/src/observability/mod.rs +++ /dev/null @@ -1,59 +0,0 @@ -use tracing::span::Id; -use tracing::{Level, Metadata, Subscriber}; -use tracing_subscriber::filter::{FilterFn, LevelFilter, filter_fn}; -use tracing_subscriber::layer::Context; -use tracing_subscriber::registry::LookupSpan; - -use crate::constants::FUNCTION_TRACE_TARGET; - -pub mod function_trace; - -pub use function_trace::{FunctionTrace, FunctionTraceEvent}; - -pub fn function_trace_filter() -> FilterFn) -> bool> { - filter_fn(|metadata| { - metadata.is_span() - && metadata.target() == FUNCTION_TRACE_TARGET - && *metadata.level() == Level::TRACE - }) - .with_max_level_hint(LevelFilter::TRACE) -} - -pub fn span_depth(context: &Context<'_, S>, id: &Id) -> usize -where - S: Subscriber + for<'lookup> LookupSpan<'lookup>, -{ - context - .span(id) - .map(|span| span.scope().skip(1).count()) - .unwrap_or_default() -} - -#[cfg(test)] -mod tests { - use tracing::instrument::WithSubscriber; - - use super::*; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn instrumented_with_literal_target() {} - - #[tokio::test] - async fn literal_instrument_target_matches_filter_constant() { - assert_eq!(FUNCTION_TRACE_TARGET, "litellm::function_trace"); - - let trace = FunctionTrace::default(); - instrumented_with_literal_target() - .with_subscriber(trace.dispatcher()) - .await; - - let events = trace.events(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].id, 0); - assert_eq!(events[0].parent_id, None); - assert_eq!(events[0].function, "instrumented_with_literal_target"); - assert_eq!(events[0].module_path, Some(module_path!())); - assert_eq!(events[0].file, Some(file!())); - assert!(events[0].line.is_some()); - } -} diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/cohere.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/cohere.rs index 4c8455a171c..3691e9e1809 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure/cohere.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/cohere.rs @@ -1,5 +1,5 @@ use super::super::OcrAdapter; -use crate::Error; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::cohere::{ CohereParams, CohereResponse, transform_request, transform_response, validate_document, @@ -9,8 +9,8 @@ use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; use crate::ocr::prepare::{credential_env, transform_request_body}; use crate::ocr::registry::OcrProvider; use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; -use crate::providers::azure_ai::auth::AzureAuthInputs; use crate::url_utils::ApiUrl; +use litellm_auth_azure::AzureAuthInputs; const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs index e90c27ba59d..eba300908f1 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs @@ -1,7 +1,6 @@ use super::super::OcrAdapter; -use crate::Error; -use crate::auth::{InputSource, Sourced}; use crate::constants::{AZURE_DI_API_VERSION, AZURE_DI_SUBSCRIPTION_HEADER}; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::document_intelligence::{ self, AzureDocumentIntelligenceOperation, DocumentIntelligenceParams, @@ -10,8 +9,9 @@ use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; use crate::ocr::prepare::{credential_env, transform_request_body}; use crate::ocr::registry::OcrProvider; use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrResponseFormat}; -use crate::providers::azure_ai::auth::AzureAuthInputs; use crate::url_utils::ApiUrl; +use litellm_auth::{InputSource, Sourced}; +use litellm_auth_azure::AzureAuthInputs; mod polling; @@ -75,7 +75,6 @@ impl OcrAdapter for AzureDocumentIntelligenceAdapter { } } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_ocr_params( request: &LiteLLMOcrRequest, ) -> Result { diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs index 6ed1e4441d4..87378dccdb7 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/polling.rs @@ -77,7 +77,7 @@ async fn poll_operation( let response = tokio::time::timeout_at(deadline, crate::http_utils::http_request(builder)) .await .map_err(|_| OcrPollingError::PollTimeout)? - .map_err(crate::error::TransportError::from)?; + .map_err(crate::transport::Error::from)?; let retry = response .headers() .get(reqwest::header::RETRY_AFTER) diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs index 8639590b05c..28e09cdc80f 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/mistral.rs @@ -1,7 +1,6 @@ use super::super::OcrAdapter; -use crate::Error; -use crate::auth::{InputSource, Sourced}; use crate::constants::AZURE_AI_OCR_PATH; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; use crate::ocr::document::{inline_remote_document, validate_inline_document}; @@ -11,8 +10,9 @@ use crate::ocr::prepare::{ }; use crate::ocr::registry::OcrProvider; use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection}; -use crate::providers::azure_ai::auth::AzureAuthInputs; use crate::url_utils::ApiUrl; +use litellm_auth::{InputSource, Sourced}; +use litellm_auth_azure::AzureAuthInputs; const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs index 3d30ae6d6bd..0b2fcb0f4cb 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/mod.rs @@ -4,12 +4,12 @@ mod mistral; use std::sync::OnceLock; -use crate::Error; -use crate::auth::error::AuthConfigurationError; -use crate::auth::{InputSource, Sourced}; +use crate::ocr::Error; + use crate::ocr::error::OcrError; use crate::ocr::types::OcrConnection; -use crate::providers::azure_ai::auth::{AzureAuthInputs, AzureAuthService}; +use litellm_auth::{InputSource, Sourced}; +use litellm_auth_azure::{AzureAuthInputs, AzureAuthService}; pub(crate) use cohere::AzureCohereAdapter; pub(crate) use document_intelligence::AzureDocumentIntelligenceAdapter; @@ -26,7 +26,7 @@ async fn resolve_entra( .get_azure_ad_token(config, env_lookup) .await .or_else(|error| match error { - crate::AuthError::EmptyAzureToken => Ok(None), + litellm_auth::Error::EmptyAzureToken => Ok(None), other => Err(other), }) .map(|credential| { @@ -47,10 +47,7 @@ fn validate_destination( && connection.api_base_source == InputSource::Request && credential_source != InputSource::Request { - return Err(Error::from(crate::AuthError::Configuration( - AuthConfigurationError::RequestAzureCredentialDestination, - )) - .into()); + return Err(Error::from(litellm_auth::Error::RequestAzureCredentialDestination).into()); } Ok(()) } diff --git a/litellm-rust/crates/core/src/ocr/adapters/cohere.rs b/litellm-rust/crates/core/src/ocr/adapters/cohere.rs index 933ead7f7f7..d1faeeb7b1d 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/cohere.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/cohere.rs @@ -1,6 +1,6 @@ use super::OcrAdapter; -use crate::Error; use crate::constants::{COHERE_API_KEY_ENV, COHERE_PARSE_API_BASE}; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::cohere::{ CohereParams, CohereResponse, transform_request, transform_response, validate_document, diff --git a/litellm-rust/crates/core/src/ocr/adapters/mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/mistral.rs index cdbc2c3effc..c379462c089 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/mistral.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/mistral.rs @@ -1,6 +1,6 @@ use super::OcrAdapter; -use crate::Error; use crate::constants::MISTRAL_OCR_API_BASE; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; diff --git a/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs index 2dafe291674..40cefa05373 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/reducto/mod.rs @@ -1,8 +1,8 @@ mod legacy; mod v3; -use crate::Error; use crate::constants::{REDUCTO_API_BASE, REDUCTO_API_KEY_ENV, REDUCTO_ID_PREFIX}; +use crate::ocr::Error; use crate::ocr::document::InlineDocument; use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; use crate::ocr::types::{OcrConnection, OcrDocument}; @@ -90,7 +90,7 @@ pub(super) async fn prepare_document( ); let response = crate::http_utils::http_request(builder) .await - .map_err(crate::error::TransportError::from)?; + .map_err(crate::transport::Error::from)?; let uploaded = crate::ocr::client::read_json_response::< crate::ocr::codecs::reducto::ReductoUploadResponse, >(response, false, connection.max_response_bytes) diff --git a/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs b/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs index d16b3e7f386..fc24dbe489c 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/vertex/deepseek.rs @@ -1,7 +1,6 @@ use super::super::OcrAdapter; use super::validate_destination; -use crate::Error; -use crate::auth::vertex::{self, VertexConfig}; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::deepseek::{self, DeepSeekOcrParams, DeepSeekOcrResponse}; use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; @@ -11,6 +10,7 @@ use crate::ocr::prepare::{ use crate::ocr::registry::OcrProvider; use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; use crate::url_utils::ApiUrl; +use litellm_auth_gcp::{self as vertex, VertexConfig}; const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com"; const MODEL_NAMESPACE: &str = "deepseek-ai"; const DEFAULT_LOCATION: &str = "us-central1"; diff --git a/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs b/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs index 88c61725cee..3a1abf47ddf 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/vertex/mistral.rs @@ -1,7 +1,6 @@ use super::super::OcrAdapter; use super::validate_destination; -use crate::Error; -use crate::auth::vertex::{self, VertexConfig}; +use crate::ocr::Error; use crate::ocr::OcrClient; use crate::ocr::codecs::mistral::{self, MistralOcrParams, MistralOcrResponse}; use crate::ocr::document::{inline_remote_document, validate_inline_document}; @@ -12,6 +11,7 @@ use crate::ocr::prepare::{ use crate::ocr::registry::OcrProvider; use crate::ocr::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; use crate::url_utils::ApiUrl; +use litellm_auth_gcp::{self as vertex, VertexConfig}; const DEFAULT_LOCATION: &str = "us-central1"; #[derive(Clone, Debug)] diff --git a/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs index 270c41e647d..798510e7405 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/vertex/mod.rs @@ -1,9 +1,9 @@ mod deepseek; mod mistral; -use crate::Error; -use crate::auth::InputSource; -use crate::auth::error::AuthConfigurationError; +use crate::ocr::Error; +use litellm_auth::InputSource; + use crate::ocr::error::OcrError; use crate::ocr::types::OcrConnection; @@ -12,10 +12,7 @@ pub(crate) use mistral::VertexMistralAdapter; fn validate_destination(connection: &OcrConnection) -> Result<(), OcrError> { if connection.api_base.is_some() && connection.api_base_source == InputSource::Request { - return Err(Error::from(crate::AuthError::Configuration( - AuthConfigurationError::RequestVertexCredentialDestination, - )) - .into()); + return Err(Error::from(litellm_auth::Error::RequestVertexCredentialDestination).into()); } Ok(()) } diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index 394ca778d2f..00bfeb2b7b2 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -4,14 +4,13 @@ use std::time::Duration; use bytes::{Bytes, BytesMut}; use serde::de::DeserializeOwned; -use super::error::{OcrError, OcrResponseError}; +use super::error::{Error, OcrError, OcrResponseError}; use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; use super::wire::{DecodedOcrResponse, decode_response}; -use crate::Error; -use crate::auth::vertex::VertexAuth; use crate::constants::OCR_CONNECT_TIMEOUT_SECS; -use crate::error::TransportError; use crate::media::MediaFetcher; +use crate::transport::Error as TransportError; +use litellm_auth_gcp::VertexAuth; #[derive(Clone)] pub struct OcrClient { @@ -36,12 +35,6 @@ impl OcrClient { shared_client() } - #[tracing::instrument( - name = "ocr", - target = "litellm::function_trace", - level = "trace", - skip_all - )] pub async fn perform(&self, request: LiteLLMOcrRequest) -> Result { use super::{ NativeOutcome, OcrAdmission, OcrCall, OcrCallStep, OcrHookHost, OcrHost, @@ -164,7 +157,7 @@ pub(crate) async fn read_response_bytes( } } if !status.is_success() { - return Err(crate::error::TransportError::Http { + return Err(crate::transport::Error::Http { status: status.as_u16(), body: crate::http_utils::truncate_error_body(&String::from_utf8_lossy(&bytes)), } @@ -180,7 +173,7 @@ pub(crate) fn transport_error(error: reqwest::Error) -> Error { body: "OCR request timed out".into(), }; } - crate::error::TransportError::from(error).into() + crate::transport::Error::from(error).into() } #[cfg(test)] diff --git a/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs index 7e8ce63b379..999ac6cf032 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs @@ -5,7 +5,6 @@ use super::types::*; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn transform_ocr_request( provider_model: &str, document: OcrDocument, diff --git a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs index f76a7c2b232..018d7eb9c65 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs @@ -7,7 +7,6 @@ use crate::ocr::document::InlineDocument; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn transform_ocr_request( document: OcrDocument, ) -> Result { diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs index e60f1f5d3d6..e8073905548 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs @@ -2,7 +2,6 @@ use super::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn transform_ocr_request( model: &str, document: OcrDocument, diff --git a/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs index 7073643f6b6..f4c8338c134 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs @@ -6,12 +6,6 @@ use super::types::*; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument( - name = "transform_ocr_request", - target = "litellm::function_trace", - level = "trace", - skip_all -)] pub(crate) fn transform_v3_ocr_request( _model: &str, document: OcrDocument, @@ -23,12 +17,6 @@ pub(crate) fn transform_v3_ocr_request( }) } -#[tracing::instrument( - name = "transform_ocr_request", - target = "litellm::function_trace", - level = "trace", - skip_all -)] pub(crate) fn transform_legacy_ocr_request( _model: &str, document: OcrDocument, diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs index 82a32ac1ab5..a7afdaf8793 100644 --- a/litellm-rust/crates/core/src/ocr/document.rs +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -7,8 +7,9 @@ use serde_json::Map; use super::error::{OcrError, OcrRequestError, OcrResponseError}; use super::types::{OcrConnection, OcrDocument}; use crate::constants::{OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS}; -use crate::error::{MediaError, TransportError}; +use crate::media::Error as MediaError; use crate::media::{DownloadPolicy, MediaFetcher}; +use crate::transport::Error as TransportError; pub fn encode_file_document( bytes: &[u8], diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs index 55ea2cbcdae..1c21edb6c91 100644 --- a/litellm-rust/crates/core/src/ocr/error.rs +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -1,6 +1,106 @@ use thiserror::Error; -use crate::error::TransportError; +use crate::transport::Error as TransportError; + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum Error { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("Document URL is required")] + MissingDocumentUrl, + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("invalid provider: {0}")] + InvalidProvider(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("{0}")] + Auth(String), + #[error( + "Missing {provider} API Key - A call is being made to {provider} but no key is set either in the environment variables or via params" + )] + MissingApiKey { provider: &'static str }, + #[error( + "invalid authentication configuration: Missing Azure AI credentials - set AZURE_AI_API_KEY or configure Entra ID" + )] + MissingAzureAiCredentials, + #[error( + "invalid authentication configuration: Missing Azure Document Intelligence credentials - set AZURE_DOCUMENT_INTELLIGENCE_API_KEY or configure Entra ID" + )] + MissingAzureDocumentIntelligenceCredentials, + #[error( + "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()" + )] + MissingReductoApiKey, + #[error("upstream request failed with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("upstream network error: {0}")] + Network(String), + /// The provider was never reached: DNS, TCP, TLS or proxy setup failed + /// before any byte of the request went out. Nothing was billed, so a host + /// that keeps a reference implementation can serve the request itself. + /// A timeout is deliberately not this, since the provider may have received + /// and answered the request already. + #[error("could not reach the provider: {0}")] + Connect(String), + #[error("routing error: {0}")] + Routing(String), + /// The request is outside the surface this route covers in Rust. Hosts that + /// keep a reference implementation treat this as "fall back", not "fail". + #[error("unsupported by the rust path: {0}")] + Unsupported(&'static str), +} + +impl Error { + pub const fn http_status_code(&self) -> Option { + match self { + Self::InvalidRequest(_) => Some(400), + Self::MissingDocumentUrl => Some(500), + Self::Http { status, .. } => Some(*status), + _ => None, + } + } +} + +impl From for Error { + fn from(error: OcrRequestError) -> Self { + match error { + OcrRequestError::MissingField(field) => Self::MissingField(field), + OcrRequestError::MissingDocumentUrl => Self::MissingDocumentUrl, + error => Self::InvalidRequest(error.to_string()), + } + } +} + +impl From for Error { + fn from(error: OcrResponseError) -> Self { + Self::InvalidResponse(error.to_string()) + } +} + +impl From for Error { + fn from(error: TransportError) -> Self { + match error { + TransportError::Http { status, body } => Self::Http { status, body }, + TransportError::Network(message) => Self::Network(message), + TransportError::Connect(message) => Self::Connect(message), + } + } +} + +impl From for Error { + fn from(error: litellm_auth::Error) -> Self { + match error { + litellm_auth::Error::MissingApiKey { provider, .. } => Self::MissingApiKey { provider }, + error => Self::Auth(error.to_string()), + } + } +} #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum OcrRequestError { @@ -83,16 +183,16 @@ pub enum OcrError { #[error("{0}")] Polling(#[from] OcrPollingError), #[error("{0}")] - Public(#[from] crate::Error), + Public(#[from] Error), } -impl From for crate::Error { +impl From for Error { fn from(error: OcrError) -> Self { match error { OcrError::Request(error) => error.into(), OcrError::Response(error) => error.into(), OcrError::Transport(error) => error.into(), - OcrError::Polling(error) => crate::Error::InvalidResponse(error.to_string()), + OcrError::Polling(error) => Error::InvalidResponse(error.to_string()), OcrError::Public(error) => error, } } diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index cd1d538aaa8..1ec02f3b622 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -3,8 +3,8 @@ use super::adapters::OcrAdapter; use super::hooks::{OcrHooks, OcrLifecycleHooks, OcrPostCallRequest}; use super::registry::OcrAdapterKind; use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse}; -use crate::Error; use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext}; +use crate::ocr::Error; use std::sync::Arc; pub(crate) async fn perform_ocr_request( diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs index 3e7507e9ed5..1d8c5953fa7 100644 --- a/litellm-rust/crates/core/src/ocr/hooks.rs +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -3,8 +3,8 @@ use std::pin::Pin; use std::sync::Arc; use super::types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrDocument}; -use crate::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; +use crate::ocr::Error; use serde::Serialize; use serde_json::Value; @@ -80,6 +80,7 @@ pub(crate) struct OcrLifecycleHooks { impl CallLifecycleHooks for OcrLifecycleHooks { + type Error = crate::ocr::Error; type PreCallFuture<'a> = OcrHookFuture<'a, LiteLLMOcrRequest>; type DuringCallFuture<'a> = OcrHookFuture<'a, LiteLLMOcrRequest>; type SuccessFuture<'a> = OcrLogFuture<'a>; @@ -125,12 +126,6 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, @@ -140,12 +135,6 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, diff --git a/litellm-rust/crates/core/src/ocr/lifecycle.rs b/litellm-rust/crates/core/src/ocr/lifecycle.rs index 92c9d4b717c..efa2b1f2873 100644 --- a/litellm-rust/crates/core/src/ocr/lifecycle.rs +++ b/litellm-rust/crates/core/src/ocr/lifecycle.rs @@ -10,13 +10,13 @@ use super::hooks::{ OcrPreCallRequest, }; use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient}; -use crate::AuthError; -use crate::Error; -use crate::auth::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; use crate::call_lifecycle::host::{ HostCall, HostCallFuture, HostCallStep, HostFailure, HostLifecycle, HostPhase, }; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming}; +use crate::ocr::Error; +use litellm_auth::Error as AuthError; +use litellm_auth::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; pub type NativeResult = Result, Error>; @@ -84,7 +84,7 @@ impl OcrHostOperation { pub enum OcrHostResult { Request(Result<(Box, bool), Error>), - Lifecycle(Result<(), HostFailure>), + Lifecycle(Result<(), HostFailure>), AzureAdToken(Result), PreCall(Result), DuringCall(Result), @@ -256,7 +256,7 @@ impl OcrCall { Ok(self.host_step(operation)) } - fn accept(&mut self, result: Result<(), HostFailure>) { + fn accept(&mut self, result: Result<(), HostFailure>) { let cancelled = matches!(&result, Err(HostFailure::Cancelled(_))); if let Some(error) = self.lifecycle.accept(result) { if cancelled { @@ -268,7 +268,7 @@ impl OcrCall { } } - pub async fn interrupt(&mut self, failure: HostFailure) -> Result { + pub async fn interrupt(&mut self, failure: HostFailure) -> Result { if self.completed { return Err(Error::InvalidRequest( "OCR call cannot be interrupted after completion".into(), @@ -286,6 +286,7 @@ impl OcrCall { } impl HostCall for OcrCall { + type Error = crate::ocr::Error; type Operation = OcrHostOperation; type Result = OcrHostResult; type Complete = LiteLLMOcrResponse; @@ -293,14 +294,14 @@ impl HostCall for OcrCall { fn resume( &mut self, result: Option, - ) -> HostCallFuture<'_, Self::Operation, Self::Complete> { + ) -> HostCallFuture<'_, Self::Operation, Self::Complete, Self::Error> { Box::pin(OcrCall::resume(self, result)) } fn interrupt( &mut self, - failure: HostFailure, - ) -> HostCallFuture<'_, Self::Operation, Self::Complete> { + failure: HostFailure, + ) -> HostCallFuture<'_, Self::Operation, Self::Complete, Self::Error> { Box::pin(OcrCall::interrupt(self, failure)) } } diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index e29fd6ac572..3b51ff98356 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -3,6 +3,7 @@ pub mod client; mod codecs; mod document; pub mod error; +pub use error::Error; mod handler; pub mod hooks; mod lifecycle; diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 9934a1d9a14..5a48206d53c 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -14,7 +14,6 @@ pub(crate) struct ParsedProviderParams { pub extra_params: Map, } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn _prepare_ocr_request( request: &LiteLLMOcrRequest, ) -> Result, OcrRequestError> { @@ -120,7 +119,7 @@ pub(crate) fn build_http_request( .timeout(request.connection.timeout); crate::http_utils::with_headers(builder, headers, crate::http_utils::HeaderPolicy::All) .build() - .map_err(crate::error::TransportError::from) + .map_err(crate::transport::Error::from) .map_err(OcrError::from) } diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs index ed7d4fd5cf2..17185a02020 100644 --- a/litellm-rust/crates/core/src/ocr/registry.rs +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -1,6 +1,6 @@ use super::adapters::OcrAdapter; -use crate::Error; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::ocr::Error; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; macro_rules! define_adapter_types { ($( $variant:ident, $adapter:ty, $instance:expr, $provider:ident; )+) => { diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 76df8b42806..69e6982414b 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -7,9 +7,9 @@ use serde_json::{Map, Value}; use super::hooks::{NoopOcrHooks, OcrHooks}; use super::registry::{OcrAdapterKind, resolve_wire_adapter}; -use crate::Error; -use crate::auth::{InputSource, TokenProviderHandle}; use crate::constants::OCR_HTTP_TIMEOUT_SECS; +use crate::ocr::Error; +use litellm_auth::{InputSource, TokenProviderHandle}; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs index 6dc6b34b73d..93816effcb1 100644 --- a/litellm-rust/crates/core/src/ocr/wire.rs +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -4,8 +4,8 @@ use std::collections::BTreeMap; use std::time::Duration; use super::types::{LiteLLMOcrRequest, OcrConnection, OcrDocument}; -use crate::Error; -use crate::auth::InputSource; +use crate::ocr::Error; +use litellm_auth::InputSource; use serde::{ Deserialize, de::{DeserializeOwned, IntoDeserializer}, diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs index b22de6c47de..2cc94751fb4 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::Error; +use crate::chat_completions::Error; use serde_json::json; fn messages(value: Value) -> Vec { diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index a7d5a8ad0cf..ba1a1e1d350 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -1,5 +1,6 @@ use serde_json::{Map, Value, json}; +use crate::chat_completions::Error; use crate::chat_completions::conversation::{Conversation, build_conversation}; use crate::chat_completions::transformation::{ ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message, @@ -10,7 +11,6 @@ use crate::chat_completions::types::{ ProviderChatRequestData, ProviderChatResponseData, }; use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::error::Error; use crate::providers::anthropic::messages::transformation::{ complete_anthropic_url, resolve_anthropic_api_key, }; @@ -117,7 +117,6 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { SUPPORTED_PARAMS } @@ -138,7 +137,6 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, model: &str, @@ -150,7 +148,6 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_response( &self, _model: &str, diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 3ed00b7cc5f..080f11c8cac 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,5 +1,4 @@ -use crate::auth::error::MissingCredential; -use crate::error::Error; +use crate::messages::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -18,11 +17,14 @@ pub fn non_empty(value: Option<&str>) -> Option<&str> { pub fn resolve_anthropic_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> Result { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| Error::from(crate::AuthError::from(MissingCredential::AnthropicApiKey))) + .ok_or(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: ANTHROPIC_API_KEY_ENV, + }) } pub fn complete_anthropic_url( @@ -42,7 +44,6 @@ pub fn complete_anthropic_url( } impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, @@ -57,7 +58,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - resolve_anthropic_api_key(api_key, env_lookup) + resolve_anthropic_api_key(api_key, env_lookup).map_err(Error::from) } fn auth_strategy(&self) -> MessagesAuthStrategy { @@ -115,10 +116,12 @@ mod tests { resolve_anthropic_api_key(Some(" "), &with_env).unwrap(), "sk-env" ); - assert!(matches!( - resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"), - Error::Auth(_) - )); + assert_eq!( + resolve_anthropic_api_key(None, &|_| None) + .expect_err("missing key") + .to_string(), + "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable" + ); } #[test] diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth/mod.rs deleted file mode 100644 index 33d007c1945..00000000000 --- a/litellm-rust/crates/core/src/providers/azure_ai/auth/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod credential_provider_cache; -mod native; -mod resolve; -mod types; - -pub(crate) use resolve::AzureAuthService; -pub(crate) use types::AzureAuthInputs; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 585b34f393f..182aea84ab2 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,5 +1,4 @@ -use crate::auth::error::MissingCredential; -use crate::error::Error; +use crate::messages::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use crate::messages::types::{ AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, @@ -33,7 +32,12 @@ pub fn resolve_azure_api_key( non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| Error::from(crate::AuthError::from(MissingCredential::AzureApiKey))) + .ok_or_else(|| { + Error::from(litellm_auth::Error::MissingApiKey { + provider: "Azure", + environment_variable: AZURE_API_KEY_ENV, + }) + }) } pub fn complete_azure_anthropic_url( @@ -43,7 +47,7 @@ pub fn complete_azure_anthropic_url( let api_base = non_empty(api_base) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| Error::from(crate::AuthError::from(MissingCredential::AzureApiBase)))?; + .ok_or_else(|| Error::from(litellm_auth::Error::MissingAzureApiBase))?; let api_base = api_base.trim_end_matches('/'); @@ -132,7 +136,6 @@ fn fold_system_role_messages(request: AnthropicMessagesRequest) -> AnthropicMess } impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, diff --git a/litellm-rust/crates/core/src/providers/azure_ai/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/mod.rs index 4f41d1d6abb..ba63992f3cb 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/mod.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/mod.rs @@ -1,2 +1 @@ -pub(crate) mod auth; pub mod messages; diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 9bf1f73a74d..a418e860b92 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -1,12 +1,13 @@ use serde_json::{Map, Value, json}; +use crate::audio_transcription::Error; use crate::audio_transcription::transformation::{ AudioTranscriptionAuth, AudioTranscriptionProviderConfig, }; use crate::audio_transcription::types::{ AudioTranscriptionRequestData, AudioTranscriptionResponseData, }; -use crate::error::{Error, json_type_name}; +use crate::http_utils::json_type_name; pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region}; use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; @@ -46,12 +47,10 @@ fn optional_string<'a>(params: &'a Map, key: &str) -> Option<&'a } impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_transcription_params(&self) -> &'static [&'static str] { SUPPORTED_PARAMS } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_transcription_request( &self, _model: &str, @@ -85,7 +84,6 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_transcription_response( &self, _model: &str, diff --git a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs index e5e52bfce95..b51cef7545c 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs @@ -1,930 +1 @@ -use std::collections::BTreeMap; -use std::sync::{Mutex, OnceLock}; -use std::time::Duration; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::caching::in_memory_cache::InMemoryCache; -use crate::error::Error; -use aws_credential_types::Credentials; -use aws_credential_types::provider::ProvideCredentials; -use aws_sigv4::http_request::{ - SignableBody, SignableRequest, SigningParams, SigningSettings, sign, -}; -use aws_sigv4::sign::v4; -use aws_smithy_runtime_api::client::identity::Identity; -use serde_json::{Map, Value}; -use sha2::{Digest, Sha256}; - -use super::constants::{ - AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME, - AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, - AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, - BEDROCK_SERVICE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, - SIGV4_COMPUTED_HEADER_NAMES, -}; - -const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); -const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600); - -static IAM_CREDENTIALS_CACHE: OnceLock>> = OnceLock::new(); - -fn credential_cache_ttl(flow: &AwsAuthFlow) -> Option { - match flow { - AwsAuthFlow::StaticKeys { .. } => Some(STATIC_CREDENTIALS_TTL), - AwsAuthFlow::DefaultChain => Some(AMBIENT_CREDENTIALS_TTL), - AwsAuthFlow::WebIdentity { .. } - | AwsAuthFlow::AssumeRole { .. } - | AwsAuthFlow::Profile { .. } - | AwsAuthFlow::SessionToken { .. } => None, - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct AwsAuthConfig { - pub access_key_id: Option, - pub secret_access_key: Option, - pub session_token: Option, - pub region_name: Option, - pub session_name: Option, - pub profile_name: Option, - pub role_name: Option, - pub web_identity_token: Option, - pub sts_endpoint: Option, - pub external_id: Option, -} - -impl AwsAuthConfig { - fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Self { - Self { - access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)), - secret_access_key: self - .secret_access_key - .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)), - session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)), - region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)), - session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)), - profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)), - role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)), - web_identity_token: self - .web_identity_token - .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)), - sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)), - external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum AwsAuthFlow { - WebIdentity { - token: String, - role: String, - session_name: String, - }, - AssumeRole { - role: String, - session_name: Option, - }, - Profile { - name: String, - }, - SessionToken { - access_key_id: String, - secret_access_key: String, - session_token: String, - }, - StaticKeys { - access_key_id: String, - secret_access_key: String, - region_name: String, - }, - DefaultChain, -} - -fn cache_key(config: &AwsAuthConfig, flow: &AwsAuthFlow) -> String { - let mut hasher = Sha256::new(); - hasher.update(format!("{config:?}:{flow:?}")); - format!("{:x}", hasher.finalize()) -} - -fn get_cached_credentials(key: &str) -> Option { - let cache = IAM_CREDENTIALS_CACHE.get_or_init(|| Mutex::new(InMemoryCache::default())); - let mut entries = cache.lock().ok()?; - entries.get_cache(key) -} - -fn set_cached_credentials(key: String, credentials: Credentials, ttl: Duration) { - let cache = IAM_CREDENTIALS_CACHE.get_or_init(|| Mutex::new(InMemoryCache::default())); - if let Ok(mut entries) = cache.lock() { - entries.set_cache(key, credentials, Some(ttl)); - } -} - -fn role_identity(arn: &str) -> Option<(&str, &str, &str)> { - let mut parts = arn.splitn(6, ':'); - let ("arn", partition, _, _, account, resource) = ( - parts.next()?, - parts.next()?, - parts.next()?, - parts.next()?, - parts.next()?, - parts.next()?, - ) else { - return None; - }; - let role = if let Some(role) = resource.strip_prefix("role/") { - role.rsplit('/').next()? - } else { - resource.strip_prefix("assumed-role/")?.split('/').next()? - }; - Some((partition, account, role)) -} - -fn same_role_arns(target: &str, caller: &str) -> bool { - role_identity(target) == role_identity(caller) -} - -pub fn classify_auth( - config: AwsAuthConfig, - env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> AwsAuthFlow { - let config = config.with_environment(env_lookup); - if let (Some(token), Some(role), Some(session_name)) = ( - config.web_identity_token.clone(), - config.role_name.clone(), - config.session_name.clone(), - ) { - return AwsAuthFlow::WebIdentity { - token, - role, - session_name, - }; - } - if let Some(role) = config.role_name.clone() { - return AwsAuthFlow::AssumeRole { - role, - session_name: config.session_name.clone(), - }; - } - if let Some(name) = config.profile_name { - return AwsAuthFlow::Profile { name }; - } - if let (Some(access_key_id), Some(secret_access_key), Some(session_token)) = ( - config.access_key_id.clone(), - config.secret_access_key.clone(), - config.session_token, - ) { - return AwsAuthFlow::SessionToken { - access_key_id, - secret_access_key, - session_token, - }; - } - if let (Some(access_key_id), Some(secret_access_key), Some(region_name)) = ( - config.access_key_id, - config.secret_access_key, - config.region_name, - ) { - return AwsAuthFlow::StaticKeys { - access_key_id, - secret_access_key, - region_name, - }; - } - AwsAuthFlow::DefaultChain -} - -pub async fn resolve_credentials( - config: AwsAuthConfig, - env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> Result { - let resolved = config.clone().with_environment(env_lookup); - let flow = classify_auth(config, env_lookup); - match flow { - AwsAuthFlow::SessionToken { - access_key_id, - secret_access_key, - session_token, - } => Ok(Credentials::new( - access_key_id, - secret_access_key, - Some(session_token), - None, - "litellm-static-session", - )), - AwsAuthFlow::StaticKeys { - access_key_id, - secret_access_key, - region_name, - } => { - let flow = AwsAuthFlow::StaticKeys { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - region_name, - }; - let key = cache_key(&resolved, &flow); - if let Some(credentials) = get_cached_credentials(&key) { - return Ok(credentials); - } - let credentials = Credentials::new( - access_key_id, - secret_access_key, - None, - None, - "litellm-static", - ); - set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL), - ); - Ok(credentials) - } - AwsAuthFlow::Profile { name } => { - let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() - .profile_name(name) - .build(); - provider - .provide_credentials() - .await - .map_err(|error| Error::Auth(format!("AWS profile credentials failed: {error}"))) - } - AwsAuthFlow::AssumeRole { role, session_name } => { - if is_already_running_as_role(&role, &resolved).await? { - let ambient_flow = AwsAuthFlow::DefaultChain; - let key = cache_key(&resolved, &ambient_flow); - if let Some(credentials) = get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider.provide_credentials().await.map_err(|error| { - Error::Auth(format!("AWS default credentials failed: {error}")) - })?; - set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - return Ok(credentials); - } - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - if let (Some(access_key_id), Some(secret_access_key)) = - (resolved.access_key_id, resolved.secret_access_key) - { - loader = loader.credentials_provider(Credentials::new( - access_key_id, - secret_access_key, - resolved.session_token, - None, - "litellm-role-source", - )); - } - let sdk_config = loader.load().await; - let builder = aws_config::sts::AssumeRoleProvider::builder(role); - let builder = match session_name { - Some(name) => builder.session_name(name), - None => builder.session_name(default_session_name()), - }; - let builder = match resolved.external_id { - Some(id) => builder.external_id(id), - None => builder, - }; - let provider = builder.configure(&sdk_config).build().await; - provider - .provide_credentials() - .await - .map_err(|error| Error::Auth(format!("AWS role credentials failed: {error}"))) - } - AwsAuthFlow::WebIdentity { - token, - role, - session_name, - } => { - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let client = aws_sdk_sts::Client::new(&sdk_config); - let response = client - .assume_role_with_web_identity() - .role_arn(role) - .role_session_name(session_name) - .web_identity_token(token) - .send() - .await - .map_err(|error| { - Error::Auth(format!("AWS web identity credentials failed: {error}")) - })?; - let credentials = response.credentials().ok_or_else(|| { - Error::Auth("AWS web identity response had no credentials".to_string()) - })?; - let expiration = SystemTime::try_from(*credentials.expiration()).map_err(|error| { - Error::Auth(format!("AWS web identity expiration was invalid: {error}")) - })?; - Ok(Credentials::new( - credentials.access_key_id(), - credentials.secret_access_key(), - Some(credentials.session_token().to_string()), - Some(expiration), - "litellm-web-identity", - )) - } - AwsAuthFlow::DefaultChain => { - let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain); - if let Some(credentials) = get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(|error| Error::Auth(format!("AWS default credentials failed: {error}")))?; - set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - Ok(credentials) - } - } -} - -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { - if role_identity(role).is_none() { - return Ok(false); - } - if let (Ok(current_role), Ok(token_file)) = ( - std::env::var(AWS_ROLE_ARN), - std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE), - ) && !token_file.is_empty() - { - return Ok(same_role_arns(role, ¤t_role)); - } - - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = config.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = config.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let response = match aws_sdk_sts::Client::new(&sdk_config) - .get_caller_identity() - .send() - .await - { - Ok(response) => response, - Err(_) => return Ok(false), - }; - Ok(response - .arn() - .is_some_and(|caller| same_role_arns(role, caller))) -} - -fn default_session_name() -> String { - let seconds = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |duration| duration.as_secs()); - format!("{DEFAULT_SESSION_NAME_PREFIX}-{seconds}") -} - -/// The subset of `headers` SigV4 should cover. -/// -/// Python signs only these and reattaches the rest afterwards, so a forwarded -/// client header cannot change the canonical request and invalidate the -/// signature. Signing everything instead makes the request 403 on a header the -/// caller supplied, on a deployment that works on the Python path. -pub fn aws_signature_headers(headers: &BTreeMap) -> BTreeMap { - headers - .iter() - .filter(|(name, _)| { - let name = name.to_ascii_lowercase(); - AWS_SIGNED_HEADER_NAMES.contains(&name.as_str()) - || name.starts_with("x-amz-") - || name.starts_with("x-amzn-") - }) - .map(|(name, value)| (name.clone(), value.clone())) - .collect() -} - -/// Whether the signer produces `name` itself. -/// -/// Python's reattach loop skips these, so a caller-supplied copy never reaches -/// the wire next to the computed one. -pub fn is_sigv4_computed_header(name: &str) -> bool { - SIGV4_COMPUTED_HEADER_NAMES.contains(&name.to_ascii_lowercase().as_str()) -} - -pub fn sign_bedrock_post( - url: &str, - body: &[u8], - headers: &BTreeMap, - region: &str, - credentials: &Credentials, - signing_time: SystemTime, -) -> Result, Error> { - let identity: Identity = credentials.clone().into(); - let params = v4::SigningParams::builder() - .identity(&identity) - .region(region) - .name(BEDROCK_SERVICE) - .time(signing_time) - .settings(SigningSettings::default()) - .build() - .map(SigningParams::from) - .map_err(|error| Error::Auth(format!("AWS signing parameters failed: {error}")))?; - let header_refs = headers - .iter() - .map(|(name, value)| (name.as_str(), value.as_str())); - let request = SignableRequest::new("POST", url, header_refs, SignableBody::Bytes(body)) - .map_err(|error| Error::Auth(format!("AWS signable request failed: {error}")))?; - let (instructions, _) = sign(request, ¶ms) - .map_err(|error| Error::Auth(format!("AWS request signing failed: {error}")))? - .into_parts(); - Ok(instructions - .headers() - .map(|(name, value)| { - let normalized_name = match name { - "authorization" => "Authorization", - "x-amz-date" => "X-Amz-Date", - "x-amz-security-token" => "X-Amz-Security-Token", - _ => name, - }; - (normalized_name.to_string(), value.to_string()) - }) - .collect()) -} - -/// Model-id and region parsing shared by every Bedrock route. -pub fn bedrock_model_id_and_region(model: &str) -> (String, Option) { - let mut stripped = model; - for prefix in ["bedrock/converse/", "bedrock/", "converse/"] { - if let Some(value) = stripped.strip_prefix(prefix) { - stripped = value; - break; - } - } - let mut region = None; - if let Some((candidate, remainder)) = stripped.split_once('/') - && is_bedrock_region(candidate) - { - region = Some(candidate.to_string()); - stripped = remainder; - } - for prefix in ["nova-2/", "nova/"] { - if let Some(value) = stripped.strip_prefix(prefix) { - stripped = value; - break; - } - } - if region.is_none() { - // Python splits the whole ARN and takes field 3, the region. Stripping - // `arn:` first shifts every field down one, so the region is field 2 - // here; field 3 is the account id. - region = stripped - .strip_prefix("arn:") - .and_then(|value| value.split(':').nth(2)) - .filter(|value| !value.is_empty()) - .map(str::to_string); - } - (stripped.to_string(), region) -} - -fn is_bedrock_region(value: &str) -> bool { - value.len() > 3 - && value.contains('-') - && value - .chars() - .all(|char| char.is_ascii_alphanumeric() || char == '-') -} - -pub fn resolve_bedrock_region( - model_region: Option<&str>, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> String { - if let Some(region) = optional_params - .get("aws_region_name") - .and_then(Value::as_str) - { - return region.to_string(); - } - if let Some(region) = model_region { - return region.to_string(); - } - env_lookup(AWS_REGION_NAME) - .or_else(|| env_lookup(AWS_REGION)) - .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string()) -} - -pub fn aws_auth_config( - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> AwsAuthConfig { - let value = |key: &str| { - optional_params - .get(key) - .and_then(Value::as_str) - .map(str::to_string) - }; - let env = |key: &str| env_lookup(key); - AwsAuthConfig { - access_key_id: value("aws_access_key_id").or_else(|| env("AWS_ACCESS_KEY_ID")), - secret_access_key: value("aws_secret_access_key").or_else(|| env("AWS_SECRET_ACCESS_KEY")), - session_token: value("aws_session_token").or_else(|| env("AWS_SESSION_TOKEN")), - region_name: value("aws_region_name").or_else(|| env(AWS_REGION_NAME)), - session_name: value("aws_session_name").or_else(|| env("AWS_SESSION_NAME")), - profile_name: value("aws_profile_name").or_else(|| env("AWS_PROFILE_NAME")), - role_name: value("aws_role_name").or_else(|| env("AWS_ROLE_NAME")), - web_identity_token: value("aws_web_identity_token") - .or_else(|| env("AWS_WEB_IDENTITY_TOKEN")), - sts_endpoint: value("aws_sts_endpoint").or_else(|| env("AWS_STS_ENDPOINT")), - external_id: value("aws_external_id").or_else(|| env("AWS_EXTERNAL_ID")), - } -} - -/// Credentials a host resolved through its own chain and handed down verbatim. -/// -/// A host with its own resolution (LiteLLM's Python `BaseAWSLLM`, which reads -/// profiles, STS and boto sessions) passes the result here so the core signs -/// with exactly those. Without this the core would re-derive from ambient -/// state, where an unrelated `AWS_ROLE_NAME` or `AWS_PROFILE_NAME` in the -/// environment outranks explicit keys in [`classify_auth`] and the two sides -/// would sign as different principals. -pub fn host_supplied_credentials(optional_params: &Map) -> Option { - let value = |key: &str| { - optional_params - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - }; - let access_key_id = value("aws_access_key_id")?; - let secret_access_key = value("aws_secret_access_key")?; - Some(Credentials::new( - access_key_id, - secret_access_key, - value("aws_session_token").map(str::to_string), - None, - "litellm-host-supplied", - )) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn no_env(_: &str) -> Option { - None - } - - fn parity_inputs() -> (String, Vec, BTreeMap) { - ( - "https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke" - .to_string(), - br#"{"input":"hello"}"#.to_vec(), - BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())]), - ) - } - - #[test] - fn reads_the_region_field_of_a_model_arn_not_the_account_id() { - // Python's `_get_aws_region_from_model_arn` splits the whole ARN and - // takes field 3. Stripping `arn:` first shifts every field down one, so - // the region is field 2 here. Taking field 3 after the strip returns - // the account id, which is not a region at all. - let (_, region) = bedrock_model_id_and_region( - "bedrock/arn:aws:bedrock:us-west-2:123456789012:foundation-model/anthropic.claude-v2", - ); - assert_eq!(region.as_deref(), Some("us-west-2")); - } - - #[test] - fn classification_preserves_python_precedence() { - let config = AwsAuthConfig { - access_key_id: Some("ak".into()), - secret_access_key: Some("sk".into()), - session_token: Some("token".into()), - region_name: Some("us-east-1".into()), - session_name: Some("session".into()), - profile_name: Some("profile".into()), - role_name: Some("role".into()), - web_identity_token: Some("oidc".into()), - ..Default::default() - }; - assert!(matches!( - classify_auth(config, &no_env), - AwsAuthFlow::WebIdentity { .. } - )); - } - - #[test] - fn classification_covers_fallthroughs() { - let env = |key: &str| match key { - AWS_PROFILE_NAME => Some("profile".into()), - _ => None, - }; - assert!(matches!( - classify_auth(AwsAuthConfig::default(), &env), - AwsAuthFlow::Profile { .. } - )); - assert!(matches!( - classify_auth( - AwsAuthConfig { - access_key_id: Some("ak".into()), - secret_access_key: Some("sk".into()), - session_token: Some("token".into()), - ..Default::default() - }, - &no_env - ), - AwsAuthFlow::SessionToken { .. } - )); - assert!(matches!( - classify_auth( - AwsAuthConfig { - access_key_id: Some("ak".into()), - secret_access_key: Some("sk".into()), - region_name: Some("us-east-1".into()), - ..Default::default() - }, - &no_env - ), - AwsAuthFlow::StaticKeys { .. } - )); - assert_eq!( - classify_auth(AwsAuthConfig::default(), &no_env), - AwsAuthFlow::DefaultChain - ); - } - - #[tokio::test] - async fn static_credentials_do_not_use_network() { - let credentials = resolve_credentials( - AwsAuthConfig { - access_key_id: Some("ak".into()), - secret_access_key: Some("sk".into()), - region_name: Some("us-east-1".into()), - ..Default::default() - }, - &no_env, - ) - .await - .expect("static credentials"); - assert_eq!(credentials.access_key_id(), "ak"); - assert_eq!(credentials.session_token(), None); - } - - #[test] - fn cache_policy_matches_python_flows() { - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::StaticKeys { - access_key_id: "ak".into(), - secret_access_key: "sk".into(), - region_name: "us-east-1".into(), - }), - Some(STATIC_CREDENTIALS_TTL) - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::DefaultChain), - Some(AMBIENT_CREDENTIALS_TTL) - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::SessionToken { - access_key_id: "ak".into(), - secret_access_key: "sk".into(), - session_token: "token".into(), - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::Profile { - name: "profile".into() - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::AssumeRole { - role: "arn:aws:iam::123456789012:role/demo".into(), - session_name: None, - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::WebIdentity { - token: "token".into(), - role: "arn:aws:iam::123456789012:role/demo".into(), - session_name: "session".into(), - }), - None - ); - } - - #[test] - fn cache_round_trip_preserves_credentials() { - let key = format!("cache-test-{}", std::process::id()); - let credentials = Credentials::new("cache-ak", "cache-sk", None, None, "test"); - set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL); - assert_eq!( - get_cached_credentials(&key).map(|value| value.access_key_id().to_string()), - Some("cache-ak".to_string()) - ); - } - - #[test] - fn same_role_comparison_matches_partition_account_and_role() { - assert!(same_role_arns( - "arn:aws:iam::123456789012:role/path/demo", - "arn:aws:sts::123456789012:assumed-role/demo/session" - )); - assert!(!same_role_arns( - "arn:aws:iam::123456789012:role/demo", - "arn:aws:iam::999999999999:role/demo" - )); - assert!(!same_role_arns( - "arn:aws:iam::123456789012:role/demo", - "arn:aws-cn:iam::123456789012:role/demo" - )); - assert!(!same_role_arns( - "arn:aws:iam::123456789012:user/demo", - "arn:aws:iam::123456789012:role/demo" - )); - } - - #[test] - fn a_forwarded_client_header_is_not_folded_into_the_signature() { - // Python signs only the AWS header set, so a header a caller forwarded - // cannot change the canonical request. Signing it instead makes the - // request 403 the moment anything on the wire rewrites or drops it. - let (url, body, mut headers) = parity_inputs(); - headers.insert("x-request-id".to_string(), "abc-123".to_string()); - headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); - headers.insert("x-amzn-trace-id".to_string(), "Root=1-abc".to_string()); - let signable = aws_signature_headers(&headers); - - assert!(!signable.contains_key("x-request-id")); - assert!(!signable.contains_key("Accept-Encoding")); - // The AWS-prefixed one is genuinely part of the signature. - assert!(signable.contains_key("x-amzn-trace-id")); - assert!(signable.contains_key("Content-Type")); - - let credentials = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None, - None, - "test", - ); - let signed = sign_bedrock_post( - &url, - &body, - &signable, - "us-east-1", - &credentials, - SystemTime::UNIX_EPOCH, - ) - .expect("signs"); - let authorization = signed - .get("Authorization") - .expect("carries an authorization header"); - assert!( - !authorization.contains("x-request-id"), - "forwarded header reached SignedHeaders: {authorization}" - ); - assert!( - !authorization.contains("accept-encoding"), - "forwarded header reached SignedHeaders: {authorization}" - ); - } - - #[test] - fn signing_matches_botocore_golden_vector() { - let (url, body, headers) = parity_inputs(); - let credentials = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - Some("session-token".to_string()), - None, - "test", - ); - let signed = sign_bedrock_post( - &url, - &body, - &headers, - "us-east-1", - &credentials, - UNIX_EPOCH + std::time::Duration::from_secs(1_704_164_645), - ) - .expect("golden signature"); - assert_eq!( - signed.get("X-Amz-Date").map(String::as_str), - Some("20240102T030405Z") - ); - assert_eq!( - signed.get("X-Amz-Security-Token").map(String::as_str), - Some("session-token") - ); - assert_eq!( - signed.get("Authorization").map(String::as_str), - Some( - "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20240102/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-security-token, Signature=55c027ef47527d3ad63f1735f9d099efdbc99f296ff914bd94e727e24ec0e464" - ) - ); - } - - #[test] - fn signing_without_session_token_omits_security_header() { - let (url, body, headers) = parity_inputs(); - let credentials = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None, - None, - "test", - ); - let signed = sign_bedrock_post( - &url, - &body, - &headers, - "us-east-1", - &credentials, - UNIX_EPOCH + std::time::Duration::from_secs(1_704_164_645), - ) - .expect("signature"); - assert!(!signed.contains_key("X-Amz-Security-Token")); - } - - #[ignore] - #[tokio::test] - async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box> { - let access_key_id = std::env::var("AWS_BEDROCK_TEST_ACCESS_KEY_ID")?; - let secret_access_key = std::env::var("AWS_BEDROCK_TEST_SECRET_ACCESS_KEY")?; - let body = br#"{"anthropic_version":"bedrock-2023-05-31","max_tokens":1,"messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}"#.to_vec(); - let headers = - BTreeMap::from([("Content-Type".to_string(), "application/json".to_string())]); - let credentials = resolve_credentials( - AwsAuthConfig { - access_key_id: Some(access_key_id), - secret_access_key: Some(secret_access_key), - region_name: Some("us-west-2".to_string()), - ..Default::default() - }, - &no_env, - ) - .await?; - let client = reqwest::Client::new(); - let mut failures = Vec::new(); - - for region in ["us-west-2", "us-east-1"] { - let url = format!( - "https://bedrock-runtime.{region}.amazonaws.com/model/us.anthropic.claude-opus-4-8/invoke" - ); - let signed_headers = sign_bedrock_post( - &url, - &body, - &headers, - region, - &credentials, - SystemTime::now(), - )?; - let mut request = client.post(&url).body(body.clone()); - for (name, value) in &headers { - request = request.header(name, value); - } - for (name, value) in signed_headers { - request = request.header(name, value); - } - let response = request.send().await?; - let status = response.status(); - let response_body = response.text().await?; - let snippet: String = response_body.chars().take(240).collect(); - println!("region={region} status={status} response={snippet}"); - if status == reqwest::StatusCode::OK { - return Ok(()); - } - failures.push(format!("{region}: {status} {snippet}")); - } - - panic!( - "no Bedrock region returned HTTP 200: {}", - failures.join("; ") - ); - } -} +pub use litellm_auth_aws::*; diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs index c86f061b9ca..74716a2200b 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs @@ -1,5 +1,5 @@ use super::*; -use crate::Error; +use crate::chat_completions::Error; use serde_json::json; fn messages(value: Value) -> Vec { diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index 7be3d108d44..19efaf833bd 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -1,5 +1,6 @@ use serde_json::{Map, Value, json}; +use crate::chat_completions::Error; use crate::chat_completions::conversation::{Conversation, TurnRole, build_conversation}; use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage_from_parts}; use crate::chat_completions::transformation::{ @@ -11,7 +12,6 @@ use crate::chat_completions::types::{ ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, }; -use crate::error::Error; use super::super::aws_base::{bedrock_model_id_and_region, resolve_bedrock_region}; use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; @@ -163,7 +163,6 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &[("Content-Type", "application/json")] } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { SUPPORTED_PARAMS } diff --git a/litellm-rust/crates/core/src/providers/bedrock/constants.rs b/litellm-rust/crates/core/src/providers/bedrock/constants.rs index be215cc9016..663f887c1fd 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/constants.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/constants.rs @@ -1,43 +1 @@ -pub const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; -pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; -pub const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; -pub const AWS_REGION_NAME: &str = "AWS_REGION_NAME"; -pub const AWS_REGION: &str = "AWS_REGION"; -pub const AWS_SESSION_NAME: &str = "AWS_SESSION_NAME"; -pub const AWS_PROFILE_NAME: &str = "AWS_PROFILE_NAME"; -pub const AWS_ROLE_NAME: &str = "AWS_ROLE_NAME"; -pub const AWS_WEB_IDENTITY_TOKEN: &str = "AWS_WEB_IDENTITY_TOKEN"; -pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN"; -pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; -pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT"; -pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID"; -pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; - -/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors -/// Python's `_filter_headers_for_aws_signature` allowlist. -pub const AWS_SIGNED_HEADER_NAMES: &[&str] = &[ - "host", - "content-type", - "date", - "x-amz-date", - "x-amz-security-token", - "x-amz-content-sha256", - "x-amz-algorithm", - "x-amz-credential", - "x-amz-signedheaders", - "x-amz-signature", -]; -/// Headers the signer emits itself. Mirrors Python's `SIGV4_COMPUTED_HEADERS`, -/// which the reattach loop skips so a caller's copy cannot ride alongside the -/// computed one. -pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[ - "authorization", - "x-amz-date", - "x-amz-security-token", - "date", -]; -pub const BEDROCK_SERVICE: &str = "bedrock"; -pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session"; -pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2"; -pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str = - "https://bedrock-runtime.{region}.amazonaws.com"; +pub use litellm_auth_aws::constants::*; diff --git a/litellm-rust/crates/core/src/providers/bedrock/mod.rs b/litellm-rust/crates/core/src/providers/bedrock/mod.rs index d9cd3efcb74..5c849064989 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/mod.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/mod.rs @@ -2,7 +2,6 @@ //! with Python's `BaseAWSLLM`; the broader core purity guidance is reconciled //! separately. -#[cfg(feature = "bedrock-auth")] pub mod audio_transcription; pub mod aws_base; pub mod chat_completions; diff --git a/litellm-rust/crates/core/src/routing_utils/provider.rs b/litellm-rust/crates/core/src/providers/custom_llm_provider.rs similarity index 100% rename from litellm-rust/crates/core/src/routing_utils/provider.rs rename to litellm-rust/crates/core/src/providers/custom_llm_provider.rs diff --git a/litellm-rust/crates/core/src/providers/mod.rs b/litellm-rust/crates/core/src/providers/mod.rs index 1aeb75063d6..70ca4386fff 100644 --- a/litellm-rust/crates/core/src/providers/mod.rs +++ b/litellm-rust/crates/core/src/providers/mod.rs @@ -1,5 +1,5 @@ pub mod anthropic; pub mod azure_ai; -#[cfg(feature = "bedrock-auth")] pub mod bedrock; +pub mod custom_llm_provider; pub mod openai; diff --git a/litellm-rust/crates/core/src/providers/openai/mod.rs b/litellm-rust/crates/core/src/providers/openai/mod.rs index 62fcc50f2ac..b396b037bc5 100644 --- a/litellm-rust/crates/core/src/providers/openai/mod.rs +++ b/litellm-rust/crates/core/src/providers/openai/mod.rs @@ -1,2 +1 @@ -pub mod realtime; pub mod responses; diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs b/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs deleted file mode 100644 index f1985f81b7d..00000000000 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ /dev/null @@ -1,189 +0,0 @@ -use crate::Error; -use crate::realtime::transformation::RealtimeProviderConfig; -use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; - -/// Default OpenAI API base, used when the caller does not override `api_base`. -pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com"; - -/// Path appended to the resolved host base to reach the realtime endpoint. -pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime"; - -/// Percent-encode a query value, escaping any char outside the RFC 3986 -/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime -/// model slugs have no special chars, but this stays correct for the rest. -fn percent_encode(value: &str) -> String { - let mut encoded = String::with_capacity(value.len()); - for byte in value.bytes() { - let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~'); - if unreserved { - encoded.push(byte as char); - } else { - encoded.push('%'); - encoded.push_str(&format!("{byte:02X}")); - } - } - encoded -} - -/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`. -/// -/// Blank/whitespace `api_base` is treated as absent (guard at resolution time), -/// falling back to the default. The scheme is swapped to its WebSocket -/// equivalent (`https://`→`wss://`, `http://`→`ws://`); bases already using -/// `ws`/`wss` are left untouched. A bare host or unrecognized scheme defaults to -/// secure `wss://` so we never hand a scheme-less URL to the connector (this is -/// a deliberate hardening over Python's `_construct_url`, which would emit a -/// scheme-less URL here). A trailing `/` is trimmed before the path and -/// `?model=` are appended. -pub fn complete_url(api_base: Option<&str>, model: &str) -> String { - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(OPENAI_REALTIME_DEFAULT_API_BASE); - - let base = if let Some(rest) = base.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = base.strip_prefix("http://") { - format!("ws://{rest}") - } else if base.starts_with("wss://") || base.starts_with("ws://") { - base.to_string() - } else { - format!("wss://{base}") - }; - - let base = base.trim_end_matches('/'); - - format!( - "{base}{OPENAI_REALTIME_PATH}?model={}", - percent_encode(model) - ) -} - -pub struct OpenAiRealtimeConfig; - -pub const OPENAI_REALTIME_CONFIG: OpenAiRealtimeConfig = OpenAiRealtimeConfig; - -impl RealtimeProviderConfig for OpenAiRealtimeConfig { - fn complete_url(&self, api_base: Option<&str>, model: &str) -> String { - complete_url(api_base, model) - } - - fn transform_realtime_request( - &self, - event: &RealtimeEvent, - _model: &str, - ) -> Result { - Ok(RealtimeTransformResult::passthrough(event.clone())) - } - - fn transform_realtime_response( - &self, - event: &RealtimeEvent, - _model: &str, - ) -> Result { - Ok(RealtimeTransformResult::passthrough(event.clone())) - } -} - -pub fn transform_realtime_request( - event: &RealtimeEvent, - model: &str, -) -> Result { - OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) -} - -pub fn transform_realtime_response( - event: &RealtimeEvent, - model: &str, -) -> Result { - OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn complete_url_defaults_to_openai_wss() { - assert_eq!( - complete_url(None, "gpt-4o-realtime-preview"), - "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_blank_base_uses_default() { - assert_eq!( - complete_url(Some(" "), "gpt-4o-realtime-preview"), - "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_swaps_http_to_ws() { - assert_eq!( - complete_url(Some("http://localhost:8080"), "gpt-4o-realtime-preview"), - "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_dedupes_trailing_slash() { - assert_eq!( - complete_url(Some("https://api.openai.com/"), "gpt-4o-realtime-preview"), - "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_custom_base() { - assert_eq!( - complete_url(Some("https://oai.azure.example"), "gpt-4o-realtime-preview"), - "wss://oai.azure.example/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_preserves_existing_wss_scheme() { - assert_eq!( - complete_url(Some("wss://api.openai.com"), "gpt-realtime"), - "wss://api.openai.com/v1/realtime?model=gpt-realtime" - ); - } - - #[test] - fn complete_url_bare_host_defaults_to_wss() { - assert_eq!( - complete_url(Some("api.openai.com"), "gpt-realtime"), - "wss://api.openai.com/v1/realtime?model=gpt-realtime" - ); - } - - #[test] - fn complete_url_percent_encodes_model_space() { - assert_eq!( - complete_url(None, "gpt 4o"), - "wss://api.openai.com/v1/realtime?model=gpt%204o" - ); - } - - #[test] - fn transform_realtime_request_passthrough_preserves_event() { - let event: RealtimeEvent = - serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#) - .expect("valid event"); - let result = - transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible"); - assert_eq!(result.events, vec![event]); - } - - #[test] - fn transform_realtime_response_passthrough_preserves_event() { - let event: RealtimeEvent = - serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#) - .expect("valid event"); - let result = - transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible"); - assert_eq!(result.events, vec![event]); - } -} diff --git a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs index be86bb90311..6203b195d5e 100644 --- a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs @@ -1,4 +1,4 @@ -use crate::Error; +use crate::responses::Error; use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult}; use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model}; diff --git a/litellm-rust/crates/core/src/realtime/mod.rs b/litellm-rust/crates/core/src/realtime/mod.rs deleted file mode 100644 index ec2fbb969a6..00000000000 --- a/litellm-rust/crates/core/src/realtime/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod transformation; -pub mod types; diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs deleted file mode 100644 index b08084514ef..00000000000 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::Error; -use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; - -pub trait RealtimeProviderConfig { - /// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`). - /// Pure string construction only — no network, no env. - fn complete_url(&self, api_base: Option<&str>, model: &str) -> String; - - /// Transform a client → backend event before it is forwarded upstream. - fn transform_realtime_request( - &self, - event: &RealtimeEvent, - model: &str, - ) -> Result; - - /// Transform a backend → client event before it is forwarded downstream. - fn transform_realtime_response( - &self, - event: &RealtimeEvent, - model: &str, - ) -> Result; -} diff --git a/litellm-rust/crates/core/src/realtime/types.rs b/litellm-rust/crates/core/src/realtime/types.rs deleted file mode 100644 index 3b59224b6e9..00000000000 --- a/litellm-rust/crates/core/src/realtime/types.rs +++ /dev/null @@ -1,60 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -/// A single realtime event exchanged over the WebSocket. -/// -/// The `type` discriminator is a typed field; the remaining fields are -/// preserved losslessly in `data` so a transform can pass an event through, or -/// inspect/modify specific fields, without enumerating every event variant. -/// Wire (de)serialization happens at the host edge — `core`/`providers` operate -/// only on this typed form. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct RealtimeEvent { - #[serde(rename = "type")] - pub event_type: String, - #[serde(flatten)] - pub data: Map, -} - -/// One or more typed events produced by a realtime transform. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct RealtimeTransformResult { - pub events: Vec, -} - -impl RealtimeTransformResult { - /// Forward a single event unchanged (the OpenAI baseline). - pub fn passthrough(event: RealtimeEvent) -> Self { - Self { - events: vec![event], - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn event(raw: &str) -> RealtimeEvent { - serde_json::from_str(raw).expect("valid event json") - } - - #[test] - fn realtime_event_round_trips_type_and_extra_fields() { - let raw = r#"{"type":"response.output_text.delta","delta":"hi","response_id":"r1"}"#; - let parsed = event(raw); - assert_eq!(parsed.event_type, "response.output_text.delta"); - assert_eq!(parsed.data.get("delta"), Some(&Value::String("hi".into()))); - // Re-serializing yields a semantically-equal event (key order may differ). - let reparsed: RealtimeEvent = - serde_json::from_str(&serde_json::to_string(&parsed).unwrap()).unwrap(); - assert_eq!(parsed, reparsed); - } - - #[test] - fn passthrough_produces_single_element_vec() { - let parsed = event(r#"{"type":"session.update"}"#); - let result = RealtimeTransformResult::passthrough(parsed.clone()); - assert_eq!(result.events, vec![parsed]); - } -} diff --git a/litellm-rust/crates/core/src/responses/error.rs b/litellm-rust/crates/core/src/responses/error.rs new file mode 100644 index 00000000000..8bea035f0b0 --- /dev/null +++ b/litellm-rust/crates/core/src/responses/error.rs @@ -0,0 +1,17 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("invalid provider: {0}")] + InvalidProvider(String), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("routing error: {0}")] + Routing(String), + #[error(transparent)] + Auth(#[from] litellm_auth::Error), + #[error(transparent)] + Transport(#[from] crate::transport::Error), + #[error(transparent)] + Headers(#[from] crate::http_utils::HeaderError), +} diff --git a/litellm-rust/crates/core/src/responses/instrumentation.rs b/litellm-rust/crates/core/src/responses/instrumentation.rs index b1098f4d386..b1cf5ae09d8 100644 --- a/litellm-rust/crates/core/src/responses/instrumentation.rs +++ b/litellm-rust/crates/core/src/responses/instrumentation.rs @@ -5,7 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::Value; -use crate::Error; +use super::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType}; @@ -208,6 +208,7 @@ impl ResponsesWsInstrumentation { type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { + type Error = Error; type PreCallFuture<'a> = LifecycleFuture<'a, ()>; type DuringCallFuture<'a> = LifecycleFuture<'a, ()>; type SuccessFuture<'a> = Pin + Send + 'a>>; diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/core/src/responses/mod.rs index 5ec5a2caef8..f8b6d27ffab 100644 --- a/litellm-rust/crates/core/src/responses/mod.rs +++ b/litellm-rust/crates/core/src/responses/mod.rs @@ -1,3 +1,5 @@ +mod error; +pub use error::Error; pub mod instrumentation; pub mod types; pub mod websocket; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 34213e5f6c4..ab7738e81b9 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -16,7 +16,7 @@ use tokio_tungstenite::{ Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, }; -use crate::Error; +use super::Error; use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; @@ -204,9 +204,9 @@ impl ResponsesWebSocketConnection { headers: &HashMap, timeout: Option, ) -> Result { - let mut request = url - .into_client_request() - .map_err(|error| Error::Network(error.to_string()))?; + let mut request = url.into_client_request().map_err(|error| { + Error::Transport(crate::transport::Error::Network(error.to_string())) + })?; for (name, value) in headers { let header_name = name .parse::() @@ -217,17 +217,21 @@ impl ResponsesWebSocketConnection { } let connect = connect_upstream(request); let result = match timeout { - Some(timeout) => tokio::time::timeout(timeout, connect) - .await - .map_err(|_| Error::Network("Responses WebSocket connection timed out".into()))?, + Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { + Error::Transport(crate::transport::Error::Network( + "Responses WebSocket connection timed out".into(), + )) + })?, None => connect.await, }; let (socket, _) = result.map_err(|error| match *error { - tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { - status: response.status().as_u16(), - body: String::new(), - }, - other => Error::Network(other.to_string()), + tokio_tungstenite::tungstenite::Error::Http(response) => { + Error::Transport(crate::transport::Error::Http { + status: response.status().as_u16(), + body: String::new(), + }) + } + other => Error::Transport(crate::transport::Error::Network(other.to_string())), })?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), @@ -237,12 +241,14 @@ impl ResponsesWebSocketConnection { pub async fn send_text(&self, text: String) -> Result<(), Error> { let mut socket = self.socket.lock().await; let Some(socket) = socket.as_mut() else { - return Err(Error::Network("Responses WebSocket is closed".into())); + return Err(Error::Transport(crate::transport::Error::Network( + "Responses WebSocket is closed".into(), + ))); }; socket .send(Message::Text(text)) .await - .map_err(|error| Error::Network(error.to_string())) + .map_err(|error| Error::Transport(crate::transport::Error::Network(error.to_string()))) } pub async fn recv_text(&self) -> Result, Error> { @@ -257,17 +263,18 @@ impl ResponsesWebSocketConnection { .map_err(|error| Error::InvalidResponse(error.to_string())), Some(Ok(Message::Close(_))) | None => Ok(None), Some(Ok(_)) => Ok(None), - Some(Err(error)) => Err(Error::Network(error.to_string())), + Some(Err(error)) => Err(Error::Transport(crate::transport::Error::Network( + error.to_string(), + ))), } } pub async fn close(&self) -> Result<(), Error> { let mut socket = self.socket.lock().await; if let Some(socket) = socket.as_mut() { - socket - .close(None) - .await - .map_err(|error| Error::Network(error.to_string()))?; + socket.close(None).await.map_err(|error| { + Error::Transport(crate::transport::Error::Network(error.to_string())) + })?; } *socket = None; Ok(()) diff --git a/litellm-rust/crates/core/src/router/deployment.rs b/litellm-rust/crates/core/src/router/deployment.rs deleted file mode 100644 index 1ee88e682a3..00000000000 --- a/litellm-rust/crates/core/src/router/deployment.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! `model_list` data types, mirroring Python's deployment dict. Deserialize-ready -//! so a deployment can be loaded straight from the proxy config's `model_list`. - -use serde::Deserialize; - -/// Per-deployment call parameters, mirroring Python's `litellm_params`. -#[derive(Clone, Debug, Deserialize)] -pub struct LiteLLMParams { - /// Provider model, e.g. `gpt-realtime` or `openai/gpt-realtime`. - pub model: String, - #[serde(default)] - pub api_key: Option, - #[serde(default)] - pub api_base: Option, -} - -/// One entry of the `model_list`, mirroring Python's deployment dict. -#[derive(Clone, Debug, Deserialize)] -pub struct Deployment { - /// Public alias clients request, e.g. `gpt-realtime`. - pub model_name: String, - pub litellm_params: LiteLLMParams, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn deserializes_from_model_list_entry() { - let entry = r#"{ - "model_name": "gpt-realtime", - "litellm_params": {"model": "openai/gpt-realtime", "api_base": "https://x"} - }"#; - let deployment: Deployment = serde_json::from_str(entry).expect("valid entry"); - assert_eq!(deployment.model_name, "gpt-realtime"); - assert_eq!(deployment.litellm_params.model, "openai/gpt-realtime"); - assert_eq!(deployment.litellm_params.api_key, None); - assert_eq!( - deployment.litellm_params.api_base.as_deref(), - Some("https://x") - ); - } -} diff --git a/litellm-rust/crates/core/src/router/mod.rs b/litellm-rust/crates/core/src/router/mod.rs deleted file mode 100644 index 96bc91bc6b5..00000000000 --- a/litellm-rust/crates/core/src/router/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Minimal Rust port of LiteLLM's `router.py` deployment selection. -//! -//! A [`Router`] is built from a `model_list` of [`Deployment`]s -//! (`{ model_name, litellm_params: { model, api_key, api_base } }`) and selects -//! one per request via a [`RoutingStrategy`]. For now the only strategy is -//! `simple-shuffle` — a uniform random pick within a `model_name` group. -//! -//! This stays pure (no I/O): it only *chooses* a deployment. The host (the -//! gateway) takes the chosen deployment and performs the actual provider call. -//! -//! - [`deployment`] — the `model_list` data types. -//! - [`strategy`] — how a deployment is chosen. - -mod deployment; -mod strategy; - -pub use deployment::{Deployment, LiteLLMParams}; -pub use strategy::RoutingStrategy; - -/// Load-balancing router over a `model_list`. -#[derive(Clone, Debug, Default)] -pub struct Router { - model_list: Vec, - routing_strategy: RoutingStrategy, -} - -impl Router { - /// Build a router from a `model_list` using the default `simple-shuffle` strategy. - pub fn new(model_list: Vec) -> Self { - Self { - model_list, - routing_strategy: RoutingStrategy::SimpleShuffle, - } - } - - /// All deployments in the `model_list`. Read-only; used by the host to - /// enumerate upstreams (e.g. to pre-warm a connection pool per deployment). - pub fn deployments(&self) -> &[Deployment] { - &self.model_list - } - - /// Whether any deployment is registered under `model`. - pub fn has_deployment(&self, model: &str) -> bool { - self.model_list - .iter() - .any(|deployment| deployment.model_name == model) - } - - /// Pick a deployment for `model` per the routing strategy. Returns `None` - /// when no deployment is registered under that `model_name`. - pub fn get_available_deployment(&self, model: &str) -> Option<&Deployment> { - let candidates: Vec<&Deployment> = self - .model_list - .iter() - .filter(|deployment| deployment.model_name == model) - .collect(); - self.routing_strategy.select(&candidates) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn deployment(name: &str, model: &str) -> Deployment { - Deployment { - model_name: name.to_string(), - litellm_params: LiteLLMParams { - model: model.to_string(), - api_key: None, - api_base: None, - }, - } - } - - #[test] - fn selects_a_matching_deployment() { - let router = Router::new(vec![ - deployment("gpt-realtime", "gpt-realtime"), - deployment("other", "other-model"), - ]); - let chosen = router - .get_available_deployment("gpt-realtime") - .expect("a deployment should match"); - assert_eq!(chosen.model_name, "gpt-realtime"); - } - - #[test] - fn unknown_model_returns_none() { - let router = Router::new(vec![deployment("gpt-realtime", "gpt-realtime")]); - assert!(router.get_available_deployment("missing").is_none()); - } -} diff --git a/litellm-rust/crates/core/src/router/strategy/mod.rs b/litellm-rust/crates/core/src/router/strategy/mod.rs deleted file mode 100644 index 7e8ac217db3..00000000000 --- a/litellm-rust/crates/core/src/router/strategy/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Routing policy: how the router picks one deployment from a model group. -//! -//! One module per strategy; [`RoutingStrategy::select`] dispatches to it. New -//! strategies (least-busy, latency-based, …) get their own file here. - -mod simple_shuffle; - -use super::Deployment; - -/// How the router chooses among the deployments sharing a `model_name`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum RoutingStrategy { - /// Uniform random pick among the matching deployments. - #[default] - SimpleShuffle, -} - -impl RoutingStrategy { - /// Choose one deployment from `candidates` (all sharing the requested - /// `model_name`). Returns `None` when there are no candidates. - pub fn select<'a>(&self, candidates: &[&'a Deployment]) -> Option<&'a Deployment> { - match self { - RoutingStrategy::SimpleShuffle => simple_shuffle::select(candidates), - } - } -} diff --git a/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs b/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs deleted file mode 100644 index 74ce0c21e80..00000000000 --- a/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! `simple-shuffle`: a uniform random pick among the candidate deployments. - -use rand::seq::SliceRandom; - -use crate::router::Deployment; - -/// Uniform random choice among `candidates` (all sharing the requested -/// `model_name`). Returns `None` when there are no candidates. -pub fn select<'a>(candidates: &[&'a Deployment]) -> Option<&'a Deployment> { - candidates.choose(&mut rand::thread_rng()).copied() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::router::{Deployment, LiteLLMParams}; - - fn deployment(model: &str) -> Deployment { - Deployment { - model_name: "gpt-realtime".to_string(), - litellm_params: LiteLLMParams { - model: model.to_string(), - api_key: None, - api_base: None, - }, - } - } - - #[test] - fn picks_from_candidates() { - let a = deployment("key-a"); - let b = deployment("key-b"); - let candidates = vec![&a, &b]; - for _ in 0..20 { - let chosen = select(&candidates).expect("non-empty"); - assert!(matches!( - chosen.litellm_params.model.as_str(), - "key-a" | "key-b" - )); - } - } - - #[test] - fn empty_candidates_select_none() { - assert!(select(&[]).is_none()); - } -} diff --git a/litellm-rust/crates/core/src/routing_utils/README.md b/litellm-rust/crates/core/src/routing_utils/README.md deleted file mode 100644 index 8585c18e421..00000000000 --- a/litellm-rust/crates/core/src/routing_utils/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Routing Utils - -Shared helpers for deciding how a LiteLLM model routes to an LLM provider. -Keep provider-name parsing, explicit `custom_llm_provider` handling, and model-prefix normalization here. -Do not put deployment selection or load-balancing logic here; that belongs in `router`. -Do not put provider HTTP transformation logic here; that belongs in `providers`. -Helpers in this folder should be deterministic and easy to unit test without network calls. diff --git a/litellm-rust/crates/core/src/routing_utils/mod.rs b/litellm-rust/crates/core/src/routing_utils/mod.rs deleted file mode 100644 index 8336397f870..00000000000 --- a/litellm-rust/crates/core/src/routing_utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod provider; diff --git a/litellm-rust/crates/core/src/transport/error.rs b/litellm-rust/crates/core/src/transport/error.rs new file mode 100644 index 00000000000..eff15365ea8 --- /dev/null +++ b/litellm-rust/crates/core/src/transport/error.rs @@ -0,0 +1,75 @@ +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + #[error("upstream request failed with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("upstream network error: {0}")] + Network(String), + #[error("could not reach the provider: {0}")] + Connect(String), +} + +impl Error { + pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self { + let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder()); + let message = error.without_url().to_string(); + if before_dispatch { + Self::Connect(message) + } else { + Self::Network(message) + } + } +} + +impl From for Error { + fn from(error: reqwest::Error) -> Self { + Self::Network(error.without_url().to_string()) + } +} + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn transport_errors_remove_urls_and_keep_dispatch_context() { + let error = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get("http://localhost:invalid/private?api_key=secret") + .send() + .await + .expect_err("invalid port"); + let error = crate::transport::Error::from_reqwest_before_dispatch(error); + assert!(matches!(error, crate::transport::Error::Connect(_))); + assert!(!error.to_string().contains("secret")); + assert!(!error.to_string().contains("private")); + } + + #[tokio::test] + async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() { + use std::time::Duration; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let address = listener.local_addr().expect("address"); + let request = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get(format!("http://{address}")) + .timeout(Duration::from_millis(200)) + .send(); + let (response, accepted) = tokio::join!( + request, + tokio::time::timeout(Duration::from_secs(2), listener.accept()) + ); + let _connection = accepted + .expect("accept deadline") + .expect("accepted connection"); + let error = response.expect_err("server does not respond"); + assert!(error.is_timeout()); + assert!(matches!( + crate::transport::Error::from_reqwest_before_dispatch(error), + crate::transport::Error::Network(_) + )); + } +} diff --git a/litellm-rust/crates/core/src/transport/mod.rs b/litellm-rust/crates/core/src/transport/mod.rs new file mode 100644 index 00000000000..0405e9de3c3 --- /dev/null +++ b/litellm-rust/crates/core/src/transport/mod.rs @@ -0,0 +1,2 @@ +mod error; +pub use error::Error; diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core/src/url_utils.rs index 1150f93a5c7..b8d82b7a04a 100644 --- a/litellm-rust/crates/core/src/url_utils.rs +++ b/litellm-rust/crates/core/src/url_utils.rs @@ -1,9 +1,8 @@ use std::marker::PhantomData; -use thiserror::Error; use url::Url; -#[derive(Debug, Error)] +#[derive(Debug, thiserror::Error)] pub(crate) enum ApiUrlError { #[error("invalid URL: {0}")] Parse(#[from] url::ParseError), diff --git a/litellm-rust/crates/core/tests/host_lifecycle.rs b/litellm-rust/crates/core/tests/host_lifecycle.rs index 19fb946afde..0e58462af1a 100644 --- a/litellm-rust/crates/core/tests/host_lifecycle.rs +++ b/litellm-rust/crates/core/tests/host_lifecycle.rs @@ -1,5 +1,5 @@ -use crate::Error; use crate::call_lifecycle::host::{HostFailure, HostLifecycle, HostPhase}; +use crate::ocr::Error; fn run(fail_at: Option, asynchronous: bool) -> (Vec, Vec) { let mut lifecycle = HostLifecycle::new(asynchronous); @@ -80,14 +80,14 @@ fn only_provider_and_response_construction_failures_use_provider_mapping() { fn failure_handler_errors_do_not_replace_selected_failure_or_suppress_async_dispatch() { let mut lifecycle = HostLifecycle::new(true); while lifecycle.phase() != HostPhase::Execute { - lifecycle.accept(Ok(())); + lifecycle.accept::(Ok(())); } let selected = Error::InvalidRequest("provider".into()); assert_eq!( lifecycle.accept(Err(HostFailure::Error(selected.clone()))), Some(selected) ); - lifecycle.accept(Ok(())); + lifecycle.accept::(Ok(())); for phase in [ HostPhase::DeploymentFailure, HostPhase::Failure, diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 55f8713d76e..373972cf68b 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -140,7 +140,7 @@ impl OcrHooks for RecordingHooks { Box::pin(async move { self.events.lock().unwrap().push("pre"); if self.block { - return Err(crate::Error::InvalidRequest("blocked".into())); + return Err(crate::ocr::Error::InvalidRequest("blocked".into())); } Ok(request) }) @@ -177,7 +177,7 @@ impl OcrHooks for RecordingHooks { fn failure<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a crate::Error, + _error: &'a crate::ocr::Error, _timing: &'a CallLifecycleTiming, ) -> OcrLogFuture<'a> { Box::pin(async move { @@ -251,7 +251,7 @@ async fn lifecycle_blocking_prevents_execution_and_emits_one_failure() { ..request }; let error = perform_ocr(request).await.unwrap_err(); - assert!(matches!(error, crate::Error::InvalidRequest(_))); + assert!(matches!(error, crate::ocr::Error::InvalidRequest(_))); assert_eq!(*events.lock().unwrap(), ["pre", "failure"]); } @@ -358,7 +358,7 @@ async fn fallible_host_phases_do_not_replay_or_reach_transport() { OcrHostOperation::PreCall(request) => { phases.push("pre"); result = Some(OcrHostResult::PreCall(if failure_phase == "pre" { - Err(crate::Error::InvalidRequest("pre failed".into())) + Err(crate::ocr::Error::InvalidRequest("pre failed".into())) } else { Ok(request) })); @@ -366,7 +366,7 @@ async fn fallible_host_phases_do_not_replay_or_reach_transport() { OcrHostOperation::DuringCall(request) => { phases.push("during"); result = Some(OcrHostResult::DuringCall(if failure_phase == "during" { - Err(crate::Error::InvalidRequest("during failed".into())) + Err(crate::ocr::Error::InvalidRequest("during failed".into())) } else { Ok(request) })); @@ -377,7 +377,7 @@ async fn fallible_host_phases_do_not_replay_or_reach_transport() { Ok(OcrCallStep::Complete(_)) => panic!("failed call completed"), } }; - assert!(matches!(error, crate::Error::InvalidRequest(_))); + assert!(matches!(error, crate::ocr::Error::InvalidRequest(_))); assert_eq!( phases .iter() @@ -420,7 +420,7 @@ async fn invalid_provider_response_runs_post_call_before_normalization_failure() } }; server.await.unwrap(); - assert!(matches!(error, crate::Error::InvalidResponse(_))); + assert!(matches!(error, crate::ocr::Error::InvalidResponse(_))); assert_eq!(seen.lock().unwrap().len(), 1); assert_eq!(post_calls, [json!(r#"{"pages":"invalid"}"#)]); } @@ -497,7 +497,7 @@ async fn direct_native_host_drives_the_same_state_machine() { ); assert!(matches!( call.resume(None).await, - Err(crate::Error::InvalidRequest(_)) + Err(crate::ocr::Error::InvalidRequest(_)) )); } @@ -516,7 +516,7 @@ async fn public_finalization_failure_never_dispatches_success_or_replays_provide ) else { panic!("supported call declined") }; - let selected = crate::Error::InvalidRequest("public metadata failed".into()); + let selected = crate::ocr::Error::InvalidRequest("public metadata failed".into()); let host = NoopOcrHost; let mut result = None; let mut failures = Vec::new(); @@ -531,7 +531,7 @@ async fn public_finalization_failure_never_dispatches_success_or_replays_provide assert_eq!(error, selected); failures.push("sync"); OcrHostResult::Lifecycle(Err(HostFailure::Error( - crate::Error::InvalidRequest("failure callback failed".into()), + crate::ocr::Error::InvalidRequest("failure callback failed".into()), ))) } OcrHostOperation::Lifecycle(HostPhase::AsyncFailure) => { @@ -590,7 +590,7 @@ async fn cancellation_at_provider_hook_prevents_execution_and_further_resumption OcrCallStep::Complete(_) => panic!("provider executed before pre-call result"), } } - let selected = crate::Error::InvalidRequest("cancelled".into()); + let selected = crate::ocr::Error::InvalidRequest("cancelled".into()); assert!(matches!( call.interrupt(HostFailure::Cancelled(selected.clone())).await, Err(error) if error == selected @@ -694,10 +694,7 @@ async fn oversized_error_retains_http_status_and_bounded_diagnostics_without_dra .await .unwrap_err(); match error { - super::error::OcrError::Transport(crate::error::TransportError::Http { - status, - body, - }) => { + super::error::OcrError::Transport(crate::transport::Error::Http { status, body }) => { assert_eq!(status, 429); assert_eq!( body, @@ -755,8 +752,8 @@ impl Drop for TokenFutureDrop { } } -impl crate::auth::TokenProvider for PendingToken { - fn acquire(&self) -> crate::auth::TokenFuture<'_> { +impl litellm_auth::TokenProvider for PendingToken { + fn acquire(&self) -> litellm_auth::TokenFuture<'_> { Box::pin(async move { let _guard = TokenFutureDrop(self.dropped.clone()); self.entered.notify_one(); @@ -781,7 +778,7 @@ async fn cancellation_waits_for_provider_capture_drop_even_when_acknowledgement_ extra_headers: vec![("authorization".into(), "Bearer test-key".into())], ..request.connection }, - azure_ad_token_provider: Some(crate::auth::TokenProviderHandle::new(Arc::new( + azure_ad_token_provider: Some(litellm_auth::TokenProviderHandle::new(Arc::new( PendingToken { entered: entered.clone(), dropped: dropped.clone(), @@ -811,7 +808,7 @@ async fn cancellation_waits_for_provider_capture_drop_even_when_acknowledgement_ } }).await.unwrap(); assert!(!dropped.load(Ordering::SeqCst)); - let selected = crate::Error::InvalidRequest("cancelled".into()); + let selected = crate::ocr::Error::InvalidRequest("cancelled".into()); if interrupt_acknowledgement { let mut acknowledgement = Box::pin(call.interrupt(HostFailure::Cancelled(selected.clone()))); diff --git a/litellm-rust/crates/core/tests/ocr/support.rs b/litellm-rust/crates/core/tests/ocr/support.rs index a2e67dffc7d..c7b64e300f0 100644 --- a/litellm-rust/crates/core/tests/ocr/support.rs +++ b/litellm-rust/crates/core/tests/ocr/support.rs @@ -17,7 +17,7 @@ pub(crate) fn ocr_client() -> OcrClient { pub(crate) async fn perform_ocr( request: LiteLLMOcrRequest, -) -> Result { +) -> Result { ocr_client().perform(request).await } diff --git a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs index 676799eb2fe..a73c1e7710a 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs @@ -1,7 +1,7 @@ use serde_json::{Value, json}; use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; -use crate::auth::InputSource; +use litellm_auth::InputSource; fn request_body(request: &str) -> Value { serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() diff --git a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs index 96a19dd62b4..93e9efca849 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs @@ -1,7 +1,7 @@ use serde_json::{Value, json}; use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; -use crate::auth::InputSource; +use litellm_auth::InputSource; fn request_body(request: &str) -> Value { serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 42fad740870..1562d4c1021 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -14,15 +14,11 @@ default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] panic-test = [] -trace-parity = [ - "dep:tracing", - "litellm-core/observability", -] [dependencies] futures-util.workspace = true -tracing = { workspace = true, optional = true } -litellm-core = { workspace = true, features = ["bedrock-auth"] } +litellm-core.workspace = true +litellm-auth.workspace = true litellm-token-counter.workspace = true litellm-python-interop.workspace = true pyo3.workspace = true @@ -35,7 +31,6 @@ tokio = { workspace = true, features = ["sync"] } criterion.workspace = true rstest.workspace = true tokio-tungstenite.workspace = true -tracing.workspace = true [[bench]] name = "serialization" diff --git a/litellm-rust/crates/python-bridge/src/auth.rs b/litellm-rust/crates/python-bridge/src/auth.rs index 8dc0b7aabf0..dcc1a60e9f0 100644 --- a/litellm-rust/crates/python-bridge/src/auth.rs +++ b/litellm-rust/crates/python-bridge/src/auth.rs @@ -1,4 +1,4 @@ -use litellm_core::auth::{ResolvedCredential, SecretValue}; +use litellm_auth::{ResolvedCredential, SecretValue}; use pyo3::exceptions::{PyException, PyRuntimeError, PyTypeError}; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 701c6abb68c..7ca86b3ccfa 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,4 +1,5 @@ -use litellm_core::error::Error; +use litellm_core::transport::Error as TransportError; +use litellm_core::{Error, audio_transcription, chat_completions, messages, ocr, responses}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; @@ -16,43 +17,99 @@ pyo3::create_exception!( "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." ); -pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { - match err { - Error::Auth(message) => PyValueError::new_err(message), - Error::InvalidProvider(_) - | Error::InvalidRequest(_) - | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::MissingDocumentUrl => PyValueError::new_err(err.to_string()), - other => PyRuntimeError::new_err(other.to_string()), +fn auth_is_value_error(error: &litellm_auth::Error) -> bool { + !matches!(error, litellm_auth::Error::MissingApiKey { .. }) +} + +pub(crate) fn messages_error_to_pyerr(error: messages::Error) -> PyErr { + core_error_to_pyerr(error.into()) +} + +pub(crate) fn audio_transcription_error_to_pyerr(error: audio_transcription::Error) -> PyErr { + core_error_to_pyerr(error.into()) +} + +pub(crate) fn responses_error_to_pyerr(error: responses::Error) -> PyErr { + core_error_to_pyerr(error.into()) +} + +pub(crate) fn core_error_to_pyerr(error: Error) -> PyErr { + let value_error = match &error { + Error::Ocr(error) => matches!( + error, + ocr::Error::Auth(_) + | ocr::Error::InvalidProvider(_) + | ocr::Error::InvalidRequest(_) + | ocr::Error::InvalidType { .. } + | ocr::Error::MissingField(_) + | ocr::Error::MissingDocumentUrl + ), + Error::Messages(error) => match error { + messages::Error::Auth(source) => auth_is_value_error(source), + messages::Error::InvalidProvider(_) + | messages::Error::InvalidRequest(_) + | messages::Error::Headers(_) => true, + _ => false, + }, + Error::AudioTranscription(error) => match error { + audio_transcription::Error::Auth(source) => auth_is_value_error(source), + audio_transcription::Error::InvalidProvider(_) + | audio_transcription::Error::InvalidRequest(_) + | audio_transcription::Error::Headers(_) + | audio_transcription::Error::InvalidType { .. } + | audio_transcription::Error::MissingField(_) + | audio_transcription::Error::Aws(_) => true, + _ => false, + }, + Error::ChatCompletions(error) => match error { + chat_completions::Error::Auth(source) => auth_is_value_error(source), + chat_completions::Error::InvalidProvider(_) + | chat_completions::Error::InvalidRequest(_) + | chat_completions::Error::Headers(_) + | chat_completions::Error::InvalidType { .. } + | chat_completions::Error::MissingField(_) + | chat_completions::Error::Aws(_) => true, + _ => false, + }, + Error::Responses(error) => match error { + responses::Error::Auth(source) => auth_is_value_error(source), + responses::Error::InvalidProvider(_) + | responses::Error::InvalidRequest(_) + | responses::Error::Headers(_) => true, + _ => false, + }, + }; + if value_error { + PyValueError::new_err(error.to_string()) + } else { + PyRuntimeError::new_err(error.to_string()) } } -/// Map a core error for a route whose host keeps a Python implementation. +/// Map a route error for a route whose host keeps a Python implementation. /// /// The distinction the host needs is whether the provider was already called. /// Everything raised before the request goes out is safe for the host to retry /// on its own path; anything after it is not, because the provider has already /// done the work and billed for it. -pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { - match err { +pub(crate) fn chat_completions_error_to_pyerr(error: chat_completions::Error) -> PyErr { + use chat_completions::Error; + match error { Error::Unsupported(_) | Error::Auth(_) + | Error::Aws(_) | Error::InvalidProvider(_) | Error::InvalidRequest(_) | Error::InvalidType { .. } | Error::MissingField(_) - | Error::MissingDocumentUrl - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureDocumentIntelligenceCredentials - | Error::MissingReductoApiKey - | Error::Routing(_) - // Nothing reached the provider, so serving it on Python cannot double - // bill and is the only way the caller gets an answer at all. - | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - Error::Http { status, body } => RustUpstreamError::new_err((status, body)), - Error::Network(message) | Error::InvalidResponse(message) => { + | Error::Headers(_) + | Error::Transport(TransportError::Connect(_)) => { + RustBridgeDeclined::new_err(error.to_string()) + } + Error::Transport(TransportError::Http { status, body }) => { + RustUpstreamError::new_err((status, body)) + } + Error::Transport(TransportError::Network(message)) | Error::InvalidResponse(message) => { RustUpstreamError::new_err((0u16, message)) } } @@ -63,3 +120,55 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add("RustBridgeDeclined", py.get_type::())?; module.add("RustUpstreamError", py.get_type::()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transport_status_and_dispatch_certainty_survive_python_mapping() { + Python::initialize(); + Python::attach(|py| { + let connect = chat_completions_error_to_pyerr( + TransportError::Connect("unreachable".into()).into(), + ); + assert!(connect.is_instance_of::(py)); + let network = + chat_completions_error_to_pyerr(TransportError::Network("timed out".into()).into()); + assert!(network.is_instance_of::(py)); + let upstream = chat_completions_error_to_pyerr( + TransportError::Http { + status: 429, + body: "slow down".into(), + } + .into(), + ); + assert_eq!( + upstream + .value(py) + .getattr("args") + .unwrap() + .extract::<(u16, String)>() + .unwrap(), + (429, "slow down".into()) + ); + }); + } + + #[test] + fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() { + Python::initialize(); + Python::attach(|py| { + let missing = messages_error_to_pyerr(messages::Error::Auth( + litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + }, + )); + assert!(missing.is_instance_of::(py)); + let invalid = + messages_error_to_pyerr(messages::Error::Auth(litellm_auth::Error::InvalidHeader)); + assert!(invalid.is_instance_of::(py)); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/execution.rs b/litellm-rust/crates/python-bridge/src/execution.rs index d8dda10068d..ffc4c186980 100644 --- a/litellm-rust/crates/python-bridge/src/execution.rs +++ b/litellm-rust/crates/python-bridge/src/execution.rs @@ -165,7 +165,7 @@ mod tests { use std::thread; use std::time::Instant; - use litellm_core::error::Error; + use litellm_core::messages::Error; use pyo3::panic::PanicException; use pyo3::types::{PyDict, PyModule}; use rstest::{fixture, rstest}; diff --git a/litellm-rust/crates/python-bridge/src/function_trace.rs b/litellm-rust/crates/python-bridge/src/function_trace.rs deleted file mode 100644 index bc3c962f7a3..00000000000 --- a/litellm-rust/crates/python-bridge/src/function_trace.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::fmt::Display; -use std::future::Future; - -use litellm_core::observability::{FunctionTrace, FunctionTraceEvent}; -use serde::Serialize; -use tracing::instrument::WithSubscriber; - -#[derive(Serialize)] -pub(crate) struct TracedResponse { - #[serde(skip_serializing_if = "Option::is_none")] - response: Option, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, - trace: Vec, -} - -pub(crate) async fn capture( - future: impl Future>, -) -> Result, E> -where - E: Display, -{ - let trace = FunctionTrace::default(); - let result = future.with_subscriber(trace.dispatcher()).await; - let events = trace.events(); - Ok(match result { - Ok(response) => TracedResponse { - response: Some(response), - error: None, - trace: events, - }, - Err(error) => TracedResponse { - response: None, - error: Some(error.to_string()), - trace: events, - }, - }) -} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12bc57a8931..0306990fd4d 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -3,8 +3,6 @@ mod constants; mod diagnostics; mod errors; mod execution; -#[cfg(feature = "trace-parity")] -mod function_trace; mod lifecycle; mod marshal; mod routes; @@ -15,7 +13,7 @@ use pyo3::prelude::*; use pyo3::types::PyAny; use serde_json::Value; -use crate::errors::core_error_to_pyerr; +use crate::errors::responses_error_to_pyerr; use crate::marshal::{marshal_headers, optional_timeout}; #[pyclass] @@ -39,7 +37,7 @@ impl ResponsesWebSocketConnection { pyo3_async_runtimes::tokio::future_into_py(py, async move { let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout) .await - .map_err(core_error_to_pyerr)?; + .map_err(responses_error_to_pyerr)?; Ok(ResponsesWebSocketConnection { inner }) }) } @@ -47,21 +45,24 @@ impl ResponsesWebSocketConnection { fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult> { let inner = self.inner.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - inner.send_text(text).await.map_err(core_error_to_pyerr) + inner + .send_text(text) + .await + .map_err(responses_error_to_pyerr) }) } fn recv_text<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - inner.recv_text().await.map_err(core_error_to_pyerr) + inner.recv_text().await.map_err(responses_error_to_pyerr) }) } fn close<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); pyo3_async_runtimes::tokio::future_into_py(py, async move { - inner.close().await.map_err(core_error_to_pyerr) + inner.close().await.map_err(responses_error_to_pyerr) }) } } @@ -124,39 +125,6 @@ mod tests { .filter(|name| !name.starts_with('_')) .collect(); assert_eq!(public_names, expected); - - #[cfg(not(feature = "trace-parity"))] - assert!(!module.hasattr("_trace").expect("module lookup should work")); - - #[cfg(feature = "trace-parity")] - { - let trace = module - .getattr("_trace") - .expect("trace build should expose its diagnostic namespace"); - let trace_names: Vec = trace - .cast::() - .expect("trace namespace should be a module") - .dict() - .keys() - .extract::>() - .expect("trace names should be strings") - .into_iter() - .filter(|name| !name.starts_with("__")) - .collect(); - assert_eq!( - trace_names, - [ - "ocr", - "aocr", - "transcription", - "atranscription", - "messages", - "amessages", - "chat_completions", - "achat_completions", - ] - ); - } }); } diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs b/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs index 014564ae89d..c4b8d8eaae0 100644 --- a/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs +++ b/litellm-rust/crates/python-bridge/src/lifecycle/mod.rs @@ -35,7 +35,8 @@ pub(crate) trait PythonRoute: Send + Sync { fn state_mut(&mut self) -> &mut PythonCallState; fn classify(operation: &::Operation) -> OperationClass; fn lifecycle_result() -> ::Result; - fn map_error(error: litellm_core::Error) -> PyErr; + fn map_error(error: ::Error) -> PyErr; + fn host_error(message: String) -> ::Error; fn invoke( &mut self, py: Python<'_>, @@ -46,8 +47,10 @@ pub(crate) trait PythonRoute: Send + Sync { } type NativeStep = NativeCallStep<::Operation, ::Complete>; -type NativeResult = Result, litellm_core::Error>; +type NativeResult = Result, ::Error>; type HostResumeStep = HostStep::Call>, Py>; +type NativeResume = + Option::Result, HostFailure<::Error>>>; struct NativeCallState { call: C, @@ -102,7 +105,7 @@ impl PythonLifecycle { fn resume_core( &mut self, py: Python<'_>, - result: Option::Result, HostFailure>>, + result: NativeResume, ) -> PyResult> { let call = Arc::clone(self.call.as_ref().ok_or_else(missing_state)?); let future = async move { @@ -154,8 +157,8 @@ impl PythonLifecycle { py: Python<'_>, error: PyErr, phase: Option, - ) -> HostFailure { - let native = litellm_core::Error::InvalidRequest(error.to_string()); + ) -> HostFailure<::Error> { + let native = R::host_error(error.to_string()); let cancelled = !error.is_instance_of::(py); let failure = if !cancelled { HostFailure::Error(native) @@ -596,6 +599,34 @@ mod tests { static PYTHON_GLOBALS: Mutex<()> = Mutex::new(()); + fn install_lifecycle_module(py: Python<'_>) -> Bound<'_, PyModule> { + py.run( + pyo3::ffi::c_str!( + r#" +import sys +import types + +sys.modules.setdefault('litellm', types.ModuleType('litellm')) +sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge')) +"# + ), + None, + None, + ) + .unwrap(); + let source = std::ffi::CString::new(include_str!( + "../../../../../litellm/rust_bridge/lifecycle.py" + )) + .unwrap(); + PyModule::from_code( + py, + &source, + pyo3::ffi::c_str!("lifecycle.py"), + pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), + ) + .unwrap() + } + fn install_logging_worker(py: Python<'_>, worker: &Bound<'_, PyAny>) -> PyResult<()> { py.import("litellm.litellm_core_utils.logging_worker")? .setattr("GLOBAL_LOGGING_WORKER", worker) @@ -667,6 +698,7 @@ mod tests { struct SyntheticCall(bool); impl NativeCall for SyntheticCall { + type Error = litellm_core::messages::Error; type Operation = (); type Result = (); type Complete = (); @@ -674,7 +706,7 @@ mod tests { fn resume( &mut self, result: Option, - ) -> HostCallFuture<'_, Self::Operation, Self::Complete> { + ) -> HostCallFuture<'_, Self::Operation, Self::Complete, Self::Error> { Box::pin(async move { match (self.0, result) { (false, None) => { @@ -682,7 +714,7 @@ mod tests { Ok(NativeCallStep::Host(())) } (true, Some(())) => Ok(NativeCallStep::Complete(())), - _ => Err(litellm_core::Error::InvalidRequest( + _ => Err(litellm_core::messages::Error::InvalidRequest( "invalid synthetic lifecycle state".into(), )), } @@ -691,8 +723,8 @@ mod tests { fn interrupt( &mut self, - _: HostFailure, - ) -> HostCallFuture<'_, Self::Operation, Self::Complete> { + _: HostFailure, + ) -> HostCallFuture<'_, Self::Operation, Self::Complete, Self::Error> { Box::pin(async { Ok(NativeCallStep::Complete(())) }) } } @@ -716,8 +748,12 @@ mod tests { fn lifecycle_result() {} - fn map_error(error: litellm_core::Error) -> PyErr { - crate::errors::core_error_to_pyerr(error) + fn map_error(error: litellm_core::messages::Error) -> PyErr { + crate::errors::messages_error_to_pyerr(error) + } + + fn host_error(message: String) -> litellm_core::messages::Error { + litellm_core::messages::Error::InvalidRequest(message) } fn invoke(&mut self, py: Python<'_>, _: ()) -> PyResult<()> { @@ -765,17 +801,7 @@ mod tests { .unwrap_or_else(|error| error.into_inner()); Python::initialize(); Python::attach(|py| { - let source = std::ffi::CString::new(include_str!( - "../../../../../litellm/rust_bridge/lifecycle.py" - )) - .unwrap(); - PyModule::from_code( - py, - &source, - pyo3::ffi::c_str!("lifecycle.py"), - pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), - ) - .unwrap(); + install_lifecycle_module(py); let route = SyntheticRoute( PythonCallState::new( py, @@ -811,17 +837,7 @@ mod tests { Python::initialize(); Python::attach(|py| { py.import("asyncio").unwrap(); - let source = std::ffi::CString::new(include_str!( - "../../../../../litellm/rust_bridge/lifecycle.py" - )) - .unwrap(); - let module = PyModule::from_code( - py, - &source, - pyo3::ffi::c_str!("lifecycle.py"), - pyo3::ffi::c_str!("litellm.rust_bridge.lifecycle"), - ) - .unwrap(); + let module = install_lifecycle_module(py); let locals = PyDict::new(py); locals .set_item("drive", module.getattr("drive").unwrap()) diff --git a/litellm-rust/crates/python-bridge/src/lifecycle/preparation.rs b/litellm-rust/crates/python-bridge/src/lifecycle/preparation.rs index ba4a8bb3739..e95f642e6ea 100644 --- a/litellm-rust/crates/python-bridge/src/lifecycle/preparation.rs +++ b/litellm-rust/crates/python-bridge/src/lifecycle/preparation.rs @@ -1,4 +1,4 @@ -use litellm_core::auth::{credential_default_fields, credential_index}; +use litellm_auth::{credential_default_fields, credential_index}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 5f7633a64a0..7f00298905f 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -6,7 +6,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; use serde_json::{Map, Value}; -use litellm_core::auth::InputSource; +use litellm_auth::InputSource; use litellm_python_interop::from_py_preserving_errors as from_py; pub(crate) struct RouteOptions { @@ -190,6 +190,7 @@ mod tests { #[test] fn required_shapes_preserve_nested_values_and_existing_errors() { + Python::initialize(); let nested = json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]); assert_eq!( Value::Array(required_array("messages", nested.clone()).unwrap()), diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs index f2997ee278c..68b701802a9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs @@ -5,8 +5,3 @@ use pyo3::prelude::*; pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { value::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription/value.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription/value.rs index af60515b0e2..5ecca63fcb6 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription/value.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription/value.rs @@ -1,4 +1,4 @@ -use litellm_core::Error; +use litellm_core::audio_transcription::Error; use std::future::Future; use litellm_core::audio_transcription::{ @@ -7,7 +7,7 @@ use litellm_core::audio_transcription::{ use pyo3::prelude::*; use serde_json::Value; -use crate::errors::core_error_to_pyerr; +use crate::errors::audio_transcription_error_to_pyerr; use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; fn prepare_transcription( @@ -67,5 +67,5 @@ bridge_route! { timeout_seconds: Option, }, prepare = prepare_transcription, - errors = core_error_to_pyerr, + errors = audio_transcription_error_to_pyerr, } diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs index f2997ee278c..68b701802a9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs @@ -5,8 +5,3 @@ use pyo3::prelude::*; pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { value::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/value.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/value.rs index e67bfa89cc7..09f2ada51a5 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/value.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/value.rs @@ -1,4 +1,4 @@ -use litellm_core::Error; +use litellm_core::chat_completions::Error; use std::future::Future; use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index 571042062f5..4c8d98ebe62 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -58,70 +58,6 @@ macro_rules! bridge_route { Ok(()) } - #[cfg(feature = "trace-parity")] - mod trace { - use pyo3::prelude::*; - use super::{$inputs, $map_error, $prepare}; - - #[pyfunction] - #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] - #[allow(clippy::too_many_arguments)] - fn $sync_name( - py: pyo3::Python<'_>, - $($(#[$required_attr])* $required_name: $required_type,)* - $($(#[$optional_attr])* $optional_name: $optional_type,)* - ) -> pyo3::PyResult> { - let future = $prepare($inputs { - $($required_name,)* - $($optional_name),* - })?; - $crate::execution::run_sync( - py, - $crate::function_trace::capture(future), - $map_error, - ) - } - - #[pyfunction] - #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] - #[allow(clippy::too_many_arguments)] - fn $async_name( - py: pyo3::Python<'_>, - $($(#[$required_attr])* $required_name: $required_type,)* - $($(#[$optional_attr])* $optional_name: $optional_type,)* - ) -> pyo3::PyResult> { - let future = $prepare($inputs { - $($required_name,)* - $($optional_name),* - })?; - $crate::execution::run_async( - py, - $crate::function_trace::capture(future), - $map_error, - ) - } - - pub(super) fn register( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - $crate::routes::definition::add_function( - module, - pyo3::wrap_pyfunction!($sync_name, module)?, - )?; - $crate::routes::definition::add_function( - module, - pyo3::wrap_pyfunction!($async_name, module)?, - )?; - Ok(()) - } - } - - #[cfg(feature = "trace-parity")] - pub(super) fn register_trace( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - trace::register(module) - } }; } @@ -143,7 +79,7 @@ mod tests { use std::ffi::CString; use std::sync::atomic::{AtomicBool, Ordering}; - use litellm_core::error::Error; + use litellm_core::messages::Error; use pyo3::exceptions::PyLookupError; use pyo3::types::{PyDict, PyList}; @@ -188,7 +124,6 @@ mod tests { Ok(execute_echo(inputs, drop_guard)) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] async fn execute_echo( inputs: EchoInputs, drop_guard: Option, @@ -548,33 +483,6 @@ asyncio.run(exercise()) }); } - #[cfg(feature = "trace-parity")] - #[test] - fn diagnostic_route_returns_the_response_and_filtered_trace() { - Python::initialize(); - Python::attach(|py| { - let module = PyModule::new(py, "synthetic").expect("module should be created"); - synthetic::register_trace(&module).expect("trace routes should register"); - let locals = PyDict::new(py); - locals - .set_item("routes", &module) - .expect("module should enter Python locals"); - let code = CString::new( - r#" -result = routes.echo("traced") -assert result["response"] == "traced", result -assert [event["function"] for event in result["trace"]] == ["execute_echo"], result -failure = routes.echo("error") -assert failure["error"] == "invalid request: synthetic error", failure -assert [event["function"] for event in failure["trace"]] == ["execute_echo"], failure -"#, - ) - .expect("Python source should not contain null bytes"); - py.run(&code, Some(&locals), Some(&locals)) - .expect("diagnostic route should return its response and trace"); - }); - } - #[test] fn route_registration_rejects_duplicate_python_names() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index f2997ee278c..68b701802a9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -5,8 +5,3 @@ use pyo3::prelude::*; pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { value::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/value.rs b/litellm-rust/crates/python-bridge/src/routes/messages/value.rs index b741e54f0ca..f5eb80d765c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/value.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/value.rs @@ -1,11 +1,11 @@ -use litellm_core::Error; +use litellm_core::messages::Error; use litellm_core::messages::messages as run_messages; use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; use pyo3::prelude::*; use serde_json::Value; use std::future::Future; -use crate::errors::core_error_to_pyerr; +use crate::errors::messages_error_to_pyerr; use crate::marshal::{RouteOptions, RouteOptionsInputs, required_object}; fn prepare_messages( @@ -61,5 +61,5 @@ bridge_route! { timeout_seconds: Option, }, prepare = prepare_messages, - errors = core_error_to_pyerr, + errors = messages_error_to_pyerr, } diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 97c39a5d6b3..4e2530a94f8 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -13,15 +13,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audio_transcription::register(module)?; messages::register(module)?; chat_completions::register(module)?; - - #[cfg(feature = "trace-parity")] - { - let trace = PyModule::new(module.py(), "_trace")?; - ocr::register_trace(&trace)?; - audio_transcription::register_trace(&trace)?; - messages::register_trace(&trace)?; - chat_completions::register_trace(&trace)?; - module.add_submodule(&trace)?; - } Ok(()) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs index 66bdfb7583e..e4ce813d297 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs @@ -1,4 +1,4 @@ -use litellm_core::error::Error; +use litellm_core::ocr::Error; use pyo3::prelude::*; use crate::errors::{RustUpstreamError, core_error_to_pyerr}; @@ -7,7 +7,7 @@ pub(super) fn to_pyerr(error: Error) -> PyErr { let status = error.http_status_code(); let mapped = match error { Error::Http { status, body } => RustUpstreamError::new_err((status, body)), - other => core_error_to_pyerr(other), + other => core_error_to_pyerr(other.into()), }; attach_status(mapped, status) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/lifecycle.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/lifecycle.rs index 12d902a3544..32794936899 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/lifecycle.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/lifecycle.rs @@ -1,7 +1,7 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; -use litellm_core::auth::ResolvedCredential; +use litellm_auth::ResolvedCredential; use litellm_core::ocr::hooks::{OcrDuringCallRequest, OcrPostCallRequest, OcrPreCallRequest}; use litellm_core::ocr::{OcrAdmission, OcrCall, OcrClient, OcrHostOperation, OcrHostResult}; use litellm_python_interop::{ @@ -179,10 +179,14 @@ impl PythonRoute for PythonOcrHost { OcrHostResult::Lifecycle(Ok(())) } - fn map_error(error: litellm_core::Error) -> PyErr { + fn map_error(error: litellm_core::ocr::Error) -> PyErr { ocr_error_to_pyerr(error) } + fn host_error(message: String) -> litellm_core::ocr::Error { + litellm_core::ocr::Error::InvalidRequest(message) + } + fn invoke(&mut self, py: Python<'_>, operation: OcrHostOperation) -> PyResult { Ok(match operation { OcrHostOperation::ProjectRequest => { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 10fa40b65ea..f17bf249b7f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -12,8 +12,3 @@ pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { document::register(module)?; lifecycle::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 8b6a1b02e19..8d8d5f8c518 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -177,7 +177,7 @@ pub(super) fn admitted_call(outcome: NativeOutcome) -> PyResult 0); - assert!(started.elapsed().as_secs() < 5, "{:?}", started.elapsed()); + let mut time = |len: usize| { + let piece = vec![b' '; len]; + let started = std::time::Instant::now(); + assert!(ranks.count_piece(&piece, &mut scratch) > 0); + started.elapsed() + }; + let small = (0..3).map(|_| time(1 << 14)).min().unwrap(); + let large = time(1 << 18); + assert!( + large < small * 64, + "{small:?} for 2^14 bytes, {large:?} for 2^18" + ); } #[test] diff --git a/litellm/__init__.py b/litellm/__init__.py index 3668e6efb0c..dde94d68d5f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -525,6 +525,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +http2: bool = False network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### diff --git a/litellm/constants.py b/litellm/constants.py index 745a4d9294e..ce5b65080ee 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -364,6 +364,8 @@ GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int( os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60) ) BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000 +CONTENT_FILTER_STREAMING_HOLDBACK_CHARS: Final = 50 +CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS: Final = 512 DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES: Final = 500_000 PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096 PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8 diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index d4e7fcb577e..9456817a205 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypedDict, cast import litellm @@ -20,7 +21,9 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( OTELSemconvCategory, parse_semconv_opt_in, ) +from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.db_endpoint import db_span_attributes +from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import Metric from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -288,6 +291,7 @@ class OpenTelemetryConfig: # under ``litellm.team.metadata``. Empty by default so none of a team's # metadata leaves the process until explicitly allowlisted. baggage_team_metadata_keys: list[str] = field(default_factory=list) + baggage_metadata_keys: list[str] = field(default_factory=list) # Prometheus-style include/exclude control over which attributes are stamped # on emitted metrics, to cap metric cardinality. attributes: OTELMetricAttributeFilter | None = None @@ -314,6 +318,9 @@ class OpenTelemetryConfig: self.baggage_team_metadata_keys = _normalize_team_metadata_keys( self.baggage_team_metadata_keys ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS")) + self.baggage_metadata_keys = _normalize_team_metadata_keys( + self.baggage_metadata_keys + ) or _normalize_team_metadata_keys(os.getenv("LITELLM_OTEL_BAGGAGE_METADATA_KEYS")) @classmethod def from_env(cls): @@ -366,11 +373,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): **kwargs, ): team_metadata_keys_override: Final = kwargs.pop("baggage_team_metadata_keys", None) + metadata_keys_override: Final = kwargs.pop("baggage_metadata_keys", None) metric_attributes_override: Final = kwargs.pop("attributes", None) if config is None: config = OpenTelemetryConfig.from_env() if team_metadata_keys_override is not None: config.baggage_team_metadata_keys = _normalize_team_metadata_keys(team_metadata_keys_override) + if metadata_keys_override is not None: + config.baggage_metadata_keys = _normalize_team_metadata_keys(metadata_keys_override) if metric_attributes_override is not None: config.attributes = _build_metric_attribute_filter(metric_attributes_override) @@ -1542,6 +1552,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if team_metadata: self.safe_set_attribute(span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata) + if self.config.baggage_metadata_keys: + flat_metadata: Final = MappingProxyType(dict(flatten_metadata(metadata))) + for key, value in promoted_metadata(flat_metadata, tuple(self.config.baggage_metadata_keys)).items(): + self.safe_set_attribute(span=span, key=key, value=value) + model_group: Final = standard_logging_payload.get("model_group") if model_group: self.safe_set_attribute(span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 9ac748b231c..285a5c3aa97 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -33,6 +33,7 @@ from litellm.integrations.otel.model.metadata import ( LLMCallEvent, RequestIdentity, auth_metadata, + metadata_from_request_data, model_from_request_data, ) from litellm.integrations.otel.model.payloads import ( @@ -679,7 +680,12 @@ class OpenTelemetryV2(CustomLogger): # / errors are the FastAPI instrumentor's job, so we don't touch it here. # ====================================================================== # - def seed_request_identity(self, user_api_key_dict: object, model: str | None = None) -> None: + def seed_request_identity( + self, + user_api_key_dict: object, + model: str | None = None, + request_metadata: Mapping[str, object] | None = None, + ) -> None: """Attach request-identity Baggage to the current context + server span. Seeding identity into Baggage makes **every** span emitted afterwards for @@ -691,7 +697,7 @@ class OpenTelemetryV2(CustomLogger): isn't determined yet, which is correct. """ try: - identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict) + identity: Final = RequestIdentity.from_user_api_key_auth(user_api_key_dict, request_metadata) bag: Final = promoted_baggage( identity, model, @@ -743,6 +749,7 @@ class OpenTelemetryV2(CustomLogger): self.seed_request_identity( user_api_key_dict, model=model_from_request_data(data), + request_metadata=metadata_from_request_data(data), ) return data diff --git a/litellm/integrations/otel/model/baggage.py b/litellm/integrations/otel/model/baggage.py index 2be9bb36def..131848e1380 100644 --- a/litellm/integrations/otel/model/baggage.py +++ b/litellm/integrations/otel/model/baggage.py @@ -15,9 +15,10 @@ never promoted whole. import json from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final -from litellm.integrations.otel.model.metadata import RequestIdentity +from litellm.integrations.otel.model.metadata import REQUESTER_METADATA_PATH, RequestIdentity from litellm.integrations.otel.model.semconv import GenAI, LiteLLM # Attribute key -> value extractor over (identity, request_model, @@ -79,17 +80,23 @@ def promoted_baggage( ``team_metadata_keys`` selects sub-keys of the team's metadata to promote under ``litellm.team.metadata``. Empty values are dropped. """ - out: Final[dict[str, str]] = {} - for key, extract in _PROMOTABLE.items(): - if key in promoted_keys: - value = extract(identity, request_model, team_metadata_keys) - if value: - out[key] = value - for meta_key in metadata_keys: - value = identity.metadata.get(meta_key) - if value: - out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value - return out + identity_values: Final = { + key: value + for key, extract in _PROMOTABLE.items() + if key in promoted_keys and (value := extract(identity, request_model, team_metadata_keys)) + } + return {**identity_values, **promoted_metadata(identity.metadata, metadata_keys)} + + +def promoted_metadata(metadata: Mapping[str, str], metadata_keys: tuple[str, ...]) -> Mapping[str, str]: + """Allowlisted entries of a flattened metadata mapping under ``litellm.metadata.*``.""" + return MappingProxyType( + { + f"{LiteLLM.METADATA_PREFIX}{meta_key.removeprefix(REQUESTER_METADATA_PATH)}": value + for meta_key in metadata_keys + if (value := metadata.get(meta_key)) + } + ) def _filtered_team_metadata_json( diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index bd542ddc20c..5bda66ed618 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -210,7 +210,10 @@ class OpenTelemetryV2Config(BaseSettings): validation_alias=AliasChoices("baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"), description=( "Metadata sub-keys promoted under the ``litellm.metadata.*`` " - "namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " + "namespace. A dotted path such as ``requester_metadata.trace_id`` " + "reads the caller's nested ``metadata.trace_id`` and is promoted as " + "``litellm.metadata.trace_id``; other dotted keys keep their full path. " + "Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` " "env var (comma-separated) or " "``callback_settings.otel.baggage_metadata_keys`` in config.yaml." ), diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index cc81b689708..5f90e70e119 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -49,6 +49,8 @@ if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload LANGFUSE_TRACE_NAME_HEADER: Final = "langfuse_trace_name" +REQUESTER_METADATA_KEY: Final = "requester_metadata" +REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}." @dataclass(frozen=True) @@ -78,7 +80,7 @@ class RequestIdentity: model, not just the user-facing one. """ raw_meta: Final = cast(Mapping[str, object], payload.get("metadata") or {}) - metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))} + metadata: Final = MappingProxyType(dict(flatten_metadata(raw_meta))) return cls( call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")), # StandardLoggingMetadata's canonical key is ``user_api_key_team_id``; @@ -95,7 +97,9 @@ class RequestIdentity: ) @classmethod - def from_user_api_key_auth(cls, auth: object) -> RequestIdentity: + def from_user_api_key_auth( + cls, auth: object, request_metadata: Mapping[str, object] | None = None + ) -> RequestIdentity: """Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module free of a proxy import). @@ -103,11 +107,13 @@ class RequestIdentity: guardrail, or service span is created — so the whole request's spans inherit identity, not just the LLM-call span. Metadata sub-keys use the ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS`` - promotes. + promotes; ``request_metadata`` (the caller's ``requester_metadata`` + snapshot) is flattened to dotted keys so ``requester_metadata.`` + resolves too. """ get: Final = lambda name: getattr(auth, name, None) # noqa: E731 - metadata: Final = { - meta_key: str(value) + auth_meta: Final = tuple( + (meta_key, str(value)) for meta_key, attr in ( ("user_api_key_user_id", "user_id"), ("user_api_key_org_id", "org_id"), @@ -115,7 +121,9 @@ class RequestIdentity: ("user_api_key_end_user_id", "end_user_id"), ) if (value := get(attr)) - } + ) + request_meta: Final = flatten_metadata(request_metadata) if request_metadata is not None else () + metadata: Final = MappingProxyType(dict((*request_meta, *auth_meta))) return cls( team_id=as_str(get("team_id")), team_alias=as_str(get("team_alias")), @@ -351,6 +359,35 @@ def model_from_request_data(data: object) -> str | None: return None +def metadata_from_request_data(data: object) -> Mapping[str, object] | None: + """The caller's ``requester_metadata`` snapshot from a pre-call ``data`` dict, keyed under its wrapper. + + The proxy stores it under ``metadata`` or ``litellm_metadata`` depending on the route; + the proxy-owned siblings (``user_api_key_*``, ``requester_ip_address``) are not read. + """ + top: Final = _as_str_mapping(data) + if top is None: + return None + snapshots: Final = tuple( + snapshot + for name in ("metadata", "litellm_metadata") + if (nested := _as_str_mapping(top.get(name))) is not None + and (snapshot := _as_str_mapping(nested.get(REQUESTER_METADATA_KEY))) is not None + ) + return MappingProxyType({REQUESTER_METADATA_KEY: snapshots[0]}) if snapshots else None + + +def flatten_metadata(raw: Mapping[str, object]) -> Iterator[tuple[str, str]]: + """Scalar leaves of a nested metadata mapping, keyed by their dotted path.""" + stack: Final = list(tuple(raw.items())[::-1]) # mutable-ok: iterative worklist keeps the walk off the call stack + while stack: + key, value = stack.pop() + if (nested := _as_str_mapping(value)) is not None: + stack.extend(tuple((f"{key}.{sub_key}", sub_value) for sub_key, sub_value in nested.items())[::-1]) + elif isinstance(value, (str, bool, int, float)): + yield key, str(value) + + def resolve_provider_model(payload: StandardLoggingPayload) -> str | None: """The model litellm dispatched to the provider, from the payload. diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 5d3ae444b42..4dd0deeb62b 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -1167,7 +1167,9 @@ class ModelResponseIterator: # (matches OpenAI behavior and non-streaming Anthropic implementation) if self.converted_response_format_tool: finish_reason = "stop" - usage: Final = self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) + usage: Final = ( + self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None + ) container: Final = message_delta["delta"].get("container") return finish_reason, usage, container diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index e7179aad25b..4486eb0985a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -376,10 +376,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): usage_dict: UsageDelta = LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage_delta( chunk.usage ) - merged_chunk["usage"] = usage_dict if self.applied_edits and "context_management" not in merged_chunk: merged_chunk["context_management"] = ContextManagementResponse(applied_edits=list(self.applied_edits)) - return self._augment_message_delta_usage(merged_chunk) + return self._augment_message_delta_usage({**merged_chunk, "usage": usage_dict}) def _handle_choiceless_chunk(self, chunk: "ModelResponseStream") -> bool: """Consume an OpenAI-compatible chunk that carries no ``choices``. @@ -448,8 +447,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): } iterations.append(message_iteration) augmented_usage["iterations"] = iterations - augmented["usage"] = augmented_usage - return augmented + return {**augmented, "usage": augmented_usage} def _next_compaction_event(self) -> dict[str, object] | None: """Return the next compaction content-block SSE event, or ``None``. diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index 27c90c9d71e..ba8ce7e5625 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from copy import deepcopy from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -14,6 +15,7 @@ from litellm.types.integrations.rag.bedrock_knowledgebase import ( BedrockKBResponse, BedrockKBRetrievalConfiguration, BedrockKBRetrievalQuery, + BedrockKBUserContext, ) from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -242,10 +244,29 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): retrieval_config.setdefault("vectorSearchConfiguration", {})["filter"] = filters if retrieval_config: request_body["retrievalConfiguration"] = cast(BedrockKBRetrievalConfiguration, retrieval_config) + user_context: Final = self._user_context(extra_body=extra_body, litellm_params=litellm_params) + if user_context is not None: + request_body["userContext"] = user_context litellm_logging_obj.model_call_details["query"] = query return url, request_body + @staticmethod + def _user_context( + extra_body: Mapping[str, object] | None, litellm_params: Mapping[str, object] + ) -> BedrockKBUserContext | None: + sources: Final = tuple(source for source in (extra_body, litellm_params) if isinstance(source, Mapping)) + found: Final = next( + ( + source[key] + for source in sources + for key in ("userContext", "user_context") + if source.get(key) is not None + ), + None, + ) + return None if found is None else cast(BedrockKBUserContext, found) + def sign_request( self, headers: dict, diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index f4883b57fbc..05dff0cb9d8 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -7,6 +7,7 @@ import ssl import sys import threading import time +import weakref from collections.abc import AsyncIterable, Callable, Iterable, Mapping from http.cookiejar import CookieJar, DefaultCookiePolicy from io import BytesIO @@ -74,6 +75,12 @@ _IPV4_LOCAL_ADDRESS: Final = "0.0.0.0" _HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport) +def http2_enabled() -> bool: + from litellm.secret_managers.main import str_to_bool + + return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True + + def _environment_proxy_mounts( build_proxy_transport: Callable[[str], _HttpxTransportT], ) -> Mapping[str, _HttpxTransportT | None]: @@ -179,6 +186,33 @@ def _handler_may_close_client(client_refcount: int, owns_client: bool) -> bool: return owns_client and client_refcount <= _CLIENT_REFCOUNT_WHEN_HANDLER_IS_SOLE_REFERRER +def _drop_streaming_anchor(_handler: object) -> None: + """Release a handler anchored to a streaming response. See ``_anchor_handler_to``. + + The work is the reference held until this point, so there is nothing to do here. + """ + + +def _anchor_handler_to(response: httpx.Response, handler: object) -> None: + """Keep the handler alive for as long as a streaming response can still read. + + A body still arriving reads through the handler's connection pool, and closing + the client tears that pool down. The refcount ``_handler_may_close_client`` + reads cannot see that body: the reference graph runs response -> stream -> + connection and stops there, so a client carrying one looks exactly like an + unreferenced client, and the finalizer closes it mid-body. + + ``weakref.finalize`` holds the handler in its own registry rather than on the + response, which matters twice. The handler stays out of the response's + reference cycle, so it is finalized by refcount once the anchor drops and can + still schedule an async close, instead of being finalized inside a cyclic + collection that reaps its aiohttp session in the same pass. And a handler + serving several streams collects only once every one of them is done, because + each anchor holds it separately. + """ + weakref.finalize(response, _drop_streaming_anchor, handler) + + def blocked_cookie_jar() -> CookieJar: """A jar that stores no response cookie and sends none, for httpx clients. @@ -638,6 +672,7 @@ class AsyncHTTPHandler: headers=default_headers, cookies=blocked_cookie_jar(), follow_redirects=True, + http2=http2_enabled(), ) async def close(self): @@ -771,6 +806,8 @@ class AsyncHTTPHandler: content=request_content, ) response: Final = await self.client.send(req, stream=stream) + if stream: + _anchor_handler_to(response, self) response.raise_for_status() return response except (httpx.RemoteProtocolError, httpx.ConnectError): @@ -975,6 +1012,8 @@ class AsyncHTTPHandler: content=request_content, ) response: Final = await self.client.send(req, stream=stream) + if stream: + _anchor_handler_to(response, self) response.raise_for_status() return response except (httpx.RemoteProtocolError, httpx.ConnectError): @@ -1157,6 +1196,10 @@ class AsyncHTTPHandler: from litellm.secret_managers.main import str_to_bool + if http2_enabled(): + verbose_logger.debug("LITELLM_HTTP2 enabled, using httpx transport (aiohttp has no HTTP/2 support)") + return False + ######################################################### # Check if user disabled aiohttp transport ######################################################## @@ -1287,7 +1330,7 @@ class AsyncHTTPHandler: - [Default] If force_ipv4 is False, it will return None """ if litellm.force_ipv4: - return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled()) else: return None @@ -1300,7 +1343,7 @@ class AsyncHTTPHandler: if not isinstance(transport, AsyncHTTPTransport): return None return _environment_proxy_mounts( - lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert) + lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled()) ) @@ -1342,6 +1385,7 @@ class HTTPHandler: headers=default_headers, cookies=blocked_cookie_jar(), follow_redirects=True, + http2=http2_enabled(), ) @property @@ -1439,6 +1483,8 @@ class HTTPHandler: content=request_content, ) response: Final = self.client.send(req, stream=stream) + if stream: + _anchor_handler_to(response, self) response.raise_for_status() return response except httpx.TimeoutException: @@ -1489,6 +1535,8 @@ class HTTPHandler: content=request_content, ) response: Final = self.client.send(req, stream=stream) + if stream: + _anchor_handler_to(response, self) response.raise_for_status() return response except httpx.TimeoutException: @@ -1539,6 +1587,8 @@ class HTTPHandler: content=request_content, ) response: Final = self.client.send(req, stream=stream) + if stream: + _anchor_handler_to(response, self) return response except httpx.TimeoutException: raise litellm.Timeout( @@ -1588,6 +1638,8 @@ class HTTPHandler: content=request_content, ) response: Final = self.client.send(req, stream=stream) + if stream: + _anchor_handler_to(response, self) response.raise_for_status() return response except httpx.TimeoutException: @@ -1616,7 +1668,7 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled()) else: return getattr(litellm, "sync_transport", None) @@ -1627,7 +1679,9 @@ class HTTPHandler: ) -> Mapping[str, HTTPTransport | None] | None: if not litellm.force_ipv4: return None - return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert)) + return _environment_proxy_mounts( + lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled()) + ) def get_async_httpx_client( diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 05160d83c12..b6c2b379d66 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -49,6 +49,17 @@ if TYPE_CHECKING: import tiktoken +def _map_reasoning_effort(value: object) -> object: + effort: Final[object] = cast(Mapping[str, object], value).get("effort") if isinstance(value, Mapping) else value + if effort is True: + return "medium" + if effort is False: + return "none" + if effort == "auto": + return None + return effort + + def _extract_fireworks_hidden_params(payload: dict) -> dict: """ Collect Fireworks-specific response fields (perf_metrics, prompt_token_ids, @@ -327,12 +338,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): elif param == "max_completion_tokens": optional_params["max_tokens"] = value elif param == "reasoning_effort": - if value is True: - optional_params["reasoning_effort"] = "medium" - elif value is False: - optional_params["reasoning_effort"] = "none" - elif value != "auto": - optional_params["reasoning_effort"] = value + effort = _map_reasoning_effort(value) + if effort is not None: + optional_params["reasoning_effort"] = effort elif param in supported_openai_params: if value is not None: optional_params[param] = value diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 2db6d78a218..cb6a5e4e96a 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -32,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, AsyncHTTPHandler, get_ssl_configuration, + http2_enabled, ) @@ -325,6 +326,7 @@ class BaseOpenAILLM: transport=transport, mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=None), follow_redirects=True, + http2=http2_enabled(), ) @staticmethod @@ -343,6 +345,7 @@ class BaseOpenAILLM: return httpx.Client( verify=ssl_config, follow_redirects=True, + http2=http2_enabled(), ) diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index 1f977a66186..2f638da49c8 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -49,7 +49,6 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): Inherits from OpenAIResponsesAPIConfig since XAI's Responses API is largely compatible with OpenAI's, with a few differences: - - Does not support the 'instructions' parameter - Requires code_interpreter tools to have 'container' field removed - Recommends store=false when sending images @@ -60,20 +59,6 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.XAI - def get_supported_openai_params(self, model: str) -> list: - """ - Get supported parameters for XAI Responses API. - - XAI supports most OpenAI Responses API params except 'instructions'. - """ - supported_params: Final = super().get_supported_openai_params(model) - - # Remove 'instructions' as it's not supported by XAI - if "instructions" in supported_params: - supported_params.remove("instructions") - - return supported_params - def _transform_web_search_tool(self, tool: Mapping[str, object]) -> Mapping[str, object]: """ Transform web_search tool to XAI format. @@ -158,19 +143,13 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): Map parameters for XAI Responses API. Handles XAI-specific transformations: - 1. Drops 'instructions' parameter (not supported) - 2. Transforms code_interpreter tools to remove 'container' field - 3. Transforms web_search tools to XAI format (removes search_context_size, adds filters) - 4. Transforms x_search tools to XAI format - 5. Sets store=false when images are detected (recommended by XAI) + 1. Transforms code_interpreter tools to remove 'container' field + 2. Transforms web_search tools to XAI format (removes search_context_size, adds filters) + 3. Transforms x_search tools to XAI format + 4. Sets store=false when images are detected (recommended by XAI) """ params: Final = dict(response_api_optional_params) - # Drop instructions parameter (not supported by XAI) - if "instructions" in params: - verbose_logger.debug("XAI Responses API does not support 'instructions' parameter. Dropping it.") - params.pop("instructions") - if "metadata" in params: verbose_logger.debug("XAI Responses API does not support 'metadata' parameter. Dropping it.") params.pop("metadata") diff --git a/litellm/main.py b/litellm/main.py index 9fbc5881b4f..1c6e47bfb11 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -105,7 +105,7 @@ from litellm.llms.base_llm.base_model_iterator import ( ) from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( @@ -2341,6 +2341,10 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: def _complete_aiohttp_openai( ctx: _CompletionDispatchContext, ) -> _CompletionDispatchResult: + if http2_enabled(): + verbose_logger.warning( + "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1" + ) acompletion: Final = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9e9f61507c2..dd21bbf0b25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1312,7 +1312,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1365,7 +1366,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1402,7 +1404,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1513,7 +1516,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1551,7 +1555,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1588,7 +1593,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1626,7 +1632,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1663,7 +1670,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -1701,7 +1709,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1812,7 +1821,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1848,7 +1858,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1884,7 +1895,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2029,7 +2041,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2066,7 +2079,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2103,7 +2117,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2286,7 +2301,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2323,7 +2339,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2360,7 +2377,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2505,7 +2523,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2539,7 +2558,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2573,7 +2593,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -3123,6 +3144,7 @@ "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -3514,6 +3536,7 @@ "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -3546,7 +3569,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -4151,12 +4174,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4167,13 +4193,17 @@ "azure/eu/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -4184,12 +4214,14 @@ "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4264,14 +4296,20 @@ }, "azure/eu/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -4297,14 +4335,20 @@ }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, "deprecation_date": "2027-02-09", "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -4330,8 +4374,9 @@ }, "azure/eu/gpt-5.1": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -4362,12 +4407,17 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.4e-07, + "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.38e-06, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -4398,18 +4448,20 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4433,7 +4485,7 @@ }, "azure/eu/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4441,6 +4493,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4466,12 +4519,15 @@ "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -4499,12 +4555,15 @@ "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4522,6 +4581,7 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4536,6 +4596,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4553,6 +4614,7 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -4562,12 +4624,15 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4579,12 +4644,15 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -4610,12 +4678,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4627,12 +4698,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4643,6 +4717,7 @@ "azure/global/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4674,7 +4749,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/global/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -4710,7 +4790,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -4722,6 +4803,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4753,6 +4835,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4985,8 +5068,10 @@ "azure/gpt-4.1": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -4994,6 +5079,8 @@ "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, + "output_cost_per_token_priority": 1.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5019,8 +5106,10 @@ "azure/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -5028,6 +5117,8 @@ "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, + "output_cost_per_token_priority": 1.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5053,8 +5144,10 @@ "azure/gpt-4.1-mini": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, + "input_cost_per_token_priority": 7e-07, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -5062,6 +5155,8 @@ "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, + "output_cost_per_token_priority": 2.8e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5087,8 +5182,10 @@ "azure/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, + "input_cost_per_token_priority": 7e-07, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -5096,6 +5193,8 @@ "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, + "output_cost_per_token_priority": 2.8e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5130,6 +5229,7 @@ "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5163,6 +5263,7 @@ "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5222,12 +5323,15 @@ "azure/gpt-4o-2024-05-13": { "deprecation_date": "2026-10-01", "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5238,12 +5342,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5254,13 +5361,16 @@ "azure/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, - "input_cost_per_token": 2.75e-06, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 1.1e-05, + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5447,13 +5557,16 @@ "azure/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2027-04-14", - "input_cost_per_token": 1.65e-07, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 6.6e-07, + "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5904,6 +6017,9 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_minimal_reasoning_effort": true }, "azure/gpt-5.1-chat-2025-11-13": { @@ -5942,7 +6058,8 @@ "supports_tool_choice": false, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -5957,6 +6074,7 @@ "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -5991,6 +6109,7 @@ "mode": "responses", "output_cost_per_token": 2e-06, "output_cost_per_token_priority": 3.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6015,13 +6134,19 @@ "azure/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6047,14 +6172,20 @@ }, "azure/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-02-09", "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6088,7 +6219,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6155,6 +6286,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6179,13 +6311,19 @@ "azure/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_batches": 1e-06, + "output_cost_per_token_priority": 3.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6211,14 +6349,20 @@ }, "azure/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-02-09", "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_batches": 1e-06, + "output_cost_per_token_priority": 3.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6246,12 +6390,15 @@ "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6279,12 +6426,15 @@ "cache_read_input_token_cost": 5e-09, "deprecation_date": "2027-02-09", "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6311,13 +6461,15 @@ "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", "input_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure?pivots=azure-openai&tabs=global-standard-aoai%2Cstandard-chat-completions%2Cglobal-standard#gpt-5", + "output_cost_per_token_batches": 6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6341,6 +6493,7 @@ "azure/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -6372,7 +6525,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -6408,7 +6566,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -6420,6 +6579,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6451,6 +6611,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6482,6 +6643,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6506,13 +6668,19 @@ "azure/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, + "input_cost_per_token_batches": 8.75e-07, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "output_cost_per_token_batches": 7e-06, + "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6542,6 +6710,7 @@ "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-06-08", "input_cost_per_token": 1.75e-06, + "input_cost_per_token_batches": 8.75e-07, "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -6549,7 +6718,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "output_cost_per_token_batches": 7e-06, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6587,6 +6758,7 @@ "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -6622,6 +6794,7 @@ "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -6654,6 +6827,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6688,6 +6862,7 @@ "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -6712,14 +6887,18 @@ }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-08-24", "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6743,17 +6922,20 @@ }, "azure/gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, + "input_cost_per_token_batches": 1.05e-05, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "output_cost_per_token_batches": 8.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -6779,17 +6961,20 @@ }, "azure/gpt-5.2-pro-2025-12-11": { "input_cost_per_token": 2.1e-05, + "input_cost_per_token_batches": 1.05e-05, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "output_cost_per_token_batches": 8.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -6817,6 +7002,7 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, @@ -6856,12 +7042,20 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_flex": 7.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4": { "deprecation_date": "2027-09-02", - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -6896,12 +7090,18 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4": { "deprecation_date": "2027-09-02", - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -6936,12 +7136,18 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "deprecation_date": "2027-09-02", @@ -6982,11 +7188,19 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_flex": 7.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4-2026-03-05": { - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", "input_cost_per_token": 2.75e-06, @@ -7022,11 +7236,17 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4-2026-03-05": { - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", "input_cost_per_token": 2.75e-06, @@ -7062,6 +7282,11 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -7071,6 +7296,9 @@ "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_flex": 1.5e-05, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -7078,11 +7306,15 @@ "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_flex": 9e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -7112,6 +7344,9 @@ "deprecation_date": "2027-09-07", "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_flex": 1.5e-05, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -7119,11 +7354,15 @@ "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_flex": 9e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -7202,33 +7441,42 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_creation_input_token_cost_priority": 1.25e-05, - "cache_creation_input_token_cost_above_272k_tokens_priority": 2.5e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_priority": 1e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_priority": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_flex": 2.5e-06, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_priority": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_flex": 2e-07, "deprecation_date": "2028-01-11", - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_priority": 1e-05, - "input_cost_per_token_above_272k_tokens_priority": 2e-05, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_priority": 8e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "input_cost_per_token_flex": 2e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_priority": 6e-05, - "output_cost_per_token_above_272k_tokens_priority": 9e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_priority": 4e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "output_cost_per_token_flex": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7259,17 +7507,23 @@ "azure/gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_flex": 1.25e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_priority": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_flex": 1e-07, "deprecation_date": "2028-01-11", "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_priority": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_flex": 1e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7277,13 +7531,16 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, + "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_priority": 2.4e-05, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, + "output_cost_per_token_flex": 6e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7314,17 +7571,23 @@ "azure/gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2028-01-11", "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, + "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_priority": 4e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, + "input_cost_per_token_flex": 1e-07, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7332,13 +7595,16 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_priority": 2.4e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, + "output_cost_per_token_flex": 6e-07, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7385,6 +7651,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -7542,33 +7809,34 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_272k_tokens_priority": 2.75e-05, - "cache_creation_input_token_cost_priority": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 2.2e-06, - "cache_read_input_token_cost_priority": 1.1e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2.2e-05, + "cache_creation_input_token_cost_priority": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, + "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_above_272k_tokens_priority": 2.2e-05, - "input_cost_per_token_priority": 1.1e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, + "input_cost_per_token_priority": 8.8e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_above_272k_tokens_priority": 9.9e-05, - "output_cost_per_token_priority": 6.6e-05, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, + "output_cost_per_token_above_272k_tokens_priority": 6.6e-05, + "output_cost_per_token_priority": 4.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7624,6 +7892,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7679,6 +7948,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7725,6 +7995,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -7845,33 +8116,34 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_272k_tokens_priority": 2.75e-05, - "cache_creation_input_token_cost_priority": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 2.2e-06, - "cache_read_input_token_cost_priority": 1.1e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2.2e-05, + "cache_creation_input_token_cost_priority": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, + "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_above_272k_tokens_priority": 2.2e-05, - "input_cost_per_token_priority": 1.1e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, + "input_cost_per_token_priority": 8.8e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_above_272k_tokens_priority": 9.9e-05, - "output_cost_per_token_priority": 6.6e-05, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, + "output_cost_per_token_above_272k_tokens_priority": 6.6e-05, + "output_cost_per_token_priority": 4.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7927,6 +8199,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7982,6 +8255,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8013,12 +8287,16 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, + "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_flex": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -8026,13 +8304,15 @@ "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, + "output_cost_per_token_batches": 1.5e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8064,9 +8344,10 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, @@ -8076,11 +8357,13 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8112,9 +8395,10 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, @@ -8124,11 +8408,13 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8159,11 +8445,12 @@ "azure/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, "litellm_provider": "azure", "max_input_tokens": 1050000, @@ -8172,7 +8459,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -8202,12 +8489,17 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "deprecation_date": "2027-10-26" + "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, "input_cost_per_token_priority": 1.375e-05, @@ -8247,12 +8539,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "deprecation_date": "2027-10-26" + "deprecation_date": "2027-10-26", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.65e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, "input_cost_per_token_priority": 1.375e-05, @@ -8292,7 +8587,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "deprecation_date": "2027-10-26" + "deprecation_date": "2027-10-26", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.65e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -8381,6 +8679,8 @@ "azure/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -8418,10 +8718,19 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, + "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_flex": 2.25e-06, + "output_cost_per_token_priority": 9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "deprecation_date": "2027-09-21", "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -8460,11 +8769,19 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, + "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_flex": 2.25e-06, + "output_cost_per_token_priority": 9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -8502,10 +8819,16 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_flex": 1e-07, + "output_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_flex": 6.25e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2027-09-21", "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -8544,6 +8867,11 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_flex": 1e-07, + "output_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_flex": 6.25e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-image-1": { @@ -8865,12 +9193,15 @@ "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, + "output_cost_per_token_batches": 3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8879,14 +9210,17 @@ "supports_vision": true }, "azure/o1-mini": { - "cache_read_input_token_cost": 6.05e-07, - "input_cost_per_token": 1.21e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 4.84e-06, + "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8896,12 +9230,15 @@ "azure/o1-mini-2024-09-12": { "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8917,6 +9254,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8932,6 +9270,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -8973,12 +9312,15 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_batches": 4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9014,6 +9356,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9057,12 +9400,15 @@ "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -9079,6 +9425,7 @@ "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9110,6 +9457,7 @@ "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9164,12 +9512,15 @@ "cache_read_input_token_cost": 2.75e-07, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_prompt_caching": true, @@ -9209,7 +9560,8 @@ "max_input_tokens": 8191, "max_tokens": 8191, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/text-embedding-3-small": { "deprecation_date": "2028-02-09", @@ -9218,7 +9570,8 @@ "max_input_tokens": 8191, "max_tokens": 8191, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/text-embedding-ada-002": { "deprecation_date": "2028-02-09", @@ -9227,7 +9580,8 @@ "max_input_tokens": 8191, "max_tokens": 8191, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/speech/azure-tts": { "input_cost_per_character": 1.5e-05, @@ -9267,8 +9621,10 @@ "azure/us/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -9276,6 +9632,8 @@ "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, + "output_cost_per_token_priority": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9301,8 +9659,10 @@ "azure/us/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.1e-07, + "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, "input_cost_per_token_batches": 2.2e-07, + "input_cost_per_token_priority": 7.7e-07, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -9310,6 +9670,8 @@ "mode": "chat", "output_cost_per_token": 1.76e-06, "output_cost_per_token_batches": 8.8e-07, + "output_cost_per_token_priority": 3.08e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9334,9 +9696,9 @@ }, "azure/us/gpt-4.1-nano-2025-04-14": { "deprecation_date": "2027-04-14", - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, - "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_batches": 5.5e-08, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -9344,6 +9706,7 @@ "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9369,12 +9732,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9385,13 +9751,17 @@ "azure/us/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -9402,12 +9772,14 @@ "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9482,14 +9854,20 @@ }, "azure/us/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9515,14 +9893,20 @@ }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, "deprecation_date": "2027-02-09", "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9550,12 +9934,15 @@ "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9581,8 +9968,9 @@ }, "azure/us/gpt-5.1": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -9613,12 +10001,17 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.4e-07, + "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.38e-06, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -9649,18 +10042,20 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -9684,7 +10079,7 @@ }, "azure/us/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -9692,6 +10087,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -9717,12 +10113,15 @@ "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9740,6 +10139,7 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9754,6 +10154,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9763,12 +10164,15 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9801,21 +10205,25 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": false }, "azure/us/o4-mini-2025-04-16": { - "cache_read_input_token_cost": 3.1e-07, + "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_prompt_caching": true, @@ -9861,7 +10269,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions" ], @@ -9910,7 +10318,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9925,7 +10333,7 @@ "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.828e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9941,7 +10349,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.52e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9957,7 +10365,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9972,37 +10380,37 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true }, "azure_ai/FW-GLM-5.2-Fast": { - "cache_read_input_token_cost": 2.1e-07, - "input_cost_per_token": 2.1e-06, + "cache_read_input_token_cost": 2.31e-07, + "input_cost_per_token": 2.31e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 6.6e-06, - "source": "https://docs.fireworks.ai/serverless/pricing", + "output_cost_per_token": 7.26e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true }, "azure_ai/FW-Inkling": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "max_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.05e-06, - "source": "https://fireworks.ai/models/fireworks/inkling", + "output_cost_per_token": 4.46e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text" ], @@ -10024,7 +10432,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.3e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10047,7 +10455,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10070,7 +10478,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10098,7 +10506,7 @@ "high", "max" ], - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10122,7 +10530,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.32e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10137,7 +10545,7 @@ "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.32e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10158,7 +10566,7 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text" ], @@ -10172,15 +10580,15 @@ "supports_vision": false }, "azure_ai/FW-Nemotron-3-Ultra-NVFP4": { - "cache_read_input_token_cost": 1.19e-07, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 1.3e-07, + "input_cost_per_token": 6.6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 2.4e-06, - "source": "https://fireworks.ai/models/fireworks/nemotron-3-ultra-nvfp4", + "output_cost_per_token": 2.64e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text" ], @@ -10199,7 +10607,7 @@ "mode": "image_generation", "output_cost_per_image": 0.05, "output_cost_per_image_token": 4.7e-05, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -10213,7 +10621,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0338, "output_cost_per_image_token": 3.3e-05, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -10227,7 +10635,7 @@ "mode": "image_generation", "output_cost_per_image": 0.02, "output_cost_per_image_token": 1.95e-05, - "source": "https://aka.ms/mai-image-2e-foundryblog", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/images/generations" ] @@ -10241,7 +10649,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions" ], @@ -10292,19 +10700,19 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 7.1e-07, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "input_cost_per_token": 1.41e-06, + "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 3.5e-07, - "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", + "output_cost_per_token": 1e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -10375,7 +10783,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10387,7 +10795,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10399,7 +10807,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10411,7 +10819,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10423,7 +10831,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10435,7 +10843,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10447,7 +10855,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.4e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10459,7 +10867,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10471,7 +10879,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": true }, @@ -10483,7 +10891,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/affordable-innovation-unveiling-the-pricing-of-phi-3-slms-on-models-as-a-service/4156495", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": false @@ -10496,7 +10904,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, - "source": "https://techcommunity.microsoft.com/blog/Azure-AI-Services-blog/announcing-new-phi-pricing-empowering-your-business-with-small-language-models/4395112", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { @@ -10508,20 +10916,20 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.2e-07, - "source": "https://techcommunity.microsoft.com/blog/Azure-AI-Services-blog/announcing-new-phi-pricing-empowering-your-business-with-small-language-models/4395112", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_audio_input": true, "supports_function_calling": true, "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { - "input_cost_per_token": 8e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 3.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", + "output_cost_per_token": 3e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { @@ -10532,7 +10940,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true @@ -10584,7 +10992,7 @@ "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10623,7 +11031,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_reasoning": true, "supports_tool_choice": true }, @@ -10688,7 +11096,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -10703,7 +11111,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -10719,7 +11127,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/deepseek-r1-improved-performance-higher-limits-and-transparent-pricing/4386367", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_reasoning": true, "supports_tool_choice": true }, @@ -10731,7 +11139,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/announcing-deepseek-v3-on-azure-ai-foundry-and-github/4390438", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true }, "azure_ai/deepseek-v3-0324": { @@ -10743,7 +11151,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/announcing-deepseek-v3-on-azure-ai-foundry-and-github/4390438", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10756,7 +11164,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.94e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true @@ -10770,7 +11178,7 @@ "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -10786,7 +11194,7 @@ "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -10803,7 +11211,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10817,7 +11225,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, - "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/embeddings" ], @@ -10836,7 +11244,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, - "source": "https://devblogs.microsoft.com/foundry/announcing-grok-3-and-grok-3-mini-on-azure-ai-foundry/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": false, "supports_tool_choice": true, @@ -10851,7 +11259,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.27e-06, - "source": "https://devblogs.microsoft.com/foundry/announcing-grok-3-and-grok-3-mini-on-azure-ai-foundry/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": false, @@ -10867,7 +11275,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": false, "supports_tool_choice": true, @@ -10882,7 +11290,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.27e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": false, @@ -10897,7 +11305,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10905,14 +11313,17 @@ }, "azure_ai/grok-4.3": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096", + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10923,14 +11334,17 @@ }, "azure_ai/grok-4.6": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/grok-4-6-comes-to-microsoft-foundry-models-built-for-long-horizon-reasoning-and-/4547578", + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10949,7 +11363,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10967,7 +11381,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10983,6 +11397,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10997,7 +11412,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -11011,7 +11426,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://techcommunity.microsoft.com/t5/Azure-AI-Foundry-Blog/Grok-4-0-Goes-GA-in-Microsoft-Foundry-and-Grok-4-1-Fast-Arrives/ba-p/4497964", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -11025,7 +11440,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://techcommunity.microsoft.com/t5/Azure-AI-Foundry-Blog/Grok-4-0-Goes-GA-in-Microsoft-Foundry-and-Grok-4-1-Fast-Arrives/ba-p/4497964", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -11040,7 +11455,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -11075,7 +11490,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, @@ -11092,7 +11507,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -11162,7 +11577,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.5e-06, - "source": "https://azure.microsoft.com/en-us/blog/introducing-mistral-large-3-in-microsoft-foundry-open-capable-and-ready-for-production-workloads/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -22432,6 +22847,7 @@ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { "cache_read_input_token_cost": 6e-07, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 1.2e-06, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -22819,6 +23235,7 @@ "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -22925,6 +23342,7 @@ "fireworks_ai/deepseek-v4-pro": { "cache_read_input_token_cost": 6e-07, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 1.2e-06, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -23143,6 +23561,7 @@ "fireworks_ai/minimax-m2p7": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -23664,6 +24083,7 @@ "cache_read_input_token_cost": 2.5e-08, "deprecation_date": "2026-06-01", "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_character": 3.75e-08, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -23743,6 +24163,7 @@ "cache_read_input_token_cost": 1.875e-08, "deprecation_date": "2026-06-01", "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_audio_token_batches": 3.75e-08, "input_cost_per_character": 1.875e-08, "input_cost_per_token": 7.5e-08, "input_cost_per_token_batches": 3.75e-08, @@ -23860,6 +24281,7 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, "input_cost_per_token_priority": 5.4e-07, @@ -24242,7 +24664,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -24384,6 +24807,7 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 5e-08, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_flex": 5e-08, "input_cost_per_token_priority": 1.8e-07, @@ -25001,6 +25425,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, @@ -25474,22 +25899,24 @@ } }, "gemini/gemini-robotics-er-2-preview": { - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 2e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, - "output_cost_per_token": 1e-05, + "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er-2", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25741,7 +26168,9 @@ "output_vector_size": 3072, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, "supports_multimodal": true, + "supports_vision": true, "tpm": 10000000 }, "gemini/gemini-1.5-flash": { @@ -25874,18 +26303,21 @@ } }, "gemini/gemini-2.5-flash": { + "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 3e-08, + "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 100000, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25919,6 +26351,14 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { @@ -25926,9 +26366,12 @@ "deprecation_date": "2026-10-02", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "gemini", "supports_reasoning": false, - "max_input_tokens": 32768, + "max_input_tokens": 65536, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -25937,7 +26380,7 @@ "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 100000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-flash-image", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25954,28 +26397,31 @@ "image" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, + "supports_web_search": false, "tpm": 8000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, + "supports_audio_input": false, "supports_image_size": false }, "gemini/gemini-3-pro-image": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 3.6e-06, "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, @@ -25987,7 +26433,9 @@ "rpm": 1000, "tpm": 4000000, "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image", + "output_cost_per_token_flex": 6e-06, + "output_cost_per_token_priority": 2.16e-05, + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26003,7 +26451,7 @@ ], "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26106,7 +26554,7 @@ "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -26116,7 +26564,7 @@ "output_cost_per_token_batches": 1.5e-06, "rpm": 1000, "tpm": 4000000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26133,7 +26581,7 @@ "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26201,7 +26649,7 @@ "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "rpm": 1000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26215,12 +26663,13 @@ "text", "image" ], - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": false, "supports_reasoning": false, "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, + "supports_web_search": false, "tpm": 4000000 }, "gemini/deep-research-pro-preview-12-2025": { @@ -26265,18 +26714,21 @@ } }, "gemini/gemini-2.5-flash-lite": { + "cache_read_input_audio_token_cost": 3e-08, "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_flex": 1e-08, + "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26310,6 +26762,14 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 1.5e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 1.8e-07, + "output_cost_per_token_batches": 2e-07, + "output_cost_per_token_flex": 2e-07, + "output_cost_per_token_priority": 7.2e-07, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { @@ -26555,34 +27015,46 @@ }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", + "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, + "cache_read_input_token_cost_flex": 1.25e-07, + "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, - "input_cost_per_token_priority": 1.25e-06, - "input_cost_per_token_above_200k_tokens_priority": 2.5e-06, + "input_cost_per_token_priority": 2.25e-06, + "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, - "output_cost_per_token_priority": 1e-05, - "output_cost_per_token_above_200k_tokens_priority": 1.5e-05, + "output_cost_per_token_priority": 1.8e-05, + "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "rpm": 2000, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -26613,7 +27085,11 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "google_maps_grounding_cost_per_query": 0.025 + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_flex": 6.25e-07, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": 5e-06 }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, @@ -26626,7 +27102,7 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, "rpm": 2000, - "source": "https://ai.google.dev/gemini-api/docs/computer-use", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -26754,6 +27230,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini/gemini-3.1-flash-lite": { + "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -26774,7 +27251,7 @@ "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26811,7 +27288,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, @@ -26872,13 +27350,15 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini/gemini-3-flash-preview": { + "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -26922,7 +27402,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_token_batches": 2.5e-07, + "input_cost_per_token_flex": 2.5e-07, + "output_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_flex": 1.5e-06, + "supports_audio_input": true }, "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -26931,8 +27417,8 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -27212,7 +27698,7 @@ "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06, "rpm": 2000, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-3.1-pro-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -27247,13 +27733,16 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 6e-06 }, "gemini/gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, @@ -27271,7 +27760,7 @@ "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06, "rpm": 2000, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-3.1-pro-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -27306,13 +27795,16 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 6e-06 }, "gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, @@ -27366,6 +27858,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, @@ -27637,11 +28130,13 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -27652,19 +28147,20 @@ "audio" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, + "supports_vision": false, + "supports_web_search": false, "tpm": 10000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_audio_input": false }, "gemini/gemini-exp-1114": { "input_cost_per_token": 0, @@ -30223,6 +30719,7 @@ "mode": "image_generation", "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, "output_cost_per_token_batches": 5e-06, @@ -30241,6 +30738,7 @@ "mode": "image_generation", "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, "output_cost_per_token_batches": 5e-06, @@ -30257,6 +30755,7 @@ "litellm_provider": "openai", "mode": "image_generation", "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -33033,6 +33532,7 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-10-23", "input_cost_per_image_token": 1e-05, + "input_cost_per_image_token_batches": 5e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "openai", @@ -33048,6 +33548,7 @@ "cache_read_input_token_cost": 2e-07, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 2.5e-06, + "input_cost_per_image_token_batches": 1.25e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "openai", @@ -44106,7 +44607,7 @@ "supports_tool_choice": true }, "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -44399,7 +44900,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -44414,7 +44915,7 @@ "supports_vision": true }, "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", "max_input_tokens": 514, @@ -44527,7 +45028,7 @@ "supports_tool_choice": true }, "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", @@ -44977,6 +45478,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -45009,6 +45511,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -45040,6 +45543,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -45089,7 +45593,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us-gov.nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -48221,7 +48726,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 2.5e-07 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -48298,49 +48804,56 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/imagegeneration@006": { + "deprecation_date": "2025-09-24", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.02, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.02, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-generate-002": { - "deprecation_date": "2025-11-10", + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-capability-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" }, "vertex_ai/imagen-4.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.02, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-4.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-4.0-ultra-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.06, @@ -55277,6 +55790,7 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "openai", @@ -55424,7 +55938,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -55444,7 +55958,11 @@ ], "supports_audio_input": true, "supports_audio_output": true, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -55477,7 +55995,9 @@ "supports_function_calling": true, "supports_vision": true, "supports_web_search": true, - "gemini_audio_only_live": true + "gemini_audio_only_live": true, + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, @@ -55539,7 +56059,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -55561,7 +56081,11 @@ "supports_audio_output": true, "tpm": 250000, "rpm": 10, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini/gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -55596,32 +56120,48 @@ "supports_web_search": true, "tpm": 250000, "rpm": 10, - "gemini_audio_only_live": true + "gemini_audio_only_live": true, + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "audio_speech", + "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, - "source": "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", + "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" - ] + ], + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-flash-latest": { "cache_read_input_token_cost": 3e-08, @@ -58195,6 +58735,9 @@ ], "supports_audio_input": true, "supports_audio_output": true, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false, "tpm": 250000 }, "gemini/gemini-3.5-transcribe": { @@ -58216,7 +58759,8 @@ ], "supports_audio_input": true, "tpm": 800000, - "rpm": 2000 + "rpm": 2000, + "supports_function_calling": false }, "gemini/gemini-3.5-transcribe-live": { "input_cost_per_audio_token": 3.5e-06, @@ -58236,7 +58780,8 @@ ], "supports_audio_input": true, "tpm": 250000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false }, "vertex_ai/gemini-3.5-transcribe-preview": { "input_cost_per_audio_token": 2e-06, @@ -60875,7 +61420,7 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", @@ -61738,7 +62283,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -65816,6 +66361,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { + "deprecation_date": "2024-08-22", "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65823,6 +66369,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { + "deprecation_date": "2025-12-23", "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "mode": "chat", @@ -65830,6 +66377,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65837,6 +66385,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { + "deprecation_date": "2025-11-13", "input_cost_per_token": 1.6e-06, "litellm_provider": "together_ai", "mode": "chat", @@ -65930,6 +66479,7 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "together_ai/google/gemma-2-27b-it": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65954,6 +66504,7 @@ "source": "https://developers.openai.com/api/docs/pricing" }, "together_ai/meta-llama/Llama-3-8b-chat-hf": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65982,6 +66533,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { + "deprecation_date": "2025-12-23", "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65989,6 +66541,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65996,6 +66549,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66003,6 +66557,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66017,6 +66572,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2-72B-Instruct": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 9e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66024,6 +66580,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2-VL-72B-Instruct": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "mode": "chat", @@ -66045,6 +66602,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { + "deprecation_date": "2025-11-13", "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66052,12 +66610,598 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { + "deprecation_date": "2026-01-05", "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://api.together.ai/v1/models" }, + "azure/eu/codex-mini": { + "cache_read_input_token_cost": 4.13e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/computer-use-preview": { + "input_cost_per_token": 3.3e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4.1": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_priority": 9.63e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "output_cost_per_token_priority": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4.1-mini": { + "cache_read_input_token_cost": 1.1e-07, + "cache_read_input_token_cost_priority": 1.93e-07, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_batches": 2.2e-07, + "input_cost_per_token_priority": 7.7e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.76e-06, + "output_cost_per_token_batches": 8.8e-07, + "output_cost_per_token_priority": 3.08e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4.1-nano": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.1e-07, + "input_cost_per_token_batches": 5.5e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4o-2024-05-13": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_batches": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5": { + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-codex": { + "cache_read_input_token_cost": 1.38e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-mini": { + "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, + "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-nano": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-pro": { + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000132, + "output_cost_per_token_batches": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.1-codex-max": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_batches": 9.625e-07, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_batches": 7.7e-06, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2-codex": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2-pro": { + "input_cost_per_token": 2.31e-05, + "input_cost_per_token_batches": 1.155e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.0001848, + "output_cost_per_token_batches": 9.24e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.3-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.3-codex": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.4-mini": { + "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_priority": 1.65e-07, + "input_cost_per_token": 8.25e-07, + "input_cost_per_token_batches": 4.125e-07, + "input_cost_per_token_priority": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.95e-06, + "output_cost_per_token_batches": 2.475e-06, + "output_cost_per_token_priority": 9.9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.4-nano": { + "cache_read_input_token_cost": 2.2e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.375e-06, + "output_cost_per_token_batches": 6.875e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.4-pro": { + "input_cost_per_token": 3.3e-05, + "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_batches": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000198, + "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_batches": 9.9e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-6-astra": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o1-mini": { + "cache_read_input_token_cost": 6.05e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o1-preview": { + "cache_read_input_token_cost": 8.25e-06, + "input_cost_per_token": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o3-2025-04-16": { + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o3-deep-research": { + "cache_read_input_token_cost": 2.75e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o4-mini-2025-04-16": { + "cache_read_input_token_cost": 3.03e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/text-embedding-3-large": { + "input_cost_per_token": 1.43e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/text-embedding-3-small": { + "input_cost_per_token": 2.2e-08, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/text-embedding-ada-002": { + "input_cost_per_token": 1.1e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "gemini/gemini-3.8-live": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 4.5e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, + "tpm": 250000, + "rpm": 10, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true + }, + "gemini/gemini-3.8-live-extended-thinking": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 4.5e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, + "tpm": 250000, + "rpm": 10, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true + }, + "azure/us/codex-mini": { + "cache_read_input_token_cost": 4.13e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/computer-use-preview": { + "input_cost_per_token": 3.3e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4.1": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_priority": 9.63e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "output_cost_per_token_priority": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4.1-mini": { + "cache_read_input_token_cost": 1.1e-07, + "cache_read_input_token_cost_priority": 1.93e-07, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_batches": 2.2e-07, + "input_cost_per_token_priority": 7.7e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.76e-06, + "output_cost_per_token_batches": 8.8e-07, + "output_cost_per_token_priority": 3.08e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4.1-nano": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.1e-07, + "input_cost_per_token_batches": 5.5e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4o-2024-05-13": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_batches": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5": { + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-codex": { + "cache_read_input_token_cost": 1.38e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-mini": { + "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, + "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-nano": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-pro": { + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000132, + "output_cost_per_token_batches": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.1-codex-max": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_batches": 9.625e-07, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_batches": 7.7e-06, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2-codex": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2-pro": { + "input_cost_per_token": 2.31e-05, + "input_cost_per_token_batches": 1.155e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.0001848, + "output_cost_per_token_batches": 9.24e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.3-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.3-codex": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.4-mini": { + "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_priority": 1.65e-07, + "input_cost_per_token": 8.25e-07, + "input_cost_per_token_batches": 4.125e-07, + "input_cost_per_token_priority": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.95e-06, + "output_cost_per_token_batches": 2.475e-06, + "output_cost_per_token_priority": 9.9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.4-nano": { + "cache_read_input_token_cost": 2.2e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.375e-06, + "output_cost_per_token_batches": 6.875e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.4-pro": { + "input_cost_per_token": 3.3e-05, + "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_batches": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000198, + "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_batches": 9.9e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/o1-mini": { + "cache_read_input_token_cost": 6.05e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/o1-preview": { + "cache_read_input_token_cost": 8.25e-06, + "input_cost_per_token": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/o3-deep-research": { + "cache_read_input_token_cost": 2.75e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/text-embedding-3-large": { + "input_cost_per_token": 1.43e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/text-embedding-3-small": { + "input_cost_per_token": 2.2e-08, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/text-embedding-ada-002": { + "input_cost_per_token": 1.1e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, "litellm_provider": "aihubmix", diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index 35a30127e27..2b13baa624b 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -1,6 +1,8 @@ """Bridge token flow: litellm identity resolution and the DCR-bridge oauth_delegate mint/refresh pipeline.""" import math +import os +import secrets from dataclasses import dataclass from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Literal @@ -12,6 +14,9 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + _V2_GCM_PREFIX, # pyright: ignore[reportPrivateUsage] # reuse the encrypted credential's format discriminator +) from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: @@ -24,6 +29,7 @@ if TYPE_CHECKING: UpstreamTokenGrant, ) from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.handle_jwt import JWTIdentity def _litellm_key_from_request(request: Request) -> str | None: @@ -48,6 +54,64 @@ def _litellm_key_from_request(request: Request) -> str | None: return None +async def oauth_authorization_uses_gateway_credential(request: Request) -> bool: + """Classify credentials for browser authorize; candidates still require full authorization.""" + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration + jwt_handler, + master_key, + user_custom_auth, + ) + + if "x-litellm-api-key" in request.headers: + return True + token: Final = _litellm_key_from_request(request) + if token is None: + return "authorization" in request.headers + if token.startswith("sk-") or (master_key and secrets.compare_digest(token.encode(), master_key.encode())): + return True + if user_custom_auth is not None or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: + return True + if not JWTHandler.is_jwt(token): + return await _opaque_bearer_is_gateway_credential(token) + claims: Final = JWTHandler.get_unverified_claims(token) + issuer: Final = claims.get("iss") if claims is not None else None + global_issuer: Final = os.getenv("JWT_ISSUER") + # An unscoped global validator can accept issuers absent from the configured issuer list. + if not isinstance(issuer, str) or not issuer or not global_issuer: + return True + return issuer == global_issuer or any( + issuer == configured.issuer for configured in jwt_handler.litellm_jwtauth.issuers or () + ) + + +async def _opaque_bearer_is_gateway_credential(token: str) -> bool: + from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + is_envelope, # noqa: PLC0415 # envelope imports bridge types + is_refresh_envelope, + ) + from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies + prisma_client, + user_api_key_cache, + ) + + if is_envelope(token) or is_refresh_envelope(token) or token.startswith(_V2_GCM_PREFIX): + return True + try: + if ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None: + return True + await IdentityStore(prisma_client, user_api_key_cache).resolve(hashed_token=hash_token(token)) + except KeyNotFoundError: + return False + except Exception as exc: # noqa: BLE001 # an identity lookup fault must not permit cookie fallback + verbose_logger.debug("OAuth bearer ownership could not be checked (%s)", type(exc).__name__) + return True + + def _key_is_active(key_obj: "UserAPIKeyAuth") -> bool: """``True`` when the presented key is neither blocked nor past its expiry. @@ -243,6 +307,10 @@ async def load_active_user_by_id(user_id: str) -> "LiteLLM_UserTable | _KeyResol return "no_active_key" if user_object is None: return "no_active_key" + return _active_user_record(user_object) + + +def _active_user_record(user_object: "LiteLLM_UserTable") -> "LiteLLM_UserTable | Literal['no_active_key']": if isinstance(user_object.metadata, dict) and user_object.metadata.get("scim_active") is False: return "no_active_key" return user_object @@ -301,15 +369,137 @@ async def _revalidate_active_subject(identity: "EnvelopeIdentity") -> "_KeyResol async def _extract_user_id_from_request(request: Request) -> str | None: - """The litellm ``user_id`` for the token request, so a per-user token is stored under the same - identity the egress later reads it by. Storage is best-effort, so every non-resolved outcome - (including a transient DB outage) collapses to ``None`` here and the caller simply skips the store; - the bridge mint, which must status those outcomes differently, consumes - :func:`_resolve_active_litellm_key` directly.""" - resolved: Final = await _resolve_active_litellm_key(request) - if not isinstance(resolved, _ResolvedKey): + """Resolve the caller for identity binding without granting credential-write permission.""" + from litellm.proxy.auth.handle_jwt import JWTIdentity # noqa: PLC0415 # proxy import cycle + + resolved: Final = await _resolve_request_auth(request) + if isinstance(resolved, JWTIdentity): + return resolved.user_id + return _active_key_user_id(resolved) if resolved is not None else None + + +async def authorize_oauth_credential_request(request: Request, server_id: str) -> str | None: + from litellm.proxy._types import UserAPIKeyAuth # noqa: PLC0415 # proxy import cycle + + resolved: Final = await _resolve_request_auth(request, f"/v1/mcp/server/{server_id}/oauth-user-credential") + if not isinstance(resolved, UserAPIKeyAuth) or not _active_key_user_id(resolved): + return None + if not await can_store_oauth_credential(request, resolved, server_id): + return None + return resolved.user_id + + +async def _resolve_request_auth( + request: Request, write_route: str | None = None +) -> "UserAPIKeyAuth | JWTIdentity | None": + from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle + + token: Final = _litellm_key_from_request(request) + if token is not None and JWTHandler.is_jwt(token): + return await _resolve_jwt_auth(request, token, write_route) + resolved: Final = await _resolve_active_litellm_key(request) + return resolved.key if isinstance(resolved, _ResolvedKey) else None + + +async def can_store_oauth_credential(request: Request, auth: "UserAPIKeyAuth", server_id: str) -> bool: + """Apply the same write policy to request credentials and verified signed-callback users.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # registry imports auth helpers + global_mcp_server_manager, + ) + from litellm.proxy._experimental.mcp_server.ui_session_utils import ( + can_access_mcp_server, # noqa: PLC0415 # proxy import cycle + ) + from litellm.proxy.auth.route_checks import RouteChecks # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.user_api_key_auth import ( # noqa: PLC0415 # proxy import cycle + _run_centralized_common_checks, # pyright: ignore[reportPrivateUsage] # reuse admission policy for the credential-write action + ) + + write_route: Final = f"/v1/mcp/server/{server_id}/oauth-user-credential" + try: + RouteChecks.is_virtual_key_allowed_to_call_route(route=write_route, valid_token=auth, request=request) + await _run_centralized_common_checks( + user_api_key_auth_obj=auth, + request=request, + request_data={}, + route=write_route, + ) + return await can_access_mcp_server(auth, server_id, global_mcp_server_manager.get_allowed_mcp_servers) + except Exception as exc: # noqa: BLE001 # authorization failure must never write credentials + verbose_logger.debug("OAuth credential write not authorized (%s)", type(exc).__name__) + return False + + +async def _resolve_jwt_auth( + request: Request, + token: str, + write_route: str | None, +) -> "UserAPIKeyAuth | JWTIdentity | None": + from litellm.proxy._types import UserAPIKeyAuth # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.handle_jwt import JWTAuthManager # noqa: PLC0415 # proxy import cycle + from litellm.proxy.auth.user_api_key_auth import ( # noqa: PLC0415 # proxy import cycle + _resolve_jwt_to_virtual_key, # pyright: ignore[reportPrivateUsage] # reuse admission mapping policy without provisioning a new key + ) + from litellm.proxy.proxy_server import ( # noqa: PLC0415 # proxy globals initialized at startup + general_settings, + jwt_handler, + premium_user, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if general_settings.get("enable_jwt_auth") is not True or premium_user is not True or prisma_client is None: + return None + try: + if jwt_handler.litellm_jwtauth.is_virtual_key_mapping_configured(): + claims: Final = await jwt_handler.auth_jwt(token=token) + validate: Final = jwt_handler.litellm_jwtauth.custom_validate + if validate is not None and not validate(claims): + return None + mapped: Final = await _resolve_jwt_to_virtual_key( + jwt_claims=claims, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_logging_obj, + ) + if isinstance(mapped, UserAPIKeyAuth): + return None if await _key_owner_scim_deactivated(mapped) or not _active_key_user_id(mapped) else mapped + if mapped is not None: + return None + if write_route is None: + identity: Final = await JWTAuthManager.resolve_identity( + api_key=token, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_logging_obj, + ) + if identity.user_object is not None and isinstance(_active_user_record(identity.user_object), str): + return None + return identity + authorized: Final = await JWTAuthManager.authorize_jwt( + api_key=token, + jwt_handler=jwt_handler, + request_data={}, + general_settings=general_settings, + route=write_route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_logging_obj, + request_headers=dict(request.headers), + request_method=request.method, + ) + resolved_user: Final = authorized["user_object"] + if resolved_user is not None and isinstance(_active_user_record(resolved_user), str): + return None + return JWTAuthManager.user_api_key_auth_from_result(authorized) + except Exception as exc: # noqa: BLE001 # public OAuth exchange stays available; unvalidated identities never write credentials + verbose_logger.debug("OAuth JWT identity could not be validated (%s)", type(exc).__name__) return None - return _active_key_user_id(resolved.key) _UpstreamGrantRejection = Literal["no_access_token", "expired_lifetime"] diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index bafe33d0a6b..ffb27d5f92e 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -32,6 +32,9 @@ from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( _prepare_bridge_mint, _prepare_bridge_refresh, _reload_active_user_by_id, + authorize_oauth_credential_request, + can_store_oauth_credential, + oauth_authorization_uses_gateway_credential, ) from litellm.proxy._experimental.mcp_server.faults import ( CallerRejected, @@ -836,16 +839,30 @@ async def _user_can_reach_mcp_server(user_id: str, server_id: str) -> bool: return server_id in await global_mcp_server_manager.get_allowed_mcp_servers(admitted) -async def _bridge_authorize_access_denial( - litellm_user_id: str, +async def _resolve_oauth_authorization_user( + request: Request, mcp_server: MCPServer, redirect_uri: str, state: str, -) -> RedirectResponse | None: - """The denial redirect for a signed-in user who cannot reach the target server, or None to proceed.""" - if await _user_can_reach_mcp_server(litellm_user_id, mcp_server.server_id): - return None - return _bridge_access_denied_redirect(redirect_uri, state, mcp_server) + enforce_binding: bool, +) -> str | RedirectResponse: + """Resolve the authorization subject without replacing denied credentials with cookie grants.""" + from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # proxy import cycle + _user_id_from_session_cookie, + ) + + use_gateway_credential: Final = enforce_binding and await oauth_authorization_uses_gateway_credential(request) + request_user_id: Final = ( + await authorize_oauth_credential_request(request, mcp_server.server_id) if use_gateway_credential else None + ) + if use_gateway_credential and request_user_id is None: + return _bridge_access_denied_redirect(redirect_uri, state, mcp_server) + user_id: Final = request_user_id or _user_id_from_session_cookie(request) + if user_id is None: + return _redirect_to_litellm_login(request) + if not await _user_can_reach_mcp_server(user_id, mcp_server.server_id): + return _bridge_access_denied_redirect(redirect_uri, state, mcp_server) + return user_id async def authorize_with_server( @@ -911,23 +928,12 @@ async def authorize_with_server( # Seal the authenticated caller into state so the token exchange cannot select another credential owner. litellm_user_id: str | None = None if enforce_binding or (resolved_server.is_dcr_bridge and resolved_server.is_oauth_delegate): - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # inline import avoids a module-load circular import - _user_id_from_session_cookie, + subject: Final = await _resolve_oauth_authorization_user( + request, resolved_server, redirect_uri, state, enforce_binding ) - - litellm_user_id = ( - await _extract_user_id_from_request(request) if enforce_binding else None - ) or _user_id_from_session_cookie(request) - if litellm_user_id is None: - return _redirect_to_litellm_login(request) - denial: Final = await _bridge_authorize_access_denial( - litellm_user_id=litellm_user_id, - mcp_server=resolved_server, - redirect_uri=redirect_uri, - state=state, - ) - if denial is not None: - return denial + if isinstance(subject, RedirectResponse): + return subject + litellm_user_id = subject oauth_nonce: Final = secrets.token_urlsafe(32) if enforce_binding else None encoded_state: Final = encode_state_with_base_url( @@ -1218,12 +1224,32 @@ async def exchange_token_with_server( user_id: Final = resolved_user_id if user_id: try: - await _store_per_user_token_server_side( - server=resolved_server, - user_id=user_id, - token_response=token_response, - identity_binding_proof=binding_proof, + # Identity binding above must retain the verified caller even when a write is + # denied. Authorize persistence separately, immediately before its side effect. + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + # A sealed code delegates a verified user for this authorized server. Raw + # request credentials retain their own JWT/key restrictions during resolution. + can_store: Final = ( + await can_store_oauth_credential( + request, await MCPRequestHandler.reload_admitted_user(user_id), resolved_server.server_id + ) + if bridge_identity is not None + else await authorize_oauth_credential_request(request, resolved_server.server_id) == user_id ) + if can_store: + await _store_per_user_token_server_side( + server=resolved_server, + user_id=user_id, + token_response=token_response, + identity_binding_proof=binding_proof, + ) + else: + verbose_logger.warning( + "OAuth credential storage not authorized for user=%s server=%s", + user_id, + resolved_server.server_id, + ) except Exception as exc: verbose_logger.warning( "exchange_token_with_server: server-side storage failed for user=%s server=%s: %s", @@ -1236,8 +1262,9 @@ async def exchange_token_with_server( "exchange_token_with_server: could not resolve a LiteLLM user_id for the request, " "so the per-user token for server=%s was NOT stored. The authorization_code egress " "requires the stored token, so the client will be challenged with 401 on reconnect. " - "Ensure the request carries a valid LiteLLM key (x-litellm-api-key or Authorization), " - "or store it via POST /mcp/server/{id}/oauth-user-credential.", + "Ensure the request carries a valid LiteLLM key or enabled JWT identity " + "(x-litellm-api-key or Authorization), " + "or store it via POST /v1/mcp/server/{id}/oauth-user-credential.", resolved_server.server_id, ) diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py index 125dc3d773d..fec2a1f9ee6 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -771,6 +771,7 @@ async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | N try: import litellm + from litellm.proxy._types import ModelAccessDeniedProxyException from litellm.proxy.auth.auth_checks import ( _check_team_member_model_access, can_key_call_model, @@ -884,11 +885,14 @@ async def _check_model_access(model: str, user_api_key_auth: "UserAPIKeyAuth | N ) return None except Exception as access_err: - verbose_logger.warning( - "MCP sampling: model access denied for model=%s: %s", - model, - access_err, - ) + if isinstance(access_err, ModelAccessDeniedProxyException): + verbose_logger.warning( + "MCP sampling: model access denied for model=%s: %s", + model, + access_err.sanitized_internal_message(), + ) + return ErrorData(code=-1, message=access_err.message) + verbose_logger.warning("MCP sampling: model access denied for model=%s: %s", model, access_err) return ErrorData( code=-1, message=(f"Model access denied: the API key is not authorized to use model '{model}'. {access_err}"), diff --git a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py index 188bfce1484..107a4818de1 100644 --- a/litellm/proxy/_experimental/mcp_server/ui_session_utils.py +++ b/litellm/proxy/_experimental/mcp_server/ui_session_utils.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Awaitable, Callable from typing import Final from fastapi import HTTPException @@ -137,3 +138,15 @@ async def build_effective_auth_contexts( if admitted_context is None: return team_contexts return [*team_contexts, admitted_context] + + +async def can_access_mcp_server( + user_api_key_auth: UserAPIKeyAuth, + server_id: str, + allowed_servers: Callable[[UserAPIKeyAuth], Awaitable[list[str]]], +) -> bool: + """Resolve server access through the same credential contexts as MCP management.""" + for context in await build_effective_auth_contexts(user_api_key_auth): + if server_id in await allowed_servers(context): + return True + return False diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 321f8190f13..63d0bfcc5b8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4032,6 +4032,22 @@ class ProxyException(Exception): return error_dict +class ModelAccessDeniedProxyException(ProxyException): + def __init__( + self, + message: str, + internal_message: str, + type: str, + param: str | None, + code: int | str | None, + ) -> None: + super().__init__(message=message, type=type, param=param, code=code) + self.internal_message: Final = internal_message + + def sanitized_internal_message(self) -> str: + return self.internal_message.replace("\r", "").replace("\n", "") + + class CommonProxyErrors(str, enum.Enum): db_not_connected_error = ( "DB not connected. This endpoint needs a database; set DATABASE_URL to a " diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index e3783c94dc7..ba68dc8a17f 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -60,6 +60,7 @@ from litellm.proxy._types import ( LiteLLM_UserTable, LiteLLMRoutes, LitellmUserRoles, + ModelAccessDeniedProxyException, NewTeamRequest, ProxyErrorTypes, ProxyException, @@ -71,6 +72,7 @@ from litellm.proxy.auth.budget_throttle import ( budget_throttle_percentage, should_throttle_budget_exceeded, ) +from litellm.proxy.auth.model_access_denied import model_access_denied_client_message from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec @@ -4170,8 +4172,13 @@ def _can_object_call_model( ): return True - raise ProxyException( - message=f"{object_type} not allowed to access model. This {object_type} can only access models={models}. Tried to access {model}", + internal_message: Final = ( + f"{object_type} not allowed to access model. This {object_type} can only access models={models}. " + f"Tried to access {model}" + ) + raise ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=model), + internal_message=internal_message, type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type=object_type), param="model", code=status.HTTP_403_FORBIDDEN, @@ -4796,8 +4803,13 @@ async def can_user_call_model( return True if SpecialModelNames.no_default_models.value in user_object.models: - raise ProxyException( - message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}", + internal_message: Final = ( + f"User not allowed to access model. No default model access, only team models allowed. " + f"Tried to access {model}" + ) + raise ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=model), + internal_message=internal_message, type=ProxyErrorTypes.key_model_access_denied, param="model", code=status.HTTP_403_FORBIDDEN, @@ -5398,8 +5410,13 @@ async def _check_team_member_model_access( team_id=team_object.team_id, ) except ProxyException: - raise ProxyException( - message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}", + internal_message: Final = ( + f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, " + f"Model={model}. Allowed member models = {member_allowed_models}" + ) + raise ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=model), + internal_message=internal_message, type=ProxyErrorTypes.team_model_access_denied, param="model", code=status.HTTP_403_FORBIDDEN, diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 661b6a83c38..bbe4b0f5c35 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -15,6 +15,7 @@ from litellm.integrations.otel.runtime import seed_request_identity from litellm.litellm_core_utils.core_helpers import is_expected_client_error from litellm.proxy._types import ( LitellmUserRoles, + ModelAccessDeniedProxyException, ProxyErrorTypes, ProxyException, UserAPIKeyAuth, @@ -25,6 +26,7 @@ from litellm.proxy.auth.auth_utils import ( mark_invalid_virtual_key_error, normalize_request_route, ) +from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -51,6 +53,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException: param=None, code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), ) + if isinstance(e, ModelAccessDeniedHTTPException): + return ModelAccessDeniedProxyException( + message=str(e.detail), + internal_message=e.internal_message, + type=ProxyErrorTypes.auth_error, + param="None", + code=e.status_code, + ) if isinstance(e, HTTPException): return ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 94ca3047f45..6a28cd7ff99 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -15,6 +15,7 @@ import os import re import time from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass from typing import Any, Final, Literal, NoReturn, Protocol, TypeVar, cast import httpx @@ -52,9 +53,13 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_checks import can_team_access_model +from litellm.proxy.auth.model_access_denied import ( + ModelAccessDeniedHTTPException, + model_access_denied_client_message, +) from litellm.proxy.auth.resolvers.grants import GrantResolver, UserLookup, canonical_user_id from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.auth.team_grants import team_model_aliases +from litellm.proxy.auth.team_grants import team_grants, team_model_aliases from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, get_management_object_ttl, @@ -62,6 +67,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.user_repository import UserRepository from litellm.types.agents import AgentResponse +from litellm.types.proxy.auth.auth_checks import UserNotFoundError from .auth_checks import ( _allowed_routes_check, @@ -128,6 +134,19 @@ class _UserInfoResponse(Protocol): def json(self) -> dict[str, object]: ... +@dataclass(frozen=True, slots=True) +class JWTIdentity: + user_id: str | None + user_object: LiteLLM_UserTable | None + agent_id: str | None + + +@dataclass(frozen=True, slots=True) +class _JWTProvisioning: + user_id_upsert: bool + team_id_upsert: bool + + class AgentLookup(Protocol): """The registered-agent lookups a JWT agent claim is matched against.""" @@ -1337,9 +1356,13 @@ class JWTAuthManager: return True if model not in role_based_models: - raise HTTPException( + internal_message: Final = ( + f"Role={rbac_role} not allowed to call model={model}. Allowed models={role_based_models}" + ) + raise ModelAccessDeniedHTTPException( + internal_message=internal_message, status_code=403, - detail=f"Role={rbac_role} not allowed to call model={model}. Allowed models={role_based_models}", + detail=model_access_denied_client_message(model=model), ) return True @@ -1368,9 +1391,11 @@ class JWTAuthManager: return if requested_model not in allowed_models: - raise HTTPException( + internal_message: Final = f"model={requested_model} not allowed. Allowed_models={allowed_models}" + raise ModelAccessDeniedHTTPException( + internal_message=internal_message, status_code=403, - detail={"error": f"model={requested_model} not allowed. Allowed_models={allowed_models}"}, + detail={"error": model_access_denied_client_message(model=requested_model)}, ) return @@ -1471,6 +1496,7 @@ class JWTAuthManager: user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None, proxy_logging_obj: ProxyLogging, + team_id_upsert: bool | None = None, ) -> tuple[str | None, LiteLLM_TeamTable | None]: """Find and validate specific team ID from team_id_jwt_field or team_alias_jwt_field""" individual_team_id = jwt_handler.get_team_id(token=jwt_valid_token, default_value=None) @@ -1498,7 +1524,9 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, - team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert + if team_id_upsert is None + else team_id_upsert, ) return individual_team_id, team_object except HTTPException as e: @@ -1726,6 +1754,7 @@ class JWTAuthManager: proxy_logging_obj: ProxyLogging, route: str, org_alias: str | None = None, + user_id_upsert: bool | None = None, ) -> tuple[ LiteLLM_UserTable | None, LiteLLM_OrganizationTable | None, @@ -1789,7 +1818,11 @@ class JWTAuthManager: user_id=user_id, user_email=user_email, sso_user_id=user_id, - upsert=jwt_handler.is_upsert_user_id(valid_user_email=valid_user_email), + upsert=( + jwt_handler.is_upsert_user_id(valid_user_email=valid_user_email) + if user_id_upsert is None + else user_id_upsert + ), ), team_id=team_id, ) @@ -2010,6 +2043,7 @@ class JWTAuthManager: user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None, proxy_logging_obj: ProxyLogging, + team_id_upsert: bool | None = None, ) -> None: """Attach team context from x-litellm-team-id to an admin result. @@ -2027,7 +2061,7 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, - team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert if team_id_upsert is None else team_id_upsert, ) except Exception as e: # Fall back to pre-PR admin behavior: honor the admin's @@ -2262,57 +2296,136 @@ class JWTAuthManager: request_headers: dict | None = None, request_method: str | None = None, ) -> JWTAuthBuilderResult: - """Main authentication and authorization builder""" - # Check if OIDC UserInfo endpoint is enabled, but fall back to standard - # JWT auth if the token itself is a well-formed JWT (3-part structure). - if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled and not jwt_handler.is_jwt(token=api_key): - verbose_proxy_logger.debug("OIDC UserInfo is enabled. Fetching user info from UserInfo endpoint.") - # Use the access token to fetch user info from OIDC UserInfo endpoint - jwt_valid_token: dict = await jwt_handler.get_oidc_userinfo(token=api_key) - else: - # Default behavior: decode and validate the JWT token - jwt_valid_token = await jwt_handler.auth_jwt(token=api_key) - - # Check custom validate - if jwt_handler.litellm_jwtauth.custom_validate: - if not jwt_handler.litellm_jwtauth.custom_validate(jwt_valid_token): - raise HTTPException( - status_code=403, - detail="Invalid JWT token", - ) - - # Check RBAC - rbac_role: Final = jwt_handler.get_rbac_role(token=jwt_valid_token) - await JWTAuthManager.check_rbac_role( - jwt_handler, - jwt_valid_token, - general_settings, - request_data, - route, - rbac_role, + return await JWTAuthManager.authorize_jwt( + api_key=api_key, + jwt_handler=jwt_handler, + request_data=request_data, + general_settings=general_settings, + route=route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + request_headers=request_headers, + request_method=request_method, + provisioning=_JWTProvisioning( + user_id_upsert=jwt_handler.litellm_jwtauth.user_id_upsert, + team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + ), ) + @staticmethod + async def authenticate_jwt(api_key: str, jwt_handler: JWTHandler) -> dict[str, object]: + claims: Final = ( + await jwt_handler.get_oidc_userinfo(token=api_key) + if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled and not jwt_handler.is_jwt(token=api_key) + else await jwt_handler.auth_jwt(token=api_key) + ) + validate: Final = jwt_handler.litellm_jwtauth.custom_validate + if validate is not None and not validate(claims): + raise HTTPException(status_code=403, detail="Invalid JWT token") + return claims + + @staticmethod + async def resolve_identity( + api_key: str, + jwt_handler: JWTHandler, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + ) -> JWTIdentity: + claims: Final = await JWTAuthManager.authenticate_jwt(api_key, jwt_handler) + return await JWTAuthManager._resolve_claim_identity( + claims, jwt_handler, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj + ) + + @staticmethod + async def _resolve_claim_identity( + claims: dict[str, object], + jwt_handler: JWTHandler, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + ) -> JWTIdentity: + claim_user_id, user_email, valid_user_email = await JWTAuthManager.get_user_info(jwt_handler, claims) + user_id: Final = ( + jwt_handler.get_object_id(token=claims, default_value=None) or claim_user_id + if jwt_handler.get_rbac_role(token=claims) == LitellmUserRoles.INTERNAL_USER + else claim_user_id + ) + agent_id: Final = JWTAuthManager.resolve_agent_id(jwt_handler, claims, jwt_handler.agent_lookup) + is_admin: Final = jwt_handler.is_admin(scopes=jwt_handler.get_scopes(token=claims)) + try: + user, _, _, _, canonical_id = await JWTAuthManager.get_objects( + user_id=user_id, + user_email=user_email, + org_id=None, + end_user_id=None, + team_id=None, + valid_user_email=valid_user_email, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + route="", + user_id_upsert=False, + ) + except UserNotFoundError: + if not is_admin: + raise + return JWTIdentity(user_id=user_id, user_object=None, agent_id=agent_id) + return JWTIdentity(user_id=user_id if is_admin else canonical_id, user_object=user, agent_id=agent_id) + + @staticmethod + async def authorize_jwt( + api_key: str, + jwt_handler: JWTHandler, + request_data: dict[str, object], + general_settings: dict[str, object], + route: str, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + request_headers: dict[str, str] | None = None, + request_method: str | None = None, + provisioning: _JWTProvisioning | None = None, + ) -> JWTAuthBuilderResult: + """Resolve and authorize JWT context; only normal admission supplies provisioning.""" + handler: Final = jwt_handler + jwt_valid_token: Final = await JWTAuthManager.authenticate_jwt(api_key, handler) + team_id_upsert: Final = provisioning.team_id_upsert if provisioning is not None else False + model: Final = request_data.get("model") + requested_model: Final = model if isinstance(model, str) else None + + # Check RBAC + rbac_role: Final = handler.get_rbac_role(token=jwt_valid_token) + await JWTAuthManager.check_rbac_role(handler, jwt_valid_token, general_settings, request_data, route, rbac_role) + # Check Scope Based Access - scopes: Final = jwt_handler.get_scopes(token=jwt_valid_token) - if jwt_handler.litellm_jwtauth.enforce_scope_based_access and jwt_handler.litellm_jwtauth.scope_mappings: + scopes: Final = handler.get_scopes(token=jwt_valid_token) + if handler.litellm_jwtauth.enforce_scope_based_access and handler.litellm_jwtauth.scope_mappings: JWTAuthManager.check_scope_based_access( - scope_mappings=jwt_handler.litellm_jwtauth.scope_mappings, + scope_mappings=handler.litellm_jwtauth.scope_mappings, scopes=scopes, request_data=request_data, general_settings=general_settings, ) - object_id = jwt_handler.get_object_id(token=jwt_valid_token, default_value=None) + object_id = handler.get_object_id(token=jwt_valid_token, default_value=None) # Get basic user info - user_id, user_email, valid_user_email = await JWTAuthManager.get_user_info(jwt_handler, jwt_valid_token) + user_id, user_email, valid_user_email = await JWTAuthManager.get_user_info(handler, jwt_valid_token) # Get IDs - org_id: Final = jwt_handler.get_org_id(token=jwt_valid_token, default_value=None) - end_user_id: Final = jwt_handler.get_end_user_id(token=jwt_valid_token, default_value=None) + org_id: Final = handler.get_org_id(token=jwt_valid_token, default_value=None) + end_user_id: Final = handler.get_end_user_id(token=jwt_valid_token, default_value=None) team_id: str | None = None team_object: LiteLLM_TeamTable | None = None - object_id = jwt_handler.get_object_id(token=jwt_valid_token, default_value=None) + object_id = handler.get_object_id(token=jwt_valid_token, default_value=None) if rbac_role and object_id: if rbac_role == LitellmUserRoles.TEAM: @@ -2321,14 +2434,14 @@ class JWTAuthManager: user_id = object_id agent_id: Final = JWTAuthManager.resolve_agent_id( - jwt_handler=jwt_handler, + jwt_handler=handler, jwt_valid_token=jwt_valid_token, - agent_registry=jwt_handler.agent_lookup, + agent_registry=handler.agent_lookup, ) # Check admin access admin_result: Final = await JWTAuthManager.check_admin_access( - jwt_handler, + handler, scopes, route, user_id, @@ -2343,18 +2456,24 @@ class JWTAuthManager: admin_result=admin_result, route=route, request_headers=request_headers, - jwt_handler=jwt_handler, + jwt_handler=handler, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, + team_id_upsert=team_id_upsert, ) + if provisioning is None: + identity: Final = await JWTAuthManager._resolve_claim_identity( + jwt_valid_token, handler, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj + ) + return {**admin_result, "user_object": identity.user_object} return admin_result # Get team with model access ## Check if team_id is specified via x-litellm-team-id header - all_team_ids: Final = JWTAuthManager.get_all_team_ids(jwt_handler, jwt_valid_token) - specific_team_id: Final = jwt_handler.get_team_id(token=jwt_valid_token, default_value=None) + all_team_ids: Final = JWTAuthManager.get_all_team_ids(handler, jwt_valid_token) + specific_team_id: Final = handler.get_team_id(token=jwt_valid_token, default_value=None) # The DB fallback only applies when the token carries no team identity at # all. `get_all_jwt_team_ids` ignores `team_id_default` so a configured @@ -2364,9 +2483,9 @@ class JWTAuthManager: # the RBAC team-role path (which already set `team_id`); otherwise a # provisional x-litellm-team-id header could override an RBAC-asserted team. db_team_fallback: Final = ( - jwt_handler.litellm_jwtauth.fallback_to_db_teams - and not jwt_handler.get_all_jwt_team_ids(token=jwt_valid_token) - and not jwt_handler.get_team_alias(token=jwt_valid_token, default_value=None) + handler.litellm_jwtauth.fallback_to_db_teams + and not handler.get_all_jwt_team_ids(token=jwt_valid_token) + and not handler.get_team_alias(token=jwt_valid_token, default_value=None) and team_id is None ) if specific_team_id and not db_team_fallback: @@ -2391,7 +2510,7 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, - team_id_upsert=(jwt_handler.litellm_jwtauth.team_id_upsert and not db_team_fallback), + team_id_upsert=(team_id_upsert and not db_team_fallback), ) except HTTPException: if not db_team_fallback: @@ -2403,22 +2522,23 @@ class JWTAuthManager: team_id, team_object, ) = await JWTAuthManager.find_and_validate_specific_team_id( - jwt_handler, + handler, jwt_valid_token, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj, + team_id_upsert=team_id_upsert, ) if not team_object and not team_id: ## CHECK USER GROUP ACCESS team_id, team_object = await JWTAuthManager.find_team_with_model_access( team_ids=all_team_ids, - requested_model=request_data.get("model"), + requested_model=requested_model, route=route, request_method=request_method, - jwt_handler=jwt_handler, + jwt_handler=handler, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, @@ -2442,7 +2562,7 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, - team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + team_id_upsert=team_id_upsert, ) if team_id and not JWTAuthManager._team_has_passthrough_route_access( @@ -2453,7 +2573,7 @@ class JWTAuthManager: JWTAuthManager._raise_team_passthrough_route_denial(route=route) # Extract alias fields for resolution (if configured) - org_alias: Final = jwt_handler.get_org_alias(token=jwt_valid_token, default_value=None) + org_alias: Final = handler.get_org_alias(token=jwt_valid_token, default_value=None) # get_objects returns effective_user_id for downstream spend attribution (GH #26789). ( @@ -2469,25 +2589,27 @@ class JWTAuthManager: end_user_id=end_user_id, team_id=team_id, valid_user_email=valid_user_email, - jwt_handler=jwt_handler, + jwt_handler=handler, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, route=route, org_alias=org_alias, + user_id_upsert=provisioning.user_id_upsert if provisioning is not None else False, ) # Derive org_id from org_object if resolved by alias resolved_org_id: Final = org_object.organization_id if org_object else org_id - await JWTAuthManager.sync_user_role_and_teams( - jwt_handler=jwt_handler, - jwt_valid_token=jwt_valid_token, - user_object=user_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - ) + if provisioning is not None: + await JWTAuthManager.sync_user_role_and_teams( + jwt_handler=handler, + jwt_valid_token=jwt_valid_token, + user_object=user_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) # If JWT did not resolve team_id, attempt a team fallback. if team_id is None and db_team_fallback: @@ -2498,11 +2620,11 @@ class JWTAuthManager: ) = await JWTAuthManager._resolve_db_team_fallback( user_object=user_object, user_id=user_id, - requested_model=request_data.get("model"), + requested_model=requested_model, route=route, - jwt_handler=jwt_handler, - enforce_team_based_model_access=jwt_handler.litellm_jwtauth.enforce_team_based_model_access, - team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + jwt_handler=handler, + enforce_team_based_model_access=handler.litellm_jwtauth.enforce_team_based_model_access, + team_id_upsert=team_id_upsert, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, @@ -2530,7 +2652,7 @@ class JWTAuthManager: user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, - team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert, + team_id_upsert=team_id_upsert, ) elif db_team_fallback and team_id == header_team_id: JWTAuthManager._validate_header_team_in_db_membership( @@ -2540,7 +2662,7 @@ class JWTAuthManager: if not JWTAuthManager._is_team_route_allowed( route=route, request_method=request_method, - jwt_handler=jwt_handler, + jwt_handler=handler, ): raise HTTPException( status_code=403, @@ -2550,16 +2672,17 @@ class JWTAuthManager: ) ## MAP USER TO TEAMS - await JWTAuthManager.map_user_to_teams( - user_object=user_object, - team_object=team_object, - ) + if provisioning is not None: + await JWTAuthManager.map_user_to_teams( + user_object=user_object, + team_object=team_object, + ) # Validate that a valid rbac id is returned for spend tracking JWTAuthManager.validate_object_id( user_id=user_id, team_id=team_id, - enforce_rbac=general_settings.get("enforce_rbac", False), + enforce_rbac=bool(general_settings.get("enforce_rbac", False)), is_proxy_admin=False, ) @@ -2582,3 +2705,38 @@ class JWTAuthManager: jwt_claims=jwt_valid_token, agent_id=agent_id, ) + + @staticmethod + def user_api_key_auth_from_result( + result: JWTAuthBuilderResult, + parent_otel_span: Span | None = None, + ) -> UserAPIKeyAuth: + """Keep JWT identity and permission attribution identical across consumers.""" + user: Final = result["user_object"] + admin: Final = result["is_proxy_admin"] + return UserAPIKeyAuth( + api_key=None, + user_role=( + LitellmUserRoles.PROXY_ADMIN + if admin + else LitellmUserRoles(user.user_role) + if user is not None and user.user_role is not None + else LitellmUserRoles.INTERNAL_USER + ), + user_id=result["user_id"], + user_email=result["user_email"], + team_id=result["team_id"], + org_id=result["org_id"], + end_user_id=result["end_user_id"], + parent_otel_span=parent_otel_span, + jwt_claims=result["jwt_claims"], + agent_id=result.get("agent_id"), + user_tpm_limit=user.tpm_limit if user is not None and not admin else None, + user_rpm_limit=user.rpm_limit if user is not None and not admin else None, + user_model_max_budget=user.model_max_budget if user is not None and not admin else None, + **team_grants( + team_object=result["team_object"], + team_membership=result.get("team_membership"), + user_id=result["user_id"], + ), + ) diff --git a/litellm/proxy/auth/model_access_denied.py b/litellm/proxy/auth/model_access_denied.py new file mode 100644 index 00000000000..ffb73b343cd --- /dev/null +++ b/litellm/proxy/auth/model_access_denied.py @@ -0,0 +1,18 @@ +from typing import Final + +from fastapi import HTTPException + +MODEL_ACCESS_DENIED_CLIENT_MESSAGE: Final = ( + "The requested model '{model}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) + + +def model_access_denied_client_message(model: str | list[str]) -> str: + return MODEL_ACCESS_DENIED_CLIENT_MESSAGE.format(model=model) + + +class ModelAccessDeniedHTTPException(HTTPException): + def __init__(self, internal_message: str, status_code: int, detail: str | dict[str, str]) -> None: + super().__init__(status_code=status_code, detail=detail) + self.internal_message: Final = internal_message diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index c5297ac83dc..6757c0c594d 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1669,13 +1669,11 @@ async def _user_api_key_auth_builder( is_proxy_admin: Final = result["is_proxy_admin"] team_id: Final = result["team_id"] - team_object: Final = result["team_object"] user_id: Final = result["user_id"] user_email: Final = result["user_email"] user_object: Final = result["user_object"] end_user_id = result["end_user_id"] org_id: Final = result["org_id"] - team_membership: Final[LiteLLM_TeamMembership | None] = result.get("team_membership", None) jwt_claims = result.get("jwt_claims", None) agent_id: Final[str | None] = result.get("agent_id") @@ -1693,40 +1691,9 @@ async def _user_api_key_auth_builder( value=_JWT_PROXY_ADMIN_SENTINEL, ttl=jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl, ) - return UserAPIKeyAuth( - api_key=None, - user_role=LitellmUserRoles.PROXY_ADMIN, - user_id=user_id, - user_email=user_email, - team_id=team_id, - org_id=org_id, - end_user_id=end_user_id, - parent_otel_span=parent_otel_span, - jwt_claims=jwt_claims, - agent_id=agent_id, - **team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id), - ) + return JWTAuthManager.user_api_key_auth_from_result(result, parent_otel_span) - valid_token = UserAPIKeyAuth( - api_key=None, - team_id=team_id, - user_role=( - LitellmUserRoles(user_object.user_role) - if user_object is not None and user_object.user_role is not None - else LitellmUserRoles.INTERNAL_USER - ), - user_id=user_id, - user_email=user_email, - org_id=org_id, - parent_otel_span=parent_otel_span, - end_user_id=end_user_id, - user_tpm_limit=(user_object.tpm_limit if user_object is not None else None), - user_rpm_limit=(user_object.rpm_limit if user_object is not None else None), - user_model_max_budget=(user_object.model_max_budget if user_object is not None else None), - jwt_claims=jwt_claims, - agent_id=agent_id, - **team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id), - ) + valid_token = JWTAuthManager.user_api_key_auth_from_result(result, parent_otel_span) # AUTO_REGISTER deferred from _resolve_jwt_to_virtual_key. # JWT policy (RBAC, scope, custom_validate, email-domain) @@ -2604,6 +2571,13 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc return PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() +def is_no_auth_dev_mode(master_key: str | None, general_settings: Mapping[str, object]) -> bool: + return master_key is None and not any( + general_settings.get(flag, False) + for flag in ("enable_jwt_auth", "enable_oauth2_auth", "enable_oauth2_proxy_auth") + ) + + @tracer.wrap() async def _run_centralized_common_checks( user_api_key_auth_obj: UserAPIKeyAuth, @@ -2663,11 +2637,7 @@ async def _run_centralized_common_checks( # Running common_checks would block every admin route on these # deployments where that was previously not the contract. If any # authn is enabled (JWT, OAuth2, OAuth2-proxy), authz must run. - if master_key is None and not ( - general_settings.get("enable_jwt_auth", False) - or general_settings.get("enable_oauth2_auth", False) - or general_settings.get("enable_oauth2_proxy_auth", False) - ): + if is_no_auth_dev_mode(master_key, general_settings): return if user_custom_auth is not None and not general_settings.get("custom_auth_run_common_checks", False): diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 1e684c514de..092e8eaafa1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -11,6 +11,7 @@ import os import re import time from collections.abc import AsyncGenerator, Coroutine, Mapping, Sequence +from dataclasses import dataclass, replace from datetime import datetime from re import Pattern from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, cast @@ -20,7 +21,11 @@ from fastapi import HTTPException from litellm import Router from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.constants import ( + CONTENT_FILTER_STREAMING_HOLDBACK_CHARS, + CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS, + DEFAULT_MAX_RECURSE_DEPTH, +) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ( @@ -61,6 +66,7 @@ from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 GAP_WORD_TOKENIZER: Final = re.compile(r"\b\w+\b") +SENTENCE_TERMINATORS: Final = re.compile(r"[.!?]+") WORD_NUMBER_MAP: Final = { @@ -112,6 +118,22 @@ class _CategoryConfigView(TypedDict): category_file: str | None +@dataclass(frozen=True, slots=True) +class _StreamedChoiceState: + buffered_text: str = "" + yielded_masked_text_len: int = 0 + committed_detections: tuple[ContentFilterDetection, ...] = () + latest_detections: tuple[ContentFilterDetection, ...] = () + next_trim_len: int = 0 + + +@dataclass(frozen=True, slots=True) +class _StreamedScanPlan: + context_chars: int + exception_phrases: tuple[str, ...] + conditional_words: tuple[str, ...] + + class CategoryFileData(TypedDict, total=False): category_name: str description: str @@ -976,7 +998,7 @@ class ContentFilterGuardrail(CustomGuardrail): # Split text into sentences for more precise matching # Simple sentence splitting on common terminators - sentences: Final = re.split(r"[.!?]+", text) + sentences: Final = SENTENCE_TERMINATORS.split(text) for category_name, config in self.conditional_categories.items(): identifier_words = config["identifier_words"] @@ -1950,6 +1972,81 @@ class ContentFilterGuardrail(CustomGuardrail): exception_str=exception_str, ) + def _streamed_scan_plan(self) -> _StreamedScanPlan: + """ + Per-stream inputs for buffer trimming: the retained tail length (the default + context, widened to the longest configured keyword), the category exception + phrases, which suppress matches anywhere in the scanned text, and the conditional + category words, which only match when paired inside one sentence. + """ + longest_keyword: Final = max( + map(len, (*self.blocked_words, *self.category_keywords, *self.always_block_category_keywords)), + default=0, + ) + return _StreamedScanPlan( + context_chars=max(CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS, longest_keyword), + exception_phrases=tuple( + phrase for category in self.loaded_categories.values() for phrase in category.exceptions + ), + conditional_words=tuple( + word + for config in self.conditional_categories.values() + for word in (*config["identifier_words"], *config["block_words"]) + ), + ) + + @staticmethod + def _cut_breaks_wider_context(buffered_text: str, head: str, tail: str, plan: _StreamedScanPlan) -> bool: + buffered_lower: Final = buffered_text.lower() + tail_lower: Final = tail.lower() + if any(phrase in buffered_lower and phrase not in tail_lower for phrase in plan.exception_phrases): + return True + cut_sentence: Final = ( + SENTENCE_TERMINATORS.split(head.lower())[-1] + SENTENCE_TERMINATORS.split(tail_lower, maxsplit=1)[0] + ) + return any(word in cut_sentence for word in plan.conditional_words) + + def _trim_streamed_choice_buffer( + self, state: _StreamedChoiceState, masked_text: str, plan: _StreamedScanPlan + ) -> _StreamedChoiceState: + """ + Bound the per-choice buffer rescanned on every streamed chunk. + + Once the buffer exceeds twice the scan context, drop everything but the last + context-sized tail, provided no exception phrase or unfinished conditional sentence + would leave the buffer, the two halves mask to the same output as the whole (so no + match or phrase straddles the cut), and the dropped prefix has already been yielded. + Otherwise keep the buffer and retry once it has grown by another context length. + + Detections found in the dropped prefix move to the state's committed detections. + """ + if len(state.buffered_text) <= max(2 * plan.context_chars, state.next_trim_len): + return state + deferred: Final = replace(state, next_trim_len=len(state.buffered_text) + plan.context_chars) + head: Final = state.buffered_text[: -plan.context_chars] + tail: Final = state.buffered_text[-plan.context_chars :] + if self._cut_breaks_wider_context(state.buffered_text, head, tail, plan): + return deferred + head_detections: Final[list[ContentFilterDetection]] = [] # mutable-ok: filled by _filter_single_text + try: + masked_head: Final = self._filter_single_text(head, detections=head_detections) + masked_tail: Final = self._filter_single_text(tail) + except Exception: + return deferred + if masked_head + masked_tail != masked_text or len(masked_head) > state.yielded_masked_text_len: + return deferred + return replace( + state, + buffered_text=tail, + yielded_masked_text_len=state.yielded_masked_text_len - len(masked_head), + committed_detections=state.committed_detections + tuple(head_detections), + next_trim_len=0, + ) + + @staticmethod + def _merge_detections(detections: Sequence[ContentFilterDetection]) -> tuple[ContentFilterDetection, ...]: + return tuple(detection for index, detection in enumerate(detections) if detection not in detections[:index]) + async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -1968,10 +2065,8 @@ class ContentFilterGuardrail(CustomGuardrail): and the UI Request Lifecycle panel. Mirrors apply_guardrail's finally-block contract. """ - accumulated_text_by_choice: Final[dict[int, str]] = {} - yielded_masked_text_len_by_choice: Final[dict[int, int]] = {} - latest_detections_by_choice: Final[dict[int, list[ContentFilterDetection]]] = {} - buffer_size: Final = 50 # Increased buffer to catch patterns split across many chunks + state_by_choice: Final[dict[int, _StreamedChoiceState]] = {} + plan: Final = self._streamed_scan_plan() start_time: Final = datetime.now() scan_seconds: float = 0.0 # rebind-ok: accumulates per-chunk scan time across the stream @@ -1997,69 +2092,60 @@ class ContentFilterGuardrail(CustomGuardrail): content = getattr(choice.delta, "content", None) is_final = bool(getattr(choice, "finish_reason", None)) - if isinstance(content, str) and content: - accumulated_text_by_choice[choice_index] = ( - accumulated_text_by_choice.get(choice_index, "") + content - ) - elif not is_final: + new_content = content if isinstance(content, str) else "" + if not new_content and not is_final: continue - text_to_check = accumulated_text_by_choice.get(choice_index, "") - if not text_to_check: + previous_state = state_by_choice.get(choice_index, _StreamedChoiceState()) + buffered_text = previous_state.buffered_text + new_content + if not buffered_text: continue # Add a space at the end if it's the final chunk to trigger word boundaries (\b) - text_to_scan = text_to_check + (" " if is_final else "") + text_to_scan = buffered_text + (" " if is_final else "") choice_detections: list[ContentFilterDetection] = [] scan_started = time.perf_counter() try: - # _filter_single_text scans the whole accumulated - # choice buffer every chunk, so previous-chunk - # matches are guaranteed to be re-found. Keeping - # only each choice's latest scan avoids duplicate - # detections in the final log row. masked_text = self._filter_single_text(text_to_scan, detections=choice_detections) if is_final and masked_text.endswith(" "): masked_text = masked_text[:-1] - latest_detections_by_choice[choice_index] = choice_detections + latest_detections = tuple(choice_detections) except HTTPException: - latest_detections_by_choice[choice_index] = choice_detections + state_by_choice[choice_index] = replace( + previous_state, latest_detections=tuple(choice_detections) + ) raise except Exception as e: verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e) masked_text = text_to_scan # Fallback to current text + latest_detections = previous_state.latest_detections finally: scan_seconds += time.perf_counter() - scan_started - # Determine how much can be safely yielded + safe_to_yield_len = max( + previous_state.yielded_masked_text_len, + len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS), + ) + choice.delta.content = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len] + next_state = replace( + previous_state, + buffered_text=buffered_text, + yielded_masked_text_len=safe_to_yield_len, + latest_detections=latest_detections, + ) if is_final: - safe_to_yield_len = len(masked_text) - else: - safe_to_yield_len = max(0, len(masked_text) - buffer_size) + state_by_choice[choice_index] = next_state + continue - yielded_masked_text_len = yielded_masked_text_len_by_choice.get(choice_index, 0) - if safe_to_yield_len > yielded_masked_text_len: - new_masked_content = masked_text[yielded_masked_text_len:safe_to_yield_len] - choice.delta.content = new_masked_content - yielded_masked_text_len_by_choice[choice_index] = safe_to_yield_len - else: - # Hold content by yielding empty content on this choice - # while preserving chunk metadata and other choices. - choice.delta.content = "" + trim_started = time.perf_counter() + state_by_choice[choice_index] = self._trim_streamed_choice_buffer(next_state, masked_text, plan) + scan_seconds += time.perf_counter() - trim_started yield item else: # Not a ModelResponseStream or no choices - yield as is yield item - - # Any remaining content (should have been handled by is_final, but just in case) - if any( - yielded_masked_text_len_by_choice.get(choice_index, 0) < len(accumulated_text) - for choice_index, accumulated_text in accumulated_text_by_choice.items() - ): - # We already reached the end of the generator - pass except HTTPException: status = "guardrail_intervened" raise @@ -2070,8 +2156,8 @@ class ContentFilterGuardrail(CustomGuardrail): finally: detections = [ detection - for choice_detections in latest_detections_by_choice.values() - for detection in choice_detections + for state in state_by_choice.values() + for detection in self._merge_detections((*state.committed_detections, *state.latest_detections)) ] self._count_masked_entities(detections, masked_entity_count) self._log_guardrail_information( diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 7858adeb55d..b7ab215a2cd 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -87,12 +87,40 @@ def initialize_lakera_v2(litellm_params: LitellmParams, guardrail: Guardrail): return _lakera_v2_callback +_MCP_EVENT_HOOKS: Final = frozenset( + { + GuardrailEventHooks.pre_mcp_call.value, + GuardrailEventHooks.during_mcp_call.value, + GuardrailEventHooks.post_mcp_call.value, + } +) + + +def _configured_event_hooks(mode: str | list[str] | Mode) -> tuple[str, ...]: + if isinstance(mode, str): + return (mode,) + if isinstance(mode, list): + return tuple(mode) + return tuple( + hook + for value in (*mode.tags.values(), mode.default) + if value is not None + for hook in ((value,) if isinstance(value, str) else value) + ) + + +def _is_mcp_only_mode(mode: str | list[str] | Mode) -> bool: + hooks: Final = _configured_event_hooks(mode) + return bool(hooks) and all(hook in _MCP_EVENT_HOOKS for hook in hooks) + + def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> tuple[CustomGuardrail, ...]: from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, ) - filter_scope: Final = getattr(litellm_params, "presidio_filter_scope", None) or "both" + explicit_filter_scope: Final = getattr(litellm_params, "presidio_filter_scope", None) + filter_scope: Final = explicit_filter_scope or ("input" if _is_mcp_only_mode(litellm_params.mode) else "both") run_input: Final = filter_scope in ("input", "both") run_output: Final = filter_scope in ("output", "both") diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 5cfef11df8d..f75197532b4 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -21,6 +21,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.utils import _hash_token_if_needed +from litellm.secret_managers.base_secret_manager import BaseSecretManager # NOTE: This is the prefix for all virtual keys stored in AWS Secrets Manager LITELLM_PREFIX_STORED_VIRTUAL_KEYS: Final = "litellm/" @@ -100,6 +101,7 @@ class KeyManagementEventHooks: Post /key/update processing hook Handles the following: + - Renaming the key's secret in the secret manager when the alias changes - Storing Audit Logs for key update """ from litellm.proxy.management_helpers.audit_logs import ( @@ -109,6 +111,16 @@ class KeyManagementEventHooks: ) from litellm.proxy.proxy_server import litellm_proxy_admin_name + if data.key_alias is not None and data.key_alias != existing_key_row.key_alias: + try: + await KeyManagementEventHooks._rename_virtual_key_in_secret_manager( + current_secret_name=existing_key_row.key_alias or f"virtual-key-{existing_key_row.token}", + new_secret_name=data.key_alias, + team_id=existing_key_row.team_id, + ) + except Exception as e: + verbose_proxy_logger.warning("Failed to rename virtual key in secret manager: %s", e) + if is_audit_logging_enabled(): updated_fields: Final = { **data.model_dump(exclude_none=True), @@ -153,10 +165,11 @@ class KeyManagementEventHooks: from litellm.proxy.proxy_server import litellm_proxy_admin_name # Store the generated key in the secret manager - non-blocking, independent operation - if data is not None and response.token_id is not None: + if response.token_id is not None: try: initial_secret_name: Final = existing_key_row.key_alias or f"virtual-key-{existing_key_row.token}" - new_secret_name: Final = response.key_alias or data.key_alias or initial_secret_name + requested_alias: Final = data.key_alias if data is not None else None + new_secret_name: Final = response.key_alias or requested_alias or initial_secret_name verbose_proxy_logger.info( "Updating secret in secret manager: secret_name=%s", new_secret_name, @@ -305,21 +318,66 @@ class KeyManagementEventHooks: new_secret_value: New value of the virtual key (example: sk-1234) team_id: Optional team ID to get team-specific secret manager settings """ - if litellm._key_management_settings is not None: - if litellm._key_management_settings.store_virtual_keys is True: - from litellm.secret_managers.base_secret_manager import ( - BaseSecretManager, - ) + secret_manager: Final = KeyManagementEventHooks._stored_virtual_key_secret_manager() + if secret_manager is None: + return + optional_params: Final = await KeyManagementEventHooks._get_secret_manager_optional_params(team_id) + await secret_manager.async_rotate_secret( + current_secret_name=KeyManagementEventHooks._get_secret_name(current_secret_name), + new_secret_name=KeyManagementEventHooks._get_secret_name(new_secret_name), + new_secret_value=new_secret_value, + optional_params=optional_params, + ) - # store the key in the secret manager - if isinstance(litellm.secret_manager_client, BaseSecretManager): - optional_params: Final = await KeyManagementEventHooks._get_secret_manager_optional_params(team_id) - await litellm.secret_manager_client.async_rotate_secret( - current_secret_name=KeyManagementEventHooks._get_secret_name(current_secret_name), - new_secret_name=KeyManagementEventHooks._get_secret_name(new_secret_name), - new_secret_value=new_secret_value, - optional_params=optional_params, - ) + @staticmethod + def _stored_virtual_key_secret_manager() -> BaseSecretManager | None: + """ + The secret manager client that stores virtual keys, or None when virtual keys are not stored in one + """ + if litellm._key_management_settings is None or litellm._key_management_settings.store_virtual_keys is not True: + return None + if not isinstance(litellm.secret_manager_client, BaseSecretManager): + return None + return litellm.secret_manager_client + + @staticmethod + async def _rename_virtual_key_in_secret_manager( + current_secret_name: str, + new_secret_name: str, + team_id: str | None = None, + ) -> None: + """ + Move a virtual key to a new secret name, keeping its current value + + Args: + current_secret_name: Current name of the virtual key + new_secret_name: New name of the virtual key + team_id: Optional team ID to get team-specific secret manager settings + """ + secret_manager: Final = KeyManagementEventHooks._stored_virtual_key_secret_manager() + if secret_manager is None: + return + optional_params: Final = await KeyManagementEventHooks._get_secret_manager_optional_params(team_id) + current_secret_value: Final = await secret_manager.async_read_secret( + secret_name=KeyManagementEventHooks._get_secret_name(current_secret_name), + optional_params=optional_params, + ) + if current_secret_value is None: + verbose_proxy_logger.warning( + "Secret %s not found in secret manager, skipping rename to %s", current_secret_name, new_secret_name + ) + return + verbose_proxy_logger.info( + "Renaming secret in secret manager: current_secret_name=%s new_secret_name=%s", + current_secret_name, + new_secret_name, + ) + await secret_manager.async_rotate_secret( + current_secret_name=KeyManagementEventHooks._get_secret_name(current_secret_name), + new_secret_name=KeyManagementEventHooks._get_secret_name(new_secret_name), + new_secret_value=current_secret_value, + optional_params=optional_params, + ) @staticmethod def _get_secret_name(secret_name: str) -> str: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 563db811edc..18f4280c01c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -108,6 +108,37 @@ def _trace_id_from_traceparent(traceparent: str) -> str | None: return trace_id if trace_id != "0" * 32 else None +def _trace_id_from_otel_span(span: "OtelSpan | None") -> str | None: + if span is None: + return None + try: + span_context: Final = span.get_span_context() + is_valid: Final = span_context.is_valid + trace_id: Final = span_context.trace_id + except AttributeError: + return None + if not is_valid or not isinstance(trace_id, int): + return None + return format(trace_id, "032x") + + +def add_otel_trace_id_to_request( + data: dict[str, object], _metadata_variable_name: str, parent_otel_span: "OtelSpan | None" +) -> None: + if data.get("litellm_trace_id"): + return + metadata: Final = data.get(_metadata_variable_name) + requester_metadata: Final = data.get("metadata") + if any(isinstance(m, dict) and m.get("trace_id") for m in (metadata, requester_metadata)): + return + trace_id: Final = _trace_id_from_otel_span(parent_otel_span) + if trace_id is None: + return + data["litellm_trace_id"] = trace_id # rebind-ok: data is an out-param + if isinstance(metadata, dict): + metadata["trace_id"] = trace_id # rebind-ok: metadata is the request's own out-param dict + + def _session_id_from_baggage(baggage: str) -> str | None: """Extract a session.id entry from a W3C Baggage header (https://www.w3.org/TR/baggage/), e.g. "session.id=abc-123,user.id=42".""" @@ -173,6 +204,8 @@ _ENABLE_TEAM_STALE_ALIAS_BYPASS: bool | None = None if TYPE_CHECKING: + from opentelemetry.trace import Span as OtelSpan + from litellm.integrations.otel.model.destination import OtelDestination from litellm.proxy.policy_engine.attachment_registry import AttachmentRegistry from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig @@ -2042,6 +2075,13 @@ async def add_litellm_data_to_request( data=data, _metadata_variable_name=_metadata_variable_name, ) + add_otel_trace_id_to_request( + data=data, + _metadata_variable_name=_metadata_variable_name, + parent_otel_span=user_api_key_dict.parent_otel_span + if user_api_key_dict.parent_otel_span is not None + else getattr(request.state, "parent_otel_span", None), + ) apply_missing_session_id_policy( data=data, _metadata_variable_name=_metadata_variable_name, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 4c97bbaf5de..918a55bb9ce 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -170,6 +170,7 @@ if MCP_AVAILABLE: from litellm.proxy._experimental.mcp_server.ui_session_utils import ( admitted_user_context, build_effective_auth_contexts, + can_access_mcp_server, is_ui_session_credential, ) from litellm.proxy._types import ( @@ -2483,10 +2484,11 @@ if MCP_AVAILABLE: ) return server - allowed_server_ids: Final[set[str]] = set() - for auth_context in await build_effective_auth_contexts(user_api_key_dict): - allowed_server_ids.update(await global_mcp_server_manager.get_allowed_mcp_servers(auth_context)) - if server is None or server.server_id not in allowed_server_ids: + if server is None or not await can_access_mcp_server( + user_api_key_dict, + server.server_id, + global_mcp_server_manager.get_allowed_mcp_servers, + ): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail={ diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 955e6a8002b..0fe9d1cc626 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -45,6 +45,7 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( _get_bearer_token, + is_no_auth_dev_mode, user_api_key_auth, user_api_key_auth_websocket, ) @@ -709,8 +710,7 @@ async def anthropic_proxy_route( endpoint_func: Final = create_pass_through_route( endpoint=endpoint, target=str(updated_url), - custom_headers=auth_header if auth_header is not None else {}, - _forward_headers=True, + custom_headers=_upstream_headers_for_anthropic_route(request, user_api_key_dict, auth_header), is_streaming_request=is_streaming_request, ) # dynamically construct pass-through endpoint based on incoming path received_value: Final = await endpoint_func( @@ -1989,6 +1989,19 @@ _HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"} SpecialHeaders.litellm_credential_header_names() - _VERTEX_UPSTREAM_CREDENTIAL_HEADERS ) +_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL: Final = ( + "No Anthropic credential is configured on this proxy and the request carried no upstream " + "Anthropic credential. The LiteLLM virtual key is not forwarded to Anthropic. Configure an " + "Anthropic credential (ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN, or a model with " + "use_in_pass_through: true), or send your own Anthropic API key in the x-api-key header or " + "your own Anthropic OAuth token in the Authorization header." +) + +_ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS: Final = frozenset({"authorization", "x-api-key"}) +_HEADERS_NEVER_FORWARDED_TO_ANTHROPIC: Final = frozenset({"content-length", "host", "accept-encoding"}) | ( + SpecialHeaders.litellm_credential_header_names() - _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS +) + _MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key" @@ -2026,8 +2039,11 @@ def _is_authenticated_caller_jwt(value: str, jwt_claims: Mapping[str, object]) - def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool: - """Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``.""" - from litellm.proxy.proxy_server import master_key + """Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``. + + A proxy in no-auth dev mode without custom auth authenticated nothing, so none of the caller's values is one. + """ + from litellm.proxy.proxy_server import general_settings, master_key, user_custom_auth normalized: Final = _normalize_credential_value(value) if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): @@ -2035,35 +2051,54 @@ def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAut jwt_claims: Final = user_api_key_dict.jwt_claims if jwt_claims and _is_authenticated_caller_jwt(normalized, jwt_claims): return True + if is_no_auth_dev_mode(master_key, general_settings) and user_custom_auth is None: + return False authenticated_key: Final = user_api_key_dict.api_key if authenticated_key is None: return False - if master_key is None and not normalized.startswith("sk-"): - return False stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode()) +def _caller_headers_without_litellm_secrets( + request: Request, user_api_key_dict: UserAPIKeyAuth, never_forwarded: frozenset[str] +) -> Mapping[str, str]: + incoming: Final = _safe_get_request_headers(request) + dropped_by_name: Final = never_forwarded.union( + (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) + ) + return MappingProxyType( + { + name: value + for name, value in incoming.items() + if name not in dropped_by_name and not _is_authenticated_caller_secret(value, user_api_key_dict) + } + ) + + def _forwarded_headers_for_credentialless_vertex_passthrough( request: Request, user_api_key_dict: UserAPIKeyAuth ) -> Mapping[str, str]: """Caller headers to forward on the bring-your-own-credentials Vertex branch, minus LiteLLM secrets.""" - incoming: Final = _safe_get_request_headers(request) - never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union( - (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) + forwarded: Final = _caller_headers_without_litellm_secrets( + request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_VERTEX ) - forwarded: Final = MappingProxyType( - { - name: value - for name, value in incoming.items() - if name not in never_forwarded and not _is_authenticated_caller_secret(value, user_api_key_dict) - } - ) - if "authorization" not in forwarded and "x-goog-api-key" not in forwarded: + if _VERTEX_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(forwarded): raise HTTPException(status_code=401, detail=_CREDENTIALLESS_VERTEX_MISSING_CREDENTIAL_DETAIL) return forwarded +def _upstream_headers_for_anthropic_route( + request: Request, user_api_key_dict: UserAPIKeyAuth, proxy_auth_header: Mapping[str, str] | None +) -> Mapping[str, str]: + caller_headers: Final = _caller_headers_without_litellm_secrets( + request, user_api_key_dict, _HEADERS_NEVER_FORWARDED_TO_ANTHROPIC + ) + if proxy_auth_header is None and _ANTHROPIC_UPSTREAM_CREDENTIAL_HEADERS.isdisjoint(caller_headers): + raise HTTPException(status_code=401, detail=_CREDENTIALLESS_ANTHROPIC_MISSING_CREDENTIAL_DETAIL) + return MappingProxyType({**caller_headers, **(proxy_auth_header or {})}) + + async def _prepare_vertex_auth_headers( request: Request, vertex_credentials: VertexPassThroughCredentials | None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d7964556531..23bb8b6225b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -106,6 +106,7 @@ from litellm.proxy._types import ( LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, LitellmUserRoles, + ModelAccessDeniedProxyException, PassThroughGenericEndpoint, ProxyErrorTypes, ProxyException, @@ -1668,6 +1669,7 @@ class UserAPIKeyCacheTTLEnum(enum.Enum): @app.exception_handler(ProxyException) async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions + _log_model_access_denial(exc) headers: Final = exc.headers error_dict: Final = exc.to_dict() status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR @@ -1679,6 +1681,12 @@ async def openai_exception_handler(request: Request, exc: ProxyException): ) +def _log_model_access_denial(exc: ProxyException) -> None: + if not isinstance(exc, ModelAccessDeniedProxyException): + return + verbose_proxy_logger.warning(exc.sanitized_internal_message()) + + def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Exception | None = None) -> None: parent_otel_span: Final[_Span | None] = getattr(request.state, "parent_otel_span", None) if parent_otel_span is None: @@ -11978,6 +11986,7 @@ async def realtime_websocket_endpoint( llm_router=llm_router, ) except ProxyException as e: + _log_model_access_denial(e) await _reject_realtime_session(websocket, user_api_key_dict, code=1008, reason=e.message[:120]) return await websocket.accept(**accept_kwargs) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index d6a402e1860..c09f9c755ed 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -69,6 +69,11 @@ def _response_attr(source: object, name: str) -> object: return getattr(source, name, None) +def _upstream_status_code(error: Exception) -> int: + code: Final = getattr(error, "status_code", None) + return code if isinstance(code, int) else 500 + + def _raise_vector_store_scan_depth_exceeded() -> None: raise HTTPException( status_code=400, @@ -814,6 +819,6 @@ async def rag_query( except Exception as e: verbose_proxy_logger.exception("RAG Query failed: %s", e) raise HTTPException( - status_code=500, + status_code=_upstream_status_code(e), detail={"error": str(e)}, ) diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 1f63152632e..1a5301f0579 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -245,6 +245,9 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store + top_level_filters: Final = kwargs.pop("filters", None) + filters: Final = retrieval_config.get("retrieval_filter") or retrieval_config.get("filters") or top_level_filters + filter_search_params: Final = MappingProxyType({"filters": filters} if filters else {}) # Forward allowlisted provider retrieval_config extras (region, embedding # model, bucket, credential refs) to the search call; the managed store's # params win on conflict. @@ -258,7 +261,9 @@ async def _execute_query_pipeline( if k not in _SEARCH_ARGS_SET_BY_PIPELINE } ) - forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs, **store_search_params}) + forwarded_search_params: Final = MappingProxyType( + {**provider_search_params, **kwargs, **filter_search_params, **store_search_params} + ) with _suppressed_sub_call_billing(): search_response: Final = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c28b5558c75..1b9f39449cf 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -176,6 +176,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return {**item_kwargs, "name": tool_name, **namespace_kwargs} def _is_reasoning_end(self, chunk): + if not chunk.choices: + return False delta: Final = chunk.choices[0].delta # if this indicates reasoning content, don't consider reasoning ended @@ -897,6 +899,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Change: Never return a value, just enqueue output item events if self.sent_output_item_added_event: return + if not chunk.choices: + return delta: Final = chunk.choices[0].delta self._sequence_number += 1 @@ -1224,6 +1228,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): It's unclear how users expect litellm to translate multiple-choices-per-chunk to the responses API output. """ + if not choices: + return "" choice: Final = choices[0] chat_completion_delta: Final[ChatCompletionDelta] = choice.delta return chat_completion_delta.content or "" diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 38874768ca8..8d766cf1cd0 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -32,6 +32,9 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.openai import ( @@ -257,6 +260,7 @@ class BaseResponsesAPIStreamingIterator: self._failure_handled = False # Track if failure handler has been called self._yielded_first_chunk = False self._generated_content = "" + self._generated_tool_arguments = "" self._completed_response_cached = False self._completed_response_logged = False self._completed_response_cache_hit: bool | None = None @@ -352,6 +356,10 @@ class BaseResponsesAPIStreamingIterator: _delta: Final = getattr(openai_responses_api_chunk, "delta", None) if isinstance(_delta, str): self._generated_content += _delta + elif _event_type in _TOOL_ARGUMENTS_DELTA_EVENTS: + _args_delta: Final = getattr(openai_responses_api_chunk, "delta", None) + if isinstance(_args_delta, str): + self._generated_tool_arguments += _args_delta _stream_model_id: Final = _model_id_from_metadata(self.litellm_metadata) if _event_type in ( ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, @@ -419,14 +427,41 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): - self.completed_response = openai_responses_api_chunk - _stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj) + _response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None) + _estimate_wanted: Final[bool] = _chunk_type in ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ) + _billed_response: Final[ResponsesAPIResponse | None] = _billed_terminal_response( + _response_obj, + ( + lambda: ( + _estimate_usage_safely( + self.model or "", + self.request_data.get("input"), + self.request_data, + self._generated_content + self._generated_tool_arguments, + ) + if _estimate_wanted + else None + ) + ), + ) + _terminal_chunk: Final = ( + openai_responses_api_chunk + if _billed_response is None or _billed_response is _response_obj + else openai_responses_api_chunk.model_copy(update={"response": _billed_response}) + ) + self.completed_response = _terminal_chunk + _stamp_responses_usage_cost(_billed_response, self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() else: self._handle_logging_completed_response() + return _terminal_chunk + return openai_responses_api_chunk return None @@ -655,7 +690,9 @@ class BaseResponsesAPIStreamingIterator: if cache is None: return - cached_response: Final = response_obj.model_dump_json() + cached_response: Final = _dump_json_safely(response_obj) + if cached_response is None: + return if is_async: from litellm.caching.caching_handler import create_cache_write_task @@ -1301,6 +1338,31 @@ def _add_text_like_part_events( ) +def _billed_terminal_response( + response_obj: object, estimate: Callable[[], ResponseAPIUsage | None] | None +) -> ResponsesAPIResponse | None: + if isinstance(response_obj, ResponsesAPIResponse): + return ( + response_obj + if response_obj.usage is not None or estimate is None + else response_obj.model_copy(update={"usage": estimate()}) + ) + if not isinstance(response_obj, dict): + return None + usage: Final[object] = response_obj.get("usage") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # a model_constructed terminal event leaves response as an untyped dict + return ResponsesAPIResponse.model_construct( + **{**response_obj, "usage": usage if usage is not None or estimate is None else estimate()} # pyright: ignore[reportUnknownArgumentType, reportArgumentType] # same untyped dict spread + ) + + +def _dump_json_safely(response: BaseModel) -> str | None: + try: + return response.model_dump_json() + except Exception as exc: + verbose_logger.debug("could not serialize completed response for cache: %s", exc) + return None + + def _logging_copy(event: object) -> object: """Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the @@ -1332,6 +1394,56 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None: return None +_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset( + { + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, + ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA, + ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA, + } +) + + +def _estimate_usage_from_text( + model: str, + request_input: object, + responses_api_request: Mapping[str, object], + generated_text: str, +) -> ResponseAPIUsage: + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped + input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union + responses_api_request=dict(responses_api_request), + ) + input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped + model=model, messages=messages + ) + output_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped + model=model, text=generated_text, count_response_tokens=True + ) + return ResponseAPIUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + ) + + +def _estimate_usage_safely( + model: str, + request_input: object, + responses_api_request: Mapping[str, object], + generated_text: str, +) -> ResponseAPIUsage | None: + try: + return _estimate_usage_from_text( + model=model, + request_input=request_input, + responses_api_request=responses_api_request, + generated_text=generated_text, + ) + except Exception as e: + verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e) + return None + + def _stamp_responses_usage_cost( response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None ) -> None: diff --git a/litellm/router.py b/litellm/router.py index d531072530b..789fc81d8d3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -425,12 +425,34 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () +def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: + if silent_model is None: + return () + try: + targets: Final = _SILENT_MODEL_ADAPTER.validate_python(silent_model) + except ValidationError: + verbose_router_logger.warning( + "silent_model must be a model name or a list of model names, got %r; skipping shadow traffic", + silent_model, + ) + return () + return (targets,) if isinstance(targets, str) else tuple(targets) + + +def _silent_experiment_kwargs_snapshot(kwargs: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = kwargs.get("metadata") + if not isinstance(metadata, Mapping): + return MappingProxyType({**kwargs}) + return MappingProxyType({**kwargs, "metadata": dict(metadata)}) + + def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]: """ Realtime client-secret requests carry the model inside ``session`` as well, and the caller's copy of it still @@ -2455,18 +2477,17 @@ class Router: ) silent_model: Final = litellm_params.pop("silent_model", None) - if silent_model is not None: + for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model # Use threading.Thread (not ThreadPoolExecutor) - executor.submit() # requires pickling args, which fails when kwargs contain unpicklable # objects (e.g. _thread.RLock from OTEL spans, loggers) in deployment. - thread: Final = threading.Thread( + threading.Thread( target=self._silent_experiment_completion, - args=(silent_model, messages), - kwargs=kwargs, + args=(silent_target, messages), + kwargs=_silent_experiment_kwargs_snapshot(kwargs), daemon=True, - ) - thread.start() + ).start() kwargs.setdefault("messages", messages) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) @@ -2567,9 +2588,6 @@ class Router: silent_kwargs["metadata"]["is_silent_experiment"] = True - # Force stream=False so the response is fully consumed and callbacks fire - silent_kwargs["stream"] = False - # Pop logging objects and call IDs to ensure a fresh logging context # This prevents collisions in the Proxy's database (spend_logs) silent_kwargs.pop("litellm_call_id", None) @@ -2579,6 +2597,23 @@ class Router: return silent_kwargs + async def _run_silent_experiment( + self, silent_model: str, messages: Sequence[Mapping[str, str]], silent_kwargs: Mapping[str, object] + ) -> None: + remaining_kwargs: Final = MappingProxyType( + {key: value for key, value in silent_kwargs.items() if key != "stream"} + ) + response: Final = await self.acompletion( + model=silent_model, + messages=cast(list[AllMessageValues], messages), + stream=bool(silent_kwargs.get("stream", False)), + **remaining_kwargs, + ) + if not isinstance(response, CustomStreamWrapper): + return + async for _ in response: + pass + def _silent_experiment_completion(self, silent_model: str, messages: Sequence[Mapping[str, str]], **kwargs): """ Run a silent experiment in the background (thread). @@ -2604,11 +2639,7 @@ class Router: try: async def _run_silent_completion(): - await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - **silent_kwargs, - ) + await self._run_silent_experiment(silent_model, messages, silent_kwargs) # Drain any fire-and-forget tasks (e.g. alerting hooks) # scheduled via asyncio.create_task during acompletion. pending: Final = asyncio.all_tasks() @@ -3500,11 +3531,7 @@ class Router: silent_kwargs["metadata"]["model_group"] = silent_model # Trigger the silent request - await self.acompletion( - model=silent_model, - messages=cast(list[AllMessageValues], messages), - **silent_kwargs, - ) + await self._run_silent_experiment(silent_model, messages, silent_kwargs) except Exception as e: verbose_router_logger.error("Silent experiment failed for model %s: %s", silent_model, e) @@ -3563,14 +3590,14 @@ class Router: ) silent_model: Final = litellm_params.pop("silent_model", None) - if silent_model is not None: + for silent_target in _silent_experiment_targets(silent_model): # Mirroring traffic to a secondary model # This is a silent experiment, so we don't want to block the primary request asyncio.create_task( self._silent_experiment_acompletion( - silent_model=silent_model, + silent_model=silent_target, messages=messages, # Use messages instead of *args - **kwargs, + **_silent_experiment_kwargs_snapshot(kwargs), ) ) diff --git a/litellm/types/integrations/rag/bedrock_knowledgebase.py b/litellm/types/integrations/rag/bedrock_knowledgebase.py index e3aba85ed9b..7156d8101e1 100644 --- a/litellm/types/integrations/rag/bedrock_knowledgebase.py +++ b/litellm/types/integrations/rag/bedrock_knowledgebase.py @@ -1,6 +1,6 @@ from typing import Any, Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class BedrockKBLocation(TypedDict, total=False): @@ -127,6 +127,10 @@ class BedrockKBGuardrailConfiguration(TypedDict, total=False): guardrailVersion: str | None +class BedrockKBUserContext(TypedDict): + userId: ReadOnly[str] + + class BedrockKBRequest(TypedDict, total=False): """Complete request structure for Bedrock Knowledge Base retrieval.""" @@ -134,6 +138,7 @@ class BedrockKBRequest(TypedDict, total=False): nextToken: str | None retrievalConfiguration: BedrockKBRetrievalConfiguration | None retrievalQuery: BedrockKBRetrievalQuery + userContext: ReadOnly[BedrockKBUserContext | None] ######################################################################### diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index d56ada07ed5..bcdee86360e 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -586,7 +586,7 @@ class MessageBlockDelta(TypedDict): type: Literal["message_delta"] delta: MessageDelta - usage: UsageDelta + usage: NotRequired[ReadOnly[UsageDelta]] context_management: NotRequired[ContextManagementResponse] diff --git a/litellm/types/rag.py b/litellm/types/rag.py index d1b411d8c04..629979afde9 100644 --- a/litellm/types/rag.py +++ b/litellm/types/rag.py @@ -2,10 +2,11 @@ Type definitions for RAG (Retrieval Augmented Generation) Ingest API. """ +from collections.abc import Mapping from typing import Any, Literal from pydantic import BaseModel, ConfigDict -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.utils import ModelResponse @@ -237,10 +238,11 @@ class RAGIngestRequest(BaseModel): class RAGRetrievalConfig(TypedDict, total=False): """Configuration for vector store retrieval.""" - vector_store_id: str - custom_llm_provider: str - top_k: int # max results from vector store - filters: dict[str, Any] | None # optional - vector store filters + vector_store_id: ReadOnly[str] + custom_llm_provider: ReadOnly[str] + top_k: ReadOnly[int] + filters: ReadOnly[Mapping[str, object] | None] + retrieval_filter: ReadOnly[Mapping[str, object] | None] class RAGRerankConfig(TypedDict, total=False): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9e9f61507c2..dd21bbf0b25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1312,7 +1312,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1365,7 +1366,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1402,7 +1404,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1513,7 +1516,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1551,7 +1555,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1588,7 +1593,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1626,7 +1632,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1663,7 +1670,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -1701,7 +1709,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1812,7 +1821,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1848,7 +1858,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1884,7 +1895,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2029,7 +2041,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2066,7 +2079,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2103,7 +2117,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2286,7 +2301,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2323,7 +2339,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2360,7 +2377,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2505,7 +2523,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2539,7 +2558,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2573,7 +2593,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -3123,6 +3144,7 @@ "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -3514,6 +3536,7 @@ "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -3546,7 +3569,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -4151,12 +4174,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4167,13 +4193,17 @@ "azure/eu/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -4184,12 +4214,14 @@ "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4264,14 +4296,20 @@ }, "azure/eu/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -4297,14 +4335,20 @@ }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, "deprecation_date": "2027-02-09", "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -4330,8 +4374,9 @@ }, "azure/eu/gpt-5.1": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -4362,12 +4407,17 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.4e-07, + "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.38e-06, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -4398,18 +4448,20 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4433,7 +4485,7 @@ }, "azure/eu/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4441,6 +4493,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4466,12 +4519,15 @@ "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -4499,12 +4555,15 @@ "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4522,6 +4581,7 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4536,6 +4596,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4553,6 +4614,7 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -4562,12 +4624,15 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4579,12 +4644,15 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2027-04-14", "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -4610,12 +4678,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4627,12 +4698,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -4643,6 +4717,7 @@ "azure/global/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -4674,7 +4749,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/global/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -4710,7 +4790,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -4722,6 +4803,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4753,6 +4835,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -4985,8 +5068,10 @@ "azure/gpt-4.1": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -4994,6 +5079,8 @@ "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, + "output_cost_per_token_priority": 1.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5019,8 +5106,10 @@ "azure/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_priority": 8.75e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -5028,6 +5117,8 @@ "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, + "output_cost_per_token_priority": 1.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5053,8 +5144,10 @@ "azure/gpt-4.1-mini": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, + "input_cost_per_token_priority": 7e-07, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -5062,6 +5155,8 @@ "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, + "output_cost_per_token_priority": 2.8e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5087,8 +5182,10 @@ "azure/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 1.75e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, + "input_cost_per_token_priority": 7e-07, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -5096,6 +5193,8 @@ "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, + "output_cost_per_token_priority": 2.8e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5130,6 +5229,7 @@ "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5163,6 +5263,7 @@ "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -5222,12 +5323,15 @@ "azure/gpt-4o-2024-05-13": { "deprecation_date": "2026-10-01", "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5238,12 +5342,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5254,13 +5361,16 @@ "azure/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, - "input_cost_per_token": 2.75e-06, + "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 1.1e-05, + "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5447,13 +5557,16 @@ "azure/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2027-04-14", - "input_cost_per_token": 1.65e-07, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 6.6e-07, + "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -5904,6 +6017,9 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_batches": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_minimal_reasoning_effort": true }, "azure/gpt-5.1-chat-2025-11-13": { @@ -5942,7 +6058,8 @@ "supports_tool_choice": false, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -5957,6 +6074,7 @@ "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -5991,6 +6109,7 @@ "mode": "responses", "output_cost_per_token": 2e-06, "output_cost_per_token_priority": 3.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6015,13 +6134,19 @@ "azure/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6047,14 +6172,20 @@ }, "azure/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2027-02-09", "input_cost_per_token": 1.25e-06, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6088,7 +6219,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6155,6 +6286,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6179,13 +6311,19 @@ "azure/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_batches": 1e-06, + "output_cost_per_token_priority": 3.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6211,14 +6349,20 @@ }, "azure/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-02-09", "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_batches": 1e-06, + "output_cost_per_token_priority": 3.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6246,12 +6390,15 @@ "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6279,12 +6426,15 @@ "cache_read_input_token_cost": 5e-09, "deprecation_date": "2027-02-09", "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_batches": 2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6311,13 +6461,15 @@ "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", "input_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/models-sold-directly-by-azure?pivots=azure-openai&tabs=global-standard-aoai%2Cstandard-chat-completions%2Cglobal-standard#gpt-5", + "output_cost_per_token_batches": 6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6341,6 +6493,7 @@ "azure/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -6372,7 +6525,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_priority": 2.5e-06, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_priority": 2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.1-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -6408,7 +6566,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", @@ -6420,6 +6579,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6451,6 +6611,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6482,6 +6643,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6506,13 +6668,19 @@ "azure/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, + "input_cost_per_token_batches": 8.75e-07, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "output_cost_per_token_batches": 7e-06, + "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6542,6 +6710,7 @@ "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-06-08", "input_cost_per_token": 1.75e-06, + "input_cost_per_token_batches": 8.75e-07, "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -6549,7 +6718,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, + "output_cost_per_token_batches": 7e-06, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -6587,6 +6758,7 @@ "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -6622,6 +6794,7 @@ "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -6654,6 +6827,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6688,6 +6862,7 @@ "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -6712,14 +6887,18 @@ }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, "deprecation_date": "2027-08-24", "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -6743,17 +6922,20 @@ }, "azure/gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, + "input_cost_per_token_batches": 1.05e-05, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "output_cost_per_token_batches": 8.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -6779,17 +6961,20 @@ }, "azure/gpt-5.2-pro-2025-12-11": { "input_cost_per_token": 2.1e-05, + "input_cost_per_token_batches": 1.05e-05, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, + "output_cost_per_token_batches": 8.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -6817,6 +7002,7 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, @@ -6856,12 +7042,20 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_flex": 7.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4": { "deprecation_date": "2027-09-02", - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -6896,12 +7090,18 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4": { "deprecation_date": "2027-09-02", - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -6936,12 +7136,18 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "deprecation_date": "2027-09-02", @@ -6982,11 +7188,19 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_flex": 7.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4-2026-03-05": { - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", "input_cost_per_token": 2.75e-06, @@ -7022,11 +7236,17 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4-2026-03-05": { - "cache_read_input_token_cost": 2.8e-07, + "cache_read_input_token_cost": 2.75e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_priority": 5.5e-07, "deprecation_date": "2027-09-02", "input_cost_per_token": 2.75e-06, @@ -7062,6 +7282,11 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_batches": 1.375e-06, + "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": true }, @@ -7071,6 +7296,9 @@ "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_flex": 1.5e-05, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -7078,11 +7306,15 @@ "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_flex": 9e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -7112,6 +7344,9 @@ "deprecation_date": "2027-09-07", "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_flex": 1.5e-05, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -7119,11 +7354,15 @@ "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_flex": 9e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -7202,33 +7441,42 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_creation_input_token_cost_priority": 1.25e-05, - "cache_creation_input_token_cost_above_272k_tokens_priority": 2.5e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_priority": 1e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_priority": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_flex": 2.5e-06, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_priority": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_flex": 2e-07, "deprecation_date": "2028-01-11", - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_priority": 1e-05, - "input_cost_per_token_above_272k_tokens_priority": 2e-05, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_priority": 8e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "input_cost_per_token_flex": 2e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_priority": 6e-05, - "output_cost_per_token_above_272k_tokens_priority": 9e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_priority": 4e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "output_cost_per_token_flex": 1e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7259,17 +7507,23 @@ "azure/gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_flex": 1.25e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_priority": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_flex": 1e-07, "deprecation_date": "2028-01-11", "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_priority": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_flex": 1e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7277,13 +7531,16 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, + "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_priority": 2.4e-05, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, + "output_cost_per_token_flex": 6e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7314,17 +7571,23 @@ "azure/gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2028-01-11", "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, + "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_priority": 4e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, + "input_cost_per_token_flex": 1e-07, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -7332,13 +7595,16 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_priority": 2.4e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, + "output_cost_per_token_flex": 6e-07, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7385,6 +7651,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -7542,33 +7809,34 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_272k_tokens_priority": 2.75e-05, - "cache_creation_input_token_cost_priority": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 2.2e-06, - "cache_read_input_token_cost_priority": 1.1e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2.2e-05, + "cache_creation_input_token_cost_priority": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, + "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_above_272k_tokens_priority": 2.2e-05, - "input_cost_per_token_priority": 1.1e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, + "input_cost_per_token_priority": 8.8e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_above_272k_tokens_priority": 9.9e-05, - "output_cost_per_token_priority": 6.6e-05, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, + "output_cost_per_token_above_272k_tokens_priority": 6.6e-05, + "output_cost_per_token_priority": 4.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7624,6 +7892,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7679,6 +7948,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7725,6 +7995,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -7845,33 +8116,34 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_creation_input_token_cost_above_272k_tokens_priority": 2.75e-05, - "cache_creation_input_token_cost_priority": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 2.2e-06, - "cache_read_input_token_cost_priority": 1.1e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2.2e-05, + "cache_creation_input_token_cost_priority": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.76e-06, + "cache_read_input_token_cost_priority": 8.8e-07, "deprecation_date": "2028-01-11", - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "input_cost_per_token_above_272k_tokens_priority": 2.2e-05, - "input_cost_per_token_priority": 1.1e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.76e-05, + "input_cost_per_token_priority": 8.8e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, - "output_cost_per_token_above_272k_tokens_priority": 9.9e-05, - "output_cost_per_token_priority": 6.6e-05, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, + "output_cost_per_token_above_272k_tokens_priority": 6.6e-05, + "output_cost_per_token_priority": 4.4e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7927,6 +8199,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -7982,6 +8255,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8013,12 +8287,16 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, + "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_flex": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -8026,13 +8304,15 @@ "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, + "output_cost_per_token_batches": 1.5e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8064,9 +8344,10 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, @@ -8076,11 +8357,13 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8112,9 +8395,10 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, "output_cost_per_token_above_272k_tokens": 4.95e-05, @@ -8124,11 +8408,13 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -8159,11 +8445,12 @@ "azure/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_priority": 1e-06, + "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_priority": 1e-05, + "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, "litellm_provider": "azure", "max_input_tokens": 1050000, @@ -8172,7 +8459,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -8202,12 +8489,17 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "deprecation_date": "2027-10-26" + "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, "input_cost_per_token_priority": 1.375e-05, @@ -8247,12 +8539,15 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "deprecation_date": "2027-10-26" + "deprecation_date": "2027-10-26", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.65e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "cache_read_input_token_cost_priority": 1.38e-06, + "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, "input_cost_per_token_priority": 1.375e-05, @@ -8292,7 +8587,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "deprecation_date": "2027-10-26" + "deprecation_date": "2027-10-26", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.65e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -8381,6 +8679,8 @@ "azure/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -8418,10 +8718,19 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, + "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_flex": 2.25e-06, + "output_cost_per_token_priority": 9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "deprecation_date": "2027-09-21", "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -8460,11 +8769,19 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, + "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_flex": 2.25e-06, + "output_cost_per_token_priority": 9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -8502,10 +8819,16 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_flex": 1e-07, + "output_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_flex": 6.25e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_flex": 1e-08, "deprecation_date": "2027-09-21", "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -8544,6 +8867,11 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_flex": 1e-07, + "output_cost_per_token_batches": 6.25e-07, + "output_cost_per_token_flex": 6.25e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, "azure/gpt-image-1": { @@ -8865,12 +9193,15 @@ "cache_read_input_token_cost": 7.5e-06, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.5e-05, + "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, + "output_cost_per_token_batches": 3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8879,14 +9210,17 @@ "supports_vision": true }, "azure/o1-mini": { - "cache_read_input_token_cost": 6.05e-07, - "input_cost_per_token": 1.21e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 4.84e-06, + "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8896,12 +9230,15 @@ "azure/o1-mini-2024-09-12": { "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8917,6 +9254,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -8932,6 +9270,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -8973,12 +9312,15 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_batches": 4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9014,6 +9356,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9057,12 +9400,15 @@ "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -9079,6 +9425,7 @@ "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9110,6 +9457,7 @@ "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9164,12 +9512,15 @@ "cache_read_input_token_cost": 2.75e-07, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_prompt_caching": true, @@ -9209,7 +9560,8 @@ "max_input_tokens": 8191, "max_tokens": 8191, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/text-embedding-3-small": { "deprecation_date": "2028-02-09", @@ -9218,7 +9570,8 @@ "max_input_tokens": 8191, "max_tokens": 8191, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/text-embedding-ada-002": { "deprecation_date": "2028-02-09", @@ -9227,7 +9580,8 @@ "max_input_tokens": 8191, "max_tokens": 8191, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/speech/azure-tts": { "input_cost_per_character": 1.5e-05, @@ -9267,8 +9621,10 @@ "azure/us/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_priority": 9.63e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -9276,6 +9632,8 @@ "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, + "output_cost_per_token_priority": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9301,8 +9659,10 @@ "azure/us/gpt-4.1-mini-2025-04-14": { "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.1e-07, + "cache_read_input_token_cost_priority": 1.93e-07, "input_cost_per_token": 4.4e-07, "input_cost_per_token_batches": 2.2e-07, + "input_cost_per_token_priority": 7.7e-07, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -9310,6 +9670,8 @@ "mode": "chat", "output_cost_per_token": 1.76e-06, "output_cost_per_token_batches": 8.8e-07, + "output_cost_per_token_priority": 3.08e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9334,9 +9696,9 @@ }, "azure/us/gpt-4.1-nano-2025-04-14": { "deprecation_date": "2027-04-14", - "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.1e-07, - "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_batches": 5.5e-08, "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, @@ -9344,6 +9706,7 @@ "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9369,12 +9732,15 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9385,13 +9751,17 @@ "azure/us/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -9402,12 +9772,14 @@ "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9482,14 +9854,20 @@ }, "azure/us/gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, "deprecation_date": "2027-02-09", "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9515,14 +9893,20 @@ }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, "deprecation_date": "2027-02-09", "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9550,12 +9934,15 @@ "cache_read_input_token_cost": 5.5e-09, "deprecation_date": "2027-02-09", "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9581,8 +9968,9 @@ }, "azure/us/gpt-5.1": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -9613,12 +10001,17 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.4e-07, + "cache_read_input_token_cost": 1.375e-07, "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.38e-06, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -9649,18 +10042,20 @@ "supports_tool_choice": true, "supports_vision": true, "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none" + "default_reasoning_effort": "none", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.38e-06, + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -9684,7 +10079,7 @@ }, "azure/us/gpt-5.1-codex-mini": { "deprecation_date": "2027-05-15", - "cache_read_input_token_cost": 2.8e-08, + "cache_read_input_token_cost": 2.75e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", "max_input_tokens": 272000, @@ -9692,6 +10087,7 @@ "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/responses" ], @@ -9717,12 +10113,15 @@ "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9740,6 +10139,7 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9754,6 +10154,7 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -9763,12 +10164,15 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -9801,21 +10205,25 @@ "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": false }, "azure/us/o4-mini-2025-04-16": { - "cache_read_input_token_cost": 3.1e-07, + "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_parallel_function_calling": false, "supports_prompt_caching": true, @@ -9861,7 +10269,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions" ], @@ -9910,7 +10318,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9925,7 +10333,7 @@ "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.828e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9941,7 +10349,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.52e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9957,7 +10365,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -9972,37 +10380,37 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true }, "azure_ai/FW-GLM-5.2-Fast": { - "cache_read_input_token_cost": 2.1e-07, - "input_cost_per_token": 2.1e-06, + "cache_read_input_token_cost": 2.31e-07, + "input_cost_per_token": 2.31e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 6.6e-06, - "source": "https://docs.fireworks.ai/serverless/pricing", + "output_cost_per_token": 7.26e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true }, "azure_ai/FW-Inkling": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "max_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.05e-06, - "source": "https://fireworks.ai/models/fireworks/inkling", + "output_cost_per_token": 4.46e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text" ], @@ -10024,7 +10432,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.3e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10047,7 +10455,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10070,7 +10478,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10098,7 +10506,7 @@ "high", "max" ], - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10122,7 +10530,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.32e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10137,7 +10545,7 @@ "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.32e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -10158,7 +10566,7 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/fireworks/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text" ], @@ -10172,15 +10580,15 @@ "supports_vision": false }, "azure_ai/FW-Nemotron-3-Ultra-NVFP4": { - "cache_read_input_token_cost": 1.19e-07, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 1.3e-07, + "input_cost_per_token": 6.6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 2.4e-06, - "source": "https://fireworks.ai/models/fireworks/nemotron-3-ultra-nvfp4", + "output_cost_per_token": 2.64e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text" ], @@ -10199,7 +10607,7 @@ "mode": "image_generation", "output_cost_per_image": 0.05, "output_cost_per_image_token": 4.7e-05, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -10213,7 +10621,7 @@ "mode": "image_generation", "output_cost_per_image": 0.0338, "output_cost_per_image_token": 3.3e-05, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" @@ -10227,7 +10635,7 @@ "mode": "image_generation", "output_cost_per_image": 0.02, "output_cost_per_image_token": 1.95e-05, - "source": "https://aka.ms/mai-image-2e-foundryblog", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/images/generations" ] @@ -10241,7 +10649,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/chat/completions" ], @@ -10292,19 +10700,19 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 7.1e-07, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true }, "azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "input_cost_per_token": 1.41e-06, + "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 3.5e-07, - "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", + "output_cost_per_token": 1e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -10375,7 +10783,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10387,7 +10795,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10399,7 +10807,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10411,7 +10819,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10423,7 +10831,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10435,7 +10843,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10447,7 +10855,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.4e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10459,7 +10867,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": false }, @@ -10471,7 +10879,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true, "supports_vision": true }, @@ -10483,7 +10891,7 @@ "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/affordable-innovation-unveiling-the-pricing-of-phi-3-slms-on-models-as-a-service/4156495", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": false @@ -10496,7 +10904,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, - "source": "https://techcommunity.microsoft.com/blog/Azure-AI-Services-blog/announcing-new-phi-pricing-empowering-your-business-with-small-language-models/4395112", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { @@ -10508,20 +10916,20 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.2e-07, - "source": "https://techcommunity.microsoft.com/blog/Azure-AI-Services-blog/announcing-new-phi-pricing-empowering-your-business-with-small-language-models/4395112", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_audio_input": true, "supports_function_calling": true, "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { - "input_cost_per_token": 8e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 3.2e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", + "output_cost_per_token": 3e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { @@ -10532,7 +10940,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_reasoning": true @@ -10584,7 +10992,7 @@ "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/cohere/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10623,7 +11031,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_reasoning": true, "supports_tool_choice": true }, @@ -10688,7 +11096,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -10703,7 +11111,7 @@ "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -10719,7 +11127,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/deepseek-r1-improved-performance-higher-limits-and-transparent-pricing/4386367", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_reasoning": true, "supports_tool_choice": true }, @@ -10731,7 +11139,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/announcing-deepseek-v3-on-azure-ai-foundry-and-github/4390438", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true }, "azure_ai/deepseek-v3-0324": { @@ -10743,7 +11151,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, - "source": "https://techcommunity.microsoft.com/blog/machinelearningblog/announcing-deepseek-v3-on-azure-ai-foundry-and-github/4390438", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10756,7 +11164,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.94e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true @@ -10770,7 +11178,7 @@ "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -10786,7 +11194,7 @@ "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -10803,7 +11211,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10817,7 +11225,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, - "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/embeddings" ], @@ -10836,7 +11244,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, - "source": "https://devblogs.microsoft.com/foundry/announcing-grok-3-and-grok-3-mini-on-azure-ai-foundry/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": false, "supports_tool_choice": true, @@ -10851,7 +11259,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.27e-06, - "source": "https://devblogs.microsoft.com/foundry/announcing-grok-3-and-grok-3-mini-on-azure-ai-foundry/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": false, @@ -10867,7 +11275,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": false, "supports_tool_choice": true, @@ -10882,7 +11290,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.27e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": false, @@ -10897,7 +11305,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10905,14 +11313,17 @@ }, "azure_ai/grok-4.3": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096", + "output_cost_per_token_above_200k_tokens": 5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10923,14 +11334,17 @@ }, "azure_ai/grok-4.6": { "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/grok-4-6-comes-to-microsoft-foundry-models-built-for-long-horizon-reasoning-and-/4547578", + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -10949,7 +11363,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10967,7 +11381,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10983,6 +11397,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -10997,7 +11412,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -11011,7 +11426,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://techcommunity.microsoft.com/t5/Azure-AI-Foundry-Blog/Grok-4-0-Goes-GA-in-Microsoft-Foundry-and-Grok-4-1-Fast-Arrives/ba-p/4497964", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -11025,7 +11440,7 @@ "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "source": "https://techcommunity.microsoft.com/t5/Azure-AI-Foundry-Blog/Grok-4-0-Goes-GA-in-Microsoft-Foundry-and-Grok-4-1-Fast-Arrives/ba-p/4497964", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -11040,7 +11455,7 @@ "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -11075,7 +11490,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, @@ -11092,7 +11507,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -11162,7 +11577,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.5e-06, - "source": "https://azure.microsoft.com/en-us/blog/introducing-mistral-large-3-in-microsoft-foundry-open-capable-and-ready-for-production-workloads/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -22432,6 +22847,7 @@ "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { "cache_read_input_token_cost": 6e-07, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 1.2e-06, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -22819,6 +23235,7 @@ "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -22925,6 +23342,7 @@ "fireworks_ai/deepseek-v4-pro": { "cache_read_input_token_cost": 6e-07, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 1.2e-06, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -23143,6 +23561,7 @@ "fireworks_ai/minimax-m2p7": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 6e-07, + "deprecation_date": "2026-08-27", "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -23664,6 +24083,7 @@ "cache_read_input_token_cost": 2.5e-08, "deprecation_date": "2026-06-01", "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_character": 3.75e-08, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, @@ -23743,6 +24163,7 @@ "cache_read_input_token_cost": 1.875e-08, "deprecation_date": "2026-06-01", "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_audio_token_batches": 3.75e-08, "input_cost_per_character": 1.875e-08, "input_cost_per_token": 7.5e-08, "input_cost_per_token_batches": 3.75e-08, @@ -23860,6 +24281,7 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, "input_cost_per_token_priority": 5.4e-07, @@ -24242,7 +24664,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -24384,6 +24807,7 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 5e-08, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_flex": 5e-08, "input_cost_per_token_priority": 1.8e-07, @@ -25001,6 +25425,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, @@ -25474,22 +25899,24 @@ } }, "gemini/gemini-robotics-er-2-preview": { - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 2e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, - "output_cost_per_token": 1e-05, + "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er-2", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25741,7 +26168,9 @@ "output_vector_size": 3072, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, "supports_multimodal": true, + "supports_vision": true, "tpm": 10000000 }, "gemini/gemini-1.5-flash": { @@ -25874,18 +26303,21 @@ } }, "gemini/gemini-2.5-flash": { + "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_flex": 3e-08, + "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 100000, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25919,6 +26351,14 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, + "output_cost_per_token_batches": 1.25e-06, + "output_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_priority": 4.5e-06, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { @@ -25926,9 +26366,12 @@ "deprecation_date": "2026-10-02", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, + "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.4e-07, "litellm_provider": "gemini", "supports_reasoning": false, - "max_input_tokens": 32768, + "max_input_tokens": 65536, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -25937,7 +26380,7 @@ "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "rpm": 100000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-flash-image", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25954,28 +26397,31 @@ "image" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, "supports_vision": true, - "supports_web_search": true, + "supports_web_search": false, "tpm": 8000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, + "supports_audio_input": false, "supports_image_size": false }, "gemini/gemini-3-pro-image": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 3.6e-06, "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, @@ -25987,7 +26433,9 @@ "rpm": 1000, "tpm": 4000000, "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3-pro-image", + "output_cost_per_token_flex": 6e-06, + "output_cost_per_token_priority": 2.16e-05, + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26003,7 +26451,7 @@ ], "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26106,7 +26554,7 @@ "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", - "max_input_tokens": 65536, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "image_generation", @@ -26116,7 +26564,7 @@ "output_cost_per_token_batches": 1.5e-06, "rpm": 1000, "tpm": 4000000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26133,7 +26581,7 @@ "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": false, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, @@ -26201,7 +26649,7 @@ "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, "rpm": 1000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26215,12 +26663,13 @@ "text", "image" ], - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": false, "supports_reasoning": false, "supports_response_schema": false, "supports_system_messages": true, "supports_vision": true, + "supports_web_search": false, "tpm": 4000000 }, "gemini/deep-research-pro-preview-12-2025": { @@ -26265,18 +26714,21 @@ } }, "gemini/gemini-2.5-flash-lite": { + "cache_read_input_audio_token_cost": 3e-08, "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_flex": 1e-08, + "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26310,6 +26762,14 @@ "search_context_size_high": 0.035 }, "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token_batches": 1.5e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 1.8e-07, + "output_cost_per_token_batches": 2e-07, + "output_cost_per_token_flex": 2e-07, + "output_cost_per_token_priority": 7.2e-07, + "supports_audio_input": true, "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { @@ -26555,34 +27015,46 @@ }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", + "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, + "cache_read_input_token_cost_flex": 1.25e-07, + "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, - "input_cost_per_token_priority": 1.25e-06, - "input_cost_per_token_above_200k_tokens_priority": 2.5e-06, + "input_cost_per_token_priority": 2.25e-06, + "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, - "output_cost_per_token_priority": 1e-05, - "output_cost_per_token_above_200k_tokens_priority": 1.5e-05, + "output_cost_per_token_priority": 1.8e-05, + "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "rpm": 2000, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -26613,7 +27085,11 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "google_maps_grounding_cost_per_query": 0.025 + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_token_batches": 6.25e-07, + "input_cost_per_token_flex": 6.25e-07, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_flex": 5e-06 }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, @@ -26626,7 +27102,7 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, "rpm": 2000, - "source": "https://ai.google.dev/gemini-api/docs/computer-use", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -26754,6 +27230,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini/gemini-3.1-flash-lite": { + "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -26774,7 +27251,7 @@ "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -26811,7 +27288,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, @@ -26872,13 +27350,15 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini/gemini-3-flash-preview": { + "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -26922,7 +27402,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_token_batches": 2.5e-07, + "input_cost_per_token_flex": 2.5e-07, + "output_cost_per_token_batches": 1.5e-06, + "output_cost_per_token_flex": 1.5e-06, + "supports_audio_input": true }, "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -26931,8 +27417,8 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -27212,7 +27698,7 @@ "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06, "rpm": 2000, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-3.1-pro-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -27247,13 +27733,16 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 6e-06 }, "gemini/gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, @@ -27271,7 +27760,7 @@ "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06, "rpm": 2000, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-3.1-pro-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -27306,13 +27795,16 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 6e-06 }, "gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, @@ -27366,6 +27858,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, @@ -27637,11 +28130,13 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -27652,19 +28147,20 @@ "audio" ], "supports_audio_output": false, - "supports_function_calling": true, + "supports_function_calling": false, "supports_prompt_caching": true, - "supports_response_schema": true, + "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, + "supports_vision": false, + "supports_web_search": false, "tpm": 10000000, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_audio_input": false }, "gemini/gemini-exp-1114": { "input_cost_per_token": 0, @@ -30223,6 +30719,7 @@ "mode": "image_generation", "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, "output_cost_per_token_batches": 5e-06, @@ -30241,6 +30738,7 @@ "mode": "image_generation", "output_cost_per_token": 1e-05, "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3.2e-05, "output_cost_per_token_batches": 5e-06, @@ -30257,6 +30755,7 @@ "litellm_provider": "openai", "mode": "image_generation", "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_image_token": 3e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -33033,6 +33532,7 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-10-23", "input_cost_per_image_token": 1e-05, + "input_cost_per_image_token_batches": 5e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "openai", @@ -33048,6 +33548,7 @@ "cache_read_input_token_cost": 2e-07, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 2.5e-06, + "input_cost_per_image_token_batches": 1.25e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "openai", @@ -44106,7 +44607,7 @@ "supports_tool_choice": true }, "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, @@ -44399,7 +44900,7 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, @@ -44414,7 +44915,7 @@ "supports_vision": true }, "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", "max_input_tokens": 514, @@ -44527,7 +45028,7 @@ "supports_tool_choice": true }, "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-14", + "deprecation_date": "2026-09-15", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", @@ -44977,6 +45478,7 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -45009,6 +45511,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -45040,6 +45543,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -45089,7 +45593,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, "us-gov.nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -48221,7 +48726,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token_batches": 2.5e-07 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -48298,49 +48804,56 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/imagegeneration@006": { + "deprecation_date": "2025-09-24", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.02, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.02, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-generate-002": { - "deprecation_date": "2025-11-10", + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-3.0-capability-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" }, "vertex_ai/imagen-4.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.02, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-4.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "vertex_ai/imagen-4.0-ultra-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", "output_cost_per_image": 0.06, @@ -55277,6 +55790,7 @@ "cache_read_input_token_cost": 1.25e-06, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "openai", @@ -55424,7 +55938,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -55444,7 +55958,11 @@ ], "supports_audio_input": true, "supports_audio_output": true, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -55477,7 +55995,9 @@ "supports_function_calling": true, "supports_vision": true, "supports_web_search": true, - "gemini_audio_only_live": true + "gemini_audio_only_live": true, + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, @@ -55539,7 +56059,7 @@ "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", - "max_input_tokens": 1048576, + "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "realtime", @@ -55561,7 +56081,11 @@ "supports_audio_output": true, "tpm": 250000, "rpm": 10, - "gemini_native_audio": true + "gemini_native_audio": true, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true }, "gemini/gemini-3.1-flash-live-preview": { "input_cost_per_audio_token": 3e-06, @@ -55596,32 +56120,48 @@ "supports_web_search": true, "tpm": 250000, "rpm": 10, - "gemini_audio_only_live": true + "gemini_audio_only_live": true, + "input_cost_per_second": 8.33333333333e-05, + "supports_response_schema": false }, "gemini/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "audio_speech", + "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, - "source": "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" ], "tpm": 4000000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "audio_speech", + "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/audio/speech" - ] + ], + "supports_audio_input": false, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false }, "gemini-flash-latest": { "cache_read_input_token_cost": 3e-08, @@ -58195,6 +58735,9 @@ ], "supports_audio_input": true, "supports_audio_output": true, + "supports_function_calling": false, + "supports_response_schema": false, + "supports_web_search": false, "tpm": 250000 }, "gemini/gemini-3.5-transcribe": { @@ -58216,7 +58759,8 @@ ], "supports_audio_input": true, "tpm": 800000, - "rpm": 2000 + "rpm": 2000, + "supports_function_calling": false }, "gemini/gemini-3.5-transcribe-live": { "input_cost_per_audio_token": 3.5e-06, @@ -58236,7 +58780,8 @@ ], "supports_audio_input": true, "tpm": 250000, - "rpm": 10 + "rpm": 10, + "supports_function_calling": false }, "vertex_ai/gemini-3.5-transcribe-preview": { "input_cost_per_audio_token": 2e-06, @@ -60875,7 +61420,7 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", @@ -61738,7 +62283,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "text", "image" @@ -65816,6 +66361,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { + "deprecation_date": "2024-08-22", "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65823,6 +66369,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { + "deprecation_date": "2025-12-23", "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "mode": "chat", @@ -65830,6 +66377,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65837,6 +66385,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { + "deprecation_date": "2025-11-13", "input_cost_per_token": 1.6e-06, "litellm_provider": "together_ai", "mode": "chat", @@ -65930,6 +66479,7 @@ "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "together_ai/google/gemma-2-27b-it": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65954,6 +66504,7 @@ "source": "https://developers.openai.com/api/docs/pricing" }, "together_ai/meta-llama/Llama-3-8b-chat-hf": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65982,6 +66533,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { + "deprecation_date": "2025-12-23", "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65989,6 +66541,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -65996,6 +66549,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66003,6 +66557,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66017,6 +66572,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2-72B-Instruct": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 9e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66024,6 +66580,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2-VL-72B-Instruct": { + "deprecation_date": "2025-08-28", "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "mode": "chat", @@ -66045,6 +66602,7 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { + "deprecation_date": "2025-11-13", "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "mode": "chat", @@ -66052,12 +66610,598 @@ "source": "https://api.together.ai/v1/models" }, "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { + "deprecation_date": "2026-01-05", "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://api.together.ai/v1/models" }, + "azure/eu/codex-mini": { + "cache_read_input_token_cost": 4.13e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/computer-use-preview": { + "input_cost_per_token": 3.3e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4.1": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_priority": 9.63e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "output_cost_per_token_priority": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4.1-mini": { + "cache_read_input_token_cost": 1.1e-07, + "cache_read_input_token_cost_priority": 1.93e-07, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_batches": 2.2e-07, + "input_cost_per_token_priority": 7.7e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.76e-06, + "output_cost_per_token_batches": 8.8e-07, + "output_cost_per_token_priority": 3.08e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4.1-nano": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.1e-07, + "input_cost_per_token_batches": 5.5e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-4o-2024-05-13": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_batches": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5": { + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-codex": { + "cache_read_input_token_cost": 1.38e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-mini": { + "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, + "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-nano": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5-pro": { + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000132, + "output_cost_per_token_batches": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.1-codex-max": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_batches": 9.625e-07, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_batches": 7.7e-06, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2-codex": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.2-pro": { + "input_cost_per_token": 2.31e-05, + "input_cost_per_token_batches": 1.155e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.0001848, + "output_cost_per_token_batches": 9.24e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.3-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.3-codex": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.4-mini": { + "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_priority": 1.65e-07, + "input_cost_per_token": 8.25e-07, + "input_cost_per_token_batches": 4.125e-07, + "input_cost_per_token_priority": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.95e-06, + "output_cost_per_token_batches": 2.475e-06, + "output_cost_per_token_priority": 9.9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.4-nano": { + "cache_read_input_token_cost": 2.2e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.375e-06, + "output_cost_per_token_batches": 6.875e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-5.4-pro": { + "input_cost_per_token": 3.3e-05, + "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_batches": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000198, + "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_batches": 9.9e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/gpt-6-astra": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o1-mini": { + "cache_read_input_token_cost": 6.05e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o1-preview": { + "cache_read_input_token_cost": 8.25e-06, + "input_cost_per_token": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o3-2025-04-16": { + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o3-deep-research": { + "cache_read_input_token_cost": 2.75e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/o4-mini-2025-04-16": { + "cache_read_input_token_cost": 3.03e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/text-embedding-3-large": { + "input_cost_per_token": 1.43e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/text-embedding-3-small": { + "input_cost_per_token": 2.2e-08, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/eu/text-embedding-ada-002": { + "input_cost_per_token": 1.1e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "gemini/gemini-3.8-live": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 4.5e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, + "tpm": 250000, + "rpm": 10, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true + }, + "gemini/gemini-3.8-live-extended-thinking": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "gemini", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 4.5e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supports_audio_input": true, + "tpm": 250000, + "rpm": 10, + "supports_function_calling": true, + "supports_response_schema": false, + "supports_vision": true, + "supports_web_search": true + }, + "azure/us/codex-mini": { + "cache_read_input_token_cost": 4.13e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/computer-use-preview": { + "input_cost_per_token": 3.3e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.32e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4.1": { + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_priority": 9.63e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "output_cost_per_token_priority": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4.1-mini": { + "cache_read_input_token_cost": 1.1e-07, + "cache_read_input_token_cost_priority": 1.93e-07, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_batches": 2.2e-07, + "input_cost_per_token_priority": 7.7e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.76e-06, + "output_cost_per_token_batches": 8.8e-07, + "output_cost_per_token_priority": 3.08e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4.1-nano": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.1e-07, + "input_cost_per_token_batches": 5.5e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-4o-2024-05-13": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_batches": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_batches": 8.25e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5": { + "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_priority": 2.75e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_token_batches": 6.875e-07, + "input_cost_per_token_priority": 2.75e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "output_cost_per_token_priority": 2.2e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-codex": { + "cache_read_input_token_cost": 1.38e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-mini": { + "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_priority": 4.95e-08, + "input_cost_per_token": 2.75e-07, + "input_cost_per_token_batches": 1.375e-07, + "input_cost_per_token_priority": 4.95e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "output_cost_per_token_batches": 1.1e-06, + "output_cost_per_token_priority": 3.96e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-nano": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "input_cost_per_token_batches": 2.75e-08, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5-pro": { + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000132, + "output_cost_per_token_batches": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.1-codex-max": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_batches": 9.625e-07, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_batches": 7.7e-06, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2-codex": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.2-pro": { + "input_cost_per_token": 2.31e-05, + "input_cost_per_token_batches": 1.155e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.0001848, + "output_cost_per_token_batches": 9.24e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.3-chat": { + "cache_read_input_token_cost": 1.925e-07, + "input_cost_per_token": 1.925e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.3-codex": { + "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_priority": 3.85e-07, + "input_cost_per_token": 1.925e-06, + "input_cost_per_token_priority": 3.85e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.54e-05, + "output_cost_per_token_priority": 3.08e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.4-mini": { + "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_priority": 1.65e-07, + "input_cost_per_token": 8.25e-07, + "input_cost_per_token_batches": 4.125e-07, + "input_cost_per_token_priority": 1.65e-06, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.95e-06, + "output_cost_per_token_batches": 2.475e-06, + "output_cost_per_token_priority": 9.9e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.4-nano": { + "cache_read_input_token_cost": 2.2e-08, + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 1.375e-06, + "output_cost_per_token_batches": 6.875e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/gpt-5.4-pro": { + "input_cost_per_token": 3.3e-05, + "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_batches": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 0.000198, + "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_batches": 9.9e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/o1-mini": { + "cache_read_input_token_cost": 6.05e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/o1-preview": { + "cache_read_input_token_cost": 8.25e-06, + "input_cost_per_token": 1.65e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/o3-deep-research": { + "cache_read_input_token_cost": 2.75e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "azure", + "mode": "chat", + "output_cost_per_token": 4.4e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/text-embedding-3-large": { + "input_cost_per_token": 1.43e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/text-embedding-3-small": { + "input_cost_per_token": 2.2e-08, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, + "azure/us/text-embedding-ada-002": { + "input_cost_per_token": 1.1e-07, + "litellm_provider": "azure", + "mode": "embedding", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, "litellm_provider": "aihubmix", diff --git a/pyproject.toml b/pyproject.toml index 5f12b3c7307..93ff55c4069 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0", - "httpx>=0.28.0,<1.0", + "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", "tiktoken>=0.8.0,<1.0", diff --git a/tests/code_coverage_tests/test_provider_cache.py b/tests/code_coverage_tests/test_provider_cache.py new file mode 100644 index 00000000000..828227ed239 --- /dev/null +++ b/tests/code_coverage_tests/test_provider_cache.py @@ -0,0 +1,472 @@ +from __future__ import annotations + +import os +import shutil +import socket +import subprocess +import threading +import time +import uuid +from collections.abc import Generator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, replace +from http.client import HTTPConnection +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final +from urllib.parse import urlsplit + +import pytest +from e2e_http import NetworkError, PreparedForward, RawResponse, StreamChunk, StreamHead, forward, prepare_forward +from models import LiteLLMParamsBody +from provider_cache import CacheEdge, CacheHit, CaptureLease, exact_key, successful_response +from provider_cache_redis import PUBLISH, RedisCommands, RedisResponseStore, configured_cache, redis_store +from provider_cache_routing import LIVE_PROVIDER_REQUIRED, route_cache_model +from provider_edge import configured_cache_backend, start_provider_edge +from redis.exceptions import ConnectionError as RedisConnectionError + +SECRET: Final = b"synthetic-cache-hmac-key-for-tests" +BODY: Final = b'{"model":"test","messages":[{"role":"user","content":"hello"}]}' +SUCCESS: Final = b'{"id":"provider-fixed-id","choices":[{"message":{"content":"hello"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}' +HEADERS: Final = {"content-type": "application/json", "authorization": "Bearer synthetic-account-one"} + + +class Provider(ThreadingHTTPServer): + hits: tuple[tuple[str, bytes], ...] = () + response: bytes = SUCCESS + status: int = 200 + delay: float = 0 + stream: bool = False + truncated: bool = False + cookie: str = "" + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + server: Final = self.server + assert isinstance(server, Provider) + body: Final = self.rfile.read(int(self.headers.get("content-length", "0"))) + server.hits += ((self.path, body),) + time.sleep(server.delay) + self.send_response(server.status) + if server.stream: + self.send_header("content-type", "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.end_headers() + self.wfile.write(b"%x\r\n%s\r\n" % (len(server.response), server.response)) + if server.truncated: + self.close_connection = True + return + self.wfile.write(b"0\r\n\r\n") + return + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(server.response))) + if server.cookie: + self.send_header("set-cookie", server.cookie) + self.end_headers() + self.wfile.write(server.response) + + def log_message(self, format: str, *args: object) -> None: + pass + + +@pytest.fixture +def provider() -> Generator[Provider, None, None]: + server: Final = Provider(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.fixture(scope="module") +def redis_url(tmp_path_factory: pytest.TempPathFactory) -> Generator[str, None, None]: + configured: Final = os.environ.get("E2E_CACHE_TEST_REDIS_URL") + if configured: + yield configured + return + binary: Final = shutil.which("redis-server") + assert binary is not None, "Set E2E_CACHE_TEST_REDIS_URL or install Redis for cache integration checks" + root: Final = tmp_path_factory.mktemp("provider-cache-redis") + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port: Final = probe.getsockname()[1] + with (root / "redis.log").open("wb") as log: + process: Final = subprocess.Popen( + [binary, "--bind", "127.0.0.1", "--port", str(port), "--save", "", "--appendonly", "no", "--dir", str(root)], + stdout=log, stderr=subprocess.STDOUT, + ) + try: + deadline: Final = time.monotonic() + 5 + while True: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.1): + break + except OSError: + assert process.poll() is None and time.monotonic() < deadline + time.sleep(0.02) + yield f"redis://127.0.0.1:{port}/0" + finally: + process.terminate() + process.wait(timeout=5) + + +@pytest.fixture +def store(redis_url: str) -> RedisResponseStore: + return redis_store(redis_url, "test-" + uuid.uuid4().hex) + + +@contextmanager +def edge(cache: CacheEdge, provider: Provider) -> Generator[str, None, None]: + upstream: Final = f"http://127.0.0.1:{provider.server_port}" + running: Final = start_provider_edge(cache, mounts={"openai": upstream}) + try: + yield running.edge.api_base("openai") + "/v1/chat/completions" + finally: + running.shutdown() + + +def call(url: str, body: bytes = BODY, headers: dict[str, str] = HEADERS) -> RawResponse: + result: Final = forward("POST", url, headers=headers, body=body, timeout=5) + assert isinstance(result, RawResponse), result + return result + + +def test_success_is_reusable_across_fresh_edges(store: RedisResponseStore, provider: Provider) -> None: + with edge(CacheEdge(store, SECRET), provider) as url: + assert call(url).body == SUCCESS + assert call(url).body == SUCCESS + with edge(CacheEdge(store, SECRET), provider) as other: + assert call(other).body == SUCCESS + assert len(provider.hits) == 1 + + +@pytest.mark.parametrize("body", [BODY + b" ", BODY.replace(b"hello", b"Hello"), BODY.replace(b"test", b"test2")]) +def test_any_body_change_calls_live(store: RedisResponseStore, provider: Provider, body: bytes) -> None: + with edge(CacheEdge(store, SECRET), provider) as url: + call(url) + call(url, body) + call(url, body) + assert len(provider.hits) == 2 + + +@pytest.mark.parametrize("name,value", [("authorization", "Bearer another-account"), ("x-request-id", "one"), ("anthropic-version", "new")]) +def test_changed_header_cannot_reuse(store: RedisResponseStore, provider: Provider, name: str, value: str) -> None: + with edge(CacheEdge(store, SECRET), provider) as url: + call(url) + call(url, headers=HEADERS | {name: value}) + call(url + "?x=1") + assert len(provider.hits) == 3 + + +@pytest.mark.parametrize("status,response", [(429, b'{"error":"rate limited"}'), (500, b'failed'), (200, b'{"error":"bad"}'), (200, b'not json')]) +def test_failed_provider_responses_never_enter_cache(store: RedisResponseStore, provider: Provider, status: int, response: bytes) -> None: + provider.status = status + provider.response = response + with edge(CacheEdge(store, SECRET), provider) as url: + assert call(url).status_code == status + assert call(url).body == response + assert len(provider.hits) == 2 + + +def test_cookie_setting_success_is_reused_without_the_cookie(store: RedisResponseStore, provider: Provider) -> None: + provider.cookie = "__cf_bm=synthetic-bot-management; Path=/; HttpOnly; Secure" + with edge(CacheEdge(store, SECRET), provider) as url: + replies: Final = tuple(call(url) for _ in range(2)) + assert len(provider.hits) == 1 + assert all(reply.body == SUCCESS and "set-cookie" not in reply.headers for reply in replies) + + +def test_expiry_does_not_slide(store: RedisResponseStore, provider: Provider) -> None: + short: Final = replace(store, lifetime_ms=250) + with edge(CacheEdge(short, SECRET), provider) as url: + call(url) + call(url) + time.sleep(0.3) + call(url) + call(url) + assert len(provider.hits) == 2 + + +def test_concurrent_requests_publish_atomically(store: RedisResponseStore, provider: Provider) -> None: + provider.delay = 0.15 + with edge(CacheEdge(store, SECRET), provider) as url: + with ThreadPoolExecutor(max_workers=5) as executor: + replies: Final = tuple(executor.map(lambda _: call(url).body, range(5))) + assert replies == (SUCCESS,) * 5 + assert len(provider.hits) == 1 + + +@pytest.mark.parametrize("age_past_expiry_ms", [0, 1]) +def test_expired_response_is_rejected_without_physical_eviction( + store: RedisResponseStore, age_past_expiry_ms: int, +) -> None: + response_key: Final = store.keys("expired")[0] + retained: Final = store.client.eval( + """ +local clock = redis.call('TIME') +local expires = clock[1] * 1000 + math.floor(clock[2] / 1000) - tonumber(ARGV[1]) +redis.call('HSET', KEYS[1], 'captured', expires - 86400000, 'expires', expires, 'payload', 'old-response') +return redis.call('PTTL', KEYS[1]) +""", + 1, response_key, age_past_expiry_ms, + ) + assert retained == -1 + replacement: Final = store.lookup("expired") + assert isinstance(replacement, CaptureLease) + assert replacement.expires_at_ms - replacement.captured_at_ms == 86_400_000 + assert store.publish("expired", replacement, b"fresh-response") + hit: Final = store.lookup("expired") + assert isinstance(hit, CacheHit) and hit.payload == b"fresh-response" + + +@pytest.mark.parametrize("truncated", [False, True]) +def test_stream_completion_controls_publication(store: RedisResponseStore, provider: Provider, truncated: bool) -> None: + provider.stream = True + provider.truncated = truncated + provider.response = b'data: {"choices":[{"index":0,"delta":{"content":"hello"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' + with edge(CacheEdge(store, SECRET), provider) as url: + for _ in range(2): + result: Final = forward("POST", url, headers=HEADERS, body=BODY, timeout=5) + if truncated: + assert isinstance(result, NetworkError) + else: + assert isinstance(result, RawResponse) and result.body == provider.response + assert len(provider.hits) == (2 if truncated else 1) + + +def test_store_outage_preserves_provider_success(provider: Provider) -> None: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port: Final = probe.getsockname()[1] + unavailable: Final = redis_store(f"redis://127.0.0.1:{port}/0", "unavailable") + with edge(CacheEdge(unavailable, SECRET), provider) as url: + assert call(url).body == SUCCESS + assert call(url).body == SUCCESS + assert len(provider.hits) == 2 + + +def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None: + short: Final = replace(store, lease_ms=50) + old: Final = short.lookup("key") + assert isinstance(old, CaptureLease) + time.sleep(0.08) + current: Final = short.lookup("key") + assert isinstance(current, CaptureLease) + assert not short.publish("key", old, b"old") + assert short.publish("key", current, b"new") + hit: Final = short.lookup("key") + assert isinstance(hit, CacheHit) and hit.payload == b"new" + + +def test_identity_preserves_values_and_never_contains_credentials() -> None: + variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'') + keys: Final = tuple(exact_key(SECRET, "POST", "https://example.invalid/v1/chat/completions", HEADERS, body) for body in variants) + assert len(set(keys)) == len(variants) + assert all(len(key) == 64 and "synthetic-account" not in key for key in keys) + + +@pytest.mark.parametrize("payload", [b"corrupt response", '{"response":"{}","signature":"é"}'.encode()]) +def test_corrupt_entry_is_replaced_by_same_successful_request(store: RedisResponseStore, provider: Provider, payload: bytes) -> None: + upstream: Final = f"http://127.0.0.1:{provider.server_port}/v1/chat/completions" + prepared: Final = prepare_forward("POST", upstream, HEADERS, BODY) + assert isinstance(prepared, PreparedForward) + key: Final = exact_key(SECRET, "POST", upstream, prepared.headers, BODY) + lease: Final = store.lookup(key) + assert isinstance(lease, CaptureLease) + assert store.publish(key, lease, payload) + cache: Final = CacheEdge(store, SECRET) + for _ in range(2): + head = cache.forward("POST", upstream, HEADERS, BODY, 5) + assert isinstance(head, StreamHead) + assert b"".join(step.data for step in head.steps if isinstance(step, StreamChunk)) == SUCCESS + assert len(provider.hits) == 1 + assert dict(cache.counters.counts) == { + "corrupt": 1, "misses": 1, "upstream_attempts": 1, "writes": 1, "hits": 1, + } + + +@pytest.mark.parametrize("payload", [ + b'data: {}\n\ndata: [DONE]\n\n', + b'data: {"choices":[{"index":0,"delta":{}}]}\n\ndata: [DONE]\n\n', + b'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]', + b'data: {"error":{"message":"failed"}}\n\ndata: [DONE]\n\n', +]) +def test_malformed_success_stream_is_never_cached(store: RedisResponseStore, provider: Provider, payload: bytes) -> None: + provider.stream = True + provider.response = payload + with edge(CacheEdge(store, SECRET), provider) as url: + assert call(url).body == payload + assert call(url).body == payload + assert len(provider.hits) == 2 + + +def test_anthropic_stream_requires_start_finish_and_stop() -> None: + start: Final = b'data: {"type":"message_start","message":{}}\n\n' + finish: Final = b'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}\n\n' + stop: Final = b'data: {"type":"message_stop"}\n\n' + url: Final = "https://example.invalid/v1/messages" + headers: Final = {"content-type": "text/event-stream"} + assert successful_response(url, 200, headers, start + finish + stop) + assert not successful_response(url, 200, headers, start + stop) + assert not successful_response(url, 200, headers, finish + stop) + assert not successful_response(url, 200, headers, start + finish) + + +@pytest.mark.parametrize("provider,suffix", [("openai", "/v1"), ("anthropic", "")]) +def test_normal_registration_routes_supported_providers(provider: str, suffix: str) -> None: + params: Final = LiteLLMParamsBody(model=f"{provider}/test", api_key="os.environ/SYNTHETIC_KEY", timeout=12) + routed: Final = route_cache_model(params, lambda mount: f"http://edge.invalid/{mount}", enabled=True) + assert routed.api_base == f"http://edge.invalid/{provider}{suffix}" + assert routed.model_dump(exclude={"api_base"}) == params.model_dump(exclude={"api_base"}) + assert params.api_base is None + + +@pytest.mark.parametrize("params", [ + LiteLLMParamsBody(model="bedrock/test"), + LiteLLMParamsBody(model="azure/test"), + LiteLLMParamsBody(model="openai/test", api_base="https://custom.invalid/v1"), + LiteLLMParamsBody(model="openai/test", api_base=""), + LiteLLMParamsBody(model="openai/test", litellm_credential_name="named-credential"), + LiteLLMParamsBody(model="openai/test", mock_response="synthetic"), +]) +def test_registration_preserves_unsupported_or_explicit_routes(params: LiteLLMParamsBody) -> None: + def unexpected_edge(mount: str) -> str: + pytest.fail(f"should not start edge for {mount}") + assert route_cache_model(params, unexpected_edge, enabled=True) is params + + +def test_rollback_and_live_only_policy_keep_direct_provider_route() -> None: + params: Final = LiteLLMParamsBody(model="openai/test") + assert route_cache_model(params, lambda _: "http://edge.invalid", enabled=False) is params + assert route_cache_model(params, lambda _: "http://edge.invalid", enabled=True, mode="realtime") is params + token: Final = LIVE_PROVIDER_REQUIRED.set(True) + try: + assert route_cache_model(params, lambda _: "http://edge.invalid", enabled=True) is params + finally: + LIVE_PROVIDER_REQUIRED.reset(token) + assert route_cache_model(params, lambda _: "http://edge.invalid", enabled=True).api_base == "http://edge.invalid/v1" + + +@dataclass(frozen=True) +class PublishOutage: + client: RedisCommands + + def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object: + if script == PUBLISH: + raise RedisConnectionError("synthetic publication outage") + return self.client.eval(script, numkeys, *args) + + +def test_write_outage_preserves_success_without_hidden_retry(store: RedisResponseStore, provider: Provider) -> None: + unavailable: Final = replace(store, client=PublishOutage(store.client)) + cache: Final = CacheEdge(unavailable, SECRET) + with edge(cache, provider) as url: + assert call(url).body == SUCCESS + assert call(url).body == SUCCESS + assert len(provider.hits) == 2 + assert dict(cache.counters.counts)["write_failures"] == 2 + with edge(CacheEdge(store, SECRET), provider) as url: + assert call(url).body == SUCCESS + assert call(url).body == SUCCESS + assert len(provider.hits) == 3 + + +def test_connection_failure_releases_capture_lease(store: RedisResponseStore) -> None: + with socket.socket() as unavailable: + unavailable.bind(("127.0.0.1", 0)) + url: Final = f"http://127.0.0.1:{unavailable.getsockname()[1]}/v1/chat/completions" + cache: Final = CacheEdge(store, SECRET) + assert isinstance(cache.forward("POST", url, HEADERS, BODY, 0.2), NetworkError) + prepared: Final = prepare_forward("POST", url, HEADERS, BODY) + assert isinstance(prepared, PreparedForward) + key: Final = exact_key(SECRET, "POST", url, prepared.headers, BODY) + slot: Final = store.lookup(key) + assert isinstance(slot, CaptureLease) + assert store.release(key, slot) + assert dict(cache.counters.counts)["rejected"] == 1 + + +def test_close_before_first_chunk_releases_lease(store: RedisResponseStore, provider: Provider) -> None: + url: Final = f"http://127.0.0.1:{provider.server_port}/v1/chat/completions" + cache: Final = CacheEdge(store, SECRET) + head: Final = cache.forward("POST", url, HEADERS, BODY, 5) + assert isinstance(head, StreamHead) + head.steps.close() + prepared: Final = prepare_forward("POST", url, HEADERS, BODY) + assert isinstance(prepared, PreparedForward) + key: Final = exact_key(SECRET, "POST", url, prepared.headers, BODY) + slot: Final = store.lookup(key) + assert isinstance(slot, CaptureLease) + assert store.release(key, slot) + + +def test_effective_account_change_cannot_reuse_cache( + store: RedisResponseStore, provider: Provider, monkeypatch: pytest.MonkeyPatch, tmp_path, +) -> None: + url: Final = f"http://127.0.0.1:{provider.server_port}/v1/chat/completions" + cache: Final = CacheEdge(store, SECRET) + for account in ("account-a", "account-b", "account-b"): + netrc = tmp_path / account + netrc.write_text(f"machine 127.0.0.1 login {account} password synthetic\n") + monkeypatch.setenv("NETRC", str(netrc)) + head = cache.forward("POST", url, HEADERS, BODY, 5) + assert isinstance(head, StreamHead) + assert b"".join(step.data for step in head.steps if isinstance(step, StreamChunk)) == SUCCESS + assert len(provider.hits) == 2 + assert dict(cache.counters.counts)["hits"] == 1 + + +def test_enabled_environment_reuses_store_across_fresh_backends( + redis_url: str, provider: Provider, monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("E2E_PROVIDER_CACHE", "1") + monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_URL", redis_url) + monkeypatch.setenv("E2E_PROVIDER_CACHE_HMAC_KEY", SECRET.decode()) + monkeypatch.setenv("E2E_PROVIDER_CACHE_NAMESPACE", "environment-" + uuid.uuid4().hex) + configured_cache.cache_clear() + try: + for _ in range(2): + backend = configured_cache_backend() + assert isinstance(backend, CacheEdge) + with edge(backend, provider) as url: + assert call(url).body == SUCCESS + configured_cache.cache_clear() + assert len(provider.hits) == 1 + monkeypatch.setenv("E2E_PROVIDER_CACHE", "0") + assert configured_cache_backend() is None + finally: + configured_cache.cache_clear() + + +@pytest.mark.parametrize("known_mount", (True, False)) +def test_duplicate_headers_bypass_cache_and_count_live_calls( + store: RedisResponseStore, provider: Provider, known_mount: bool, +) -> None: + cache: Final = CacheEdge(store, SECRET) + with edge(cache, provider) as url: + parsed: Final = urlsplit(url) + for _ in range(2): + connection = HTTPConnection(str(parsed.hostname), parsed.port, timeout=5) + try: + connection.putrequest("POST", parsed.path if known_mount else "/unknown/v1/chat/completions") + connection.putheader("content-length", str(len(BODY))) + connection.putheader("content-type", "application/json") + connection.putheader("x-duplicate", "first") + connection.putheader("x-duplicate", "second") + connection.endheaders(BODY) + response = connection.getresponse() + assert response.status == (200 if known_mount else 404) + payload = response.read() + assert payload == SUCCESS if known_mount else b"unknown provider mount" in payload + finally: + connection.close() + assert len(provider.hits) == (2 if known_mount else 0) + assert dict(cache.counters.counts)["duplicate_header_bypass"] == 2 + assert dict(cache.counters.counts).get("upstream_attempts", 0) == (2 if known_mount else 0) diff --git a/tests/e2e/PROVIDER_CACHE.md b/tests/e2e/PROVIDER_CACHE.md new file mode 100644 index 00000000000..8635c9ed9ae --- /dev/null +++ b/tests/e2e/PROVIDER_CACHE.md @@ -0,0 +1,33 @@ +# Shared provider-response cache + +`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI and Anthropic model registrations use the provider edge. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live + +The edge caches complete successful POST responses for `/v1/chat/completions` and `/v1/messages`, including streams. Unsupported endpoints pass through. It matches the method, original URL, effective outbound headers (including authentication and HTTP-library defaults), body presence and exact body bytes using a full keyed digest. It sends the same prepared request used for matching. No prompts, random markers, JSON values or credentials are normalized away. Provider `Set-Cookie` headers are dropped before validation and never recorded: the edge already withholds them from the proxy, and OpenAI responses always carry Cloudflare bot-management cookies + +An eligible miss calls the provider. A complete successful response is stored immediately even if a later test assertion fails. Provider errors, malformed responses, truncated streams and cancelled captures are not stored. Cache reads, writes and lease failures fall through to normal provider behavior; they introduce no provider retry. An already-started response cannot be restarted after a delivery failure + +Recordings are shared across workers and builds through dedicated Redis, separate from the candidate's own cache. They expire 86,400 seconds after capture starts, based on Redis time. Reads never extend expiry. There is no scheduled recapture: the next miss calls the provider again. Bounded coordination reduces duplicate concurrent calls, but slow or failed captures may lead to extra live calls after the wait expires + +## Configuration + +The trusted runner receives: + +- `E2E_PROVIDER_CACHE`: `1` to enable, `0` to use the normal live path +- `E2E_PROVIDER_CACHE_REDIS_URL`: authenticated dedicated Redis URL +- `E2E_PROVIDER_CACHE_HMAC_KEY`: dedicated secret containing at least 32 bytes +- `E2E_PROVIDER_CACHE_NAMESPACE`: shared environment namespace, independent of build and candidate revision +- `E2E_PROVIDER_CACHE_METRICS_DIR`: optional per-process counter artifact directory + +Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits + +Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. The provider prompt-caching tests carry it because a replayed priming response reports cache creation rather than a cache read. Ordinary assertion failures still fail E2E. The shared cache does not modify provider response IDs or make the proxy aware of replay + +## Recorded response semantics + +Replay preserves the original response ID, usage and end-to-end headers. The proxy can therefore deduplicate repeated provider IDs when storing spend-log rows, just as it does when a live upstream returns the same ID twice. One spend-log row per invocation is not guaranteed for identical recorded responses. Existing spend reconciliation requests use distinct prompt markers and retain their distinct-ID and row-count assertions; accounting tests are not automatically excluded from caching + +Provider remaining-quota headers describe the captured response. Metrics derived from them are historical on a cache hit, not a measurement of current provider capacity. Gateway-generated API-key quota headers are a separate contract. A test of fresh provider quota or timing must use the live-provider policy; replay can still exercise how the proxy processes the recorded headers + +## Qualification + +`tests/code_coverage_tests/test_provider_cache.py` exercises local HTTP providers and disposable real Redis. CI runs these checks with the existing provider-edge and replay harness tests. These component checks do not establish Buildkite deployment, full-suite cross-build reuse or a genuine 24-hour expiry observation; those require separate runtime evidence diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b1a75d5f862..829c84910a9 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -22,7 +22,6 @@ from typing import Final import pytest import requests - from e2e_config import ( CONTROL_PLANE_BASE_URL, FIXTURE_DIR, @@ -41,6 +40,7 @@ from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from models import TeamNewBody, UserNewBody, UserNewResponse +from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client @@ -85,6 +85,7 @@ def jwt_identity(idp: Keycloak, resources: ResourceManager, proxy: ProxyClient) def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "provider_live: requires actual provider timing, limits or state; bypass shared cache") config.addinivalue_line( "markers", "e2e: live test that requires a running proxy and real provider keys", @@ -192,11 +193,13 @@ def _proxy_fail_reason() -> str | None: return None +@pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked tests (unit coverage of the harness) don't touch the proxy, so they run even when none is up. Never skip for a missing proxy. Replay mode needs the proxy too: only provider-bound traffic replays from the bundle.""" + LIVE_PROVIDER_REQUIRED.set(item.get_closest_marker("provider_live") is not None) if item.get_closest_marker("e2e") is None: return reason = _proxy_fail_reason() @@ -235,6 +238,7 @@ def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]: yield so fixture finalizers replay their recorded calls first. Failed tests are left alone - their own failure already explains any unconsumed tail.""" result = yield + LIVE_PROVIDER_REQUIRED.set(False) if not item.stash.get(_CALL_PASSED, False): return result reason = replay_leftover_error( diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 1992f419823..4184b6cbefc 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -853,6 +853,7 @@ def _stream_steps(resp: requests.Response) -> Generator[StreamStep, None, None]: the chunks already delivered are exactly what makes a mid-stream failure different from a request that never streamed at all.""" try: + yield StreamChunk(b"") for piece in cast("Iterator[bytes]", resp.iter_content(chunk_size=None)): if piece: yield StreamChunk(data=piece) @@ -862,6 +863,44 @@ def _stream_steps(resp: requests.Response) -> Generator[StreamStep, None, None]: resp.close() +def primed_steps(steps: Generator[StreamStep, None, None]) -> Generator[StreamStep, None, None]: + first: Final = next(steps) + assert isinstance(first, StreamChunk) and first.data == b"" + return steps + + +@dataclass(frozen=True, slots=True, repr=False) +class PreparedForward: + request: requests.PreparedRequest + url: str + headers: dict[str, str] + + +def prepare_forward( + method: str, url: str, headers: dict[str, str], body: bytes | None, +) -> PreparedForward | NetworkError: + try: + with requests.Session() as session: + request: Final = session.prepare_request(requests.Request(method, url, headers=headers, data=body)) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + assert request.url is not None + return PreparedForward(request, request.url, dict(request.headers)) + + +def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> StreamHead | NetworkError: + try: + with requests.Session() as session: + settings: Final = session.merge_environment_settings(prepared.url, {}, True, None, None) + resp: Final = session.send(prepared.request, timeout=timeout, allow_redirects=False, **settings) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return StreamHead( + resp.status_code, {name.lower(): value for name, value in resp.headers.items()}, + primed_steps(_stream_steps(resp)), + ) + + def open_stream(url: URL, *, headers: BaseModel, json: BaseModel, timeout: float = 60.0) -> StreamHead | NetworkError: """POST a streaming request and return the moment its response head arrives, leaving the body unread behind ``StreamHead.steps``. For a test that must keep @@ -907,5 +946,5 @@ def forward_stream( return StreamHead( status_code=resp.status_code, headers={name.lower(): value for name, value in resp.headers.items()}, - steps=_stream_steps(resp), + steps=primed_steps(_stream_steps(resp)), ) diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index a18e03c982b..102b3f00698 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -44,7 +44,7 @@ from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, Usage from passthrough_client import PassthroughClient import os -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_MODEL = "vertex_ai/gemini-2.5-flash" diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index ca58c30d40c..44c416a3e78 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -24,10 +24,10 @@ from models import ( AnthropicAssistantTurn, AnthropicContentBlock, AnthropicCustomTool, + AnthropicMessagesBody, AnthropicToolChoice, AnthropicToolResultBlock, AnthropicToolResultTurn, - AnthropicMessagesBody, ChatMessage, JsonSchemaProperty, LiteLLMParamsBody, @@ -165,6 +165,7 @@ class TestAnthropicMessages: ) @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") + @pytest.mark.provider_live def test_messages_streams_completion( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_outbound_http2_e2e.py b/tests/e2e/llm_translation/test_outbound_http2_e2e.py new file mode 100644 index 00000000000..cb2182ffd62 --- /dev/null +++ b/tests/e2e/llm_translation/test_outbound_http2_e2e.py @@ -0,0 +1,208 @@ +"""Outbound HTTP/2 negotiation for LiteLLM-built httpx clients. + +Spins up a local hypercorn TLS server that offers h2 and http/1.1 over ALPN and +drives the real AsyncHTTPHandler / HTTPHandler at it, so the negotiated protocol +on the wire is the assertion. No running proxy or provider credentials needed, +which is why these tests carry no `e2e` marker (same shape as the markerless +harness checks under tests/e2e/load/). +""" + +from __future__ import annotations + +import asyncio +import datetime +import ipaddress +import socket +import threading +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Final, cast + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID +from hypercorn.asyncio import ( + serve, # pyright: ignore[reportUnknownVariableType] # hypercorn's serve signature passes through untyped worker hooks +) +from hypercorn.config import Config +from hypercorn.typing import ( + ASGIReceiveCallable, + ASGISendCallable, + HTTPResponseBodyEvent, + HTTPResponseStartEvent, + Scope, +) + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + + +def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now: Final = datetime.datetime.now(datetime.timezone.utc) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=7)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + cert_file: Final = cert_dir / "cert.pem" + key_file: Final = cert_dir / "key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_file, key_file + + +async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: + if scope["type"] != "http": + return + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + if message["type"] == "http.request" and not message["more_body"]: + break + if scope["path"] == "/stream": + await send( + HTTPResponseStartEvent( + type="http.response.start", status=200, headers=[(b"content-type", b"text/event-stream")] + ) + ) + for index in range(3): + await send( + HTTPResponseBodyEvent( + type="http.response.body", body=f"data: chunk-{index}\n\n".encode(), more_body=True + ) + ) + await send(HTTPResponseBodyEvent(type="http.response.body", body=b"", more_body=False)) + return + await send( + HTTPResponseStartEvent(type="http.response.start", status=200, headers=[(b"content-type", b"application/json")]) + ) + await send(HTTPResponseBodyEvent(type="http.response.body", body=b'{"ok": true}', more_body=False)) + + +@pytest.fixture(scope="module") +def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: + cert_dir: Final = tmp_path_factory.mktemp("h2certs") + cert_file, key_file = _write_self_signed_cert(cert_dir) + + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port: Final = cast(int, sock.getsockname()[1]) + + shutdown: Final = threading.Event() + + def _serve() -> None: + loop: Final = asyncio.new_event_loop() + config: Final = Config() + config.bind = [f"127.0.0.1:{port}"] + config.certfile = str(cert_file) + config.keyfile = str(key_file) + config.alpn_protocols = ["h2", "http/1.1"] + loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait))) + loop.close() + + thread: Final = threading.Thread(target=_serve, daemon=True) + thread.start() + + for _ in range(100): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + else: + pytest.fail("hypercorn test server did not start") + + yield f"https://127.0.0.1:{port}" + + shutdown.set() + thread.join(timeout=10) + + +def _async_exchange(base_url: str) -> tuple[str, str, bytes]: + async def _run() -> tuple[str, str, bytes]: + handler: Final = AsyncHTTPHandler(ssl_verify=False) + try: + response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) + post_version: Final = response.http_version + async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: + stream_version: Final = stream_response.http_version + body: Final = b"".join([chunk async for chunk in stream_response.aiter_bytes()]) + return post_version, stream_version, body + finally: + await handler.close() + + return asyncio.run(_run()) + + +def _sync_exchange(base_url: str) -> tuple[str, str, bytes]: + handler: Final = HTTPHandler(ssl_verify=False) + try: + response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) + post_version: Final = response.http_version + with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: + stream_version: Final = stream_response.http_version + body: Final = b"".join(stream_response.iter_bytes()) + return post_version, stream_version, body + finally: + handler.close() + + +class TestOutboundHttp2: + @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")]) + def test_async_handler_negotiates_http2_only_when_enabled( + self, + monkeypatch: pytest.MonkeyPatch, + http2_tls_server: str, + use_http2: bool, + expected_version: str, + ) -> None: + monkeypatch.setattr(litellm, "http2", use_http2) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", False) + + post_version, stream_version, body = _async_exchange(http2_tls_server) + + assert post_version == expected_version + assert stream_version == expected_version + assert b"data: chunk-0" in body + + @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")]) + def test_sync_handler_negotiates_http2_only_when_enabled( + self, + monkeypatch: pytest.MonkeyPatch, + http2_tls_server: str, + use_http2: bool, + expected_version: str, + ) -> None: + monkeypatch.setattr(litellm, "http2", use_http2) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", False) + + post_version, stream_version, body = _sync_exchange(http2_tls_server) + + assert post_version == expected_version + assert stream_version == expected_version + assert b"data: chunk-0" in body diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py index 31e74c22e17..8dd7e7c1a31 100644 --- a/tests/e2e/llm_translation/test_together_ai_e2e.py +++ b/tests/e2e/llm_translation/test_together_ai_e2e.py @@ -744,6 +744,7 @@ class TestTogetherMessages: assert "22" in text, f"the model never saw the tool result: {response.content}" @pytest.mark.covers("llm.messages.together_ai.basic.stream.works") + @pytest.mark.provider_live def test_streams_text_deltas( self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str ) -> None: diff --git a/tests/e2e/provider_cache.py b/tests/e2e/provider_cache.py new file mode 100644 index 00000000000..0c6eac75a43 --- /dev/null +++ b/tests/e2e/provider_cache.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +import io +import threading +import time +from collections.abc import Callable, Generator, Mapping +from contextlib import closing +from dataclasses import dataclass, field +from typing import Final, Literal, Protocol +from urllib.parse import urlsplit + +from e2e_http import ( + NetworkError, + StreamChunk, + StreamHead, + StreamStep, + StreamTruncation, + forward_prepared_stream, + forward_stream, + prepare_forward, + primed_steps, +) +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError + +LIFETIME_SECONDS: Final = 86_400 +MAX_REQUEST_BYTES: Final = 256 * 1024 +MAX_RESPONSE_BYTES: Final = 8 * 1024 * 1024 +UNRECORDED_RESPONSE_HEADERS: Final = frozenset({"set-cookie"}) +JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +@dataclass(frozen=True, slots=True) +class CacheHit: + payload: bytes + valid_until: float + + +@dataclass(frozen=True, slots=True) +class CaptureLease: + token: str + captured_at_ms: int + expires_at_ms: int + + +@dataclass(frozen=True, slots=True) +class CacheBusy: + pass + + +@dataclass(frozen=True, slots=True) +class CacheUnavailable: + pass + + +type CacheLookup = CacheHit | CaptureLease | CacheBusy | CacheUnavailable + + +class ResponseStore(Protocol): + def lookup(self, key: str) -> CacheLookup: ... + + def publish(self, key: str, lease: CaptureLease, payload: bytes) -> bool: ... + + def release(self, key: str, lease: CaptureLease) -> bool: ... + + def discard(self, key: str, payload: bytes) -> bool: ... + + +class CachedResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + format_version: Literal[1] = 1 + request_key: str + status_code: int + headers: dict[str, str] + chunks: tuple[str, ...] + + +class SignedResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + response: str + signature: str + + +def exact_key(secret: bytes, method: str, url: str, headers: Mapping[str, str], body: bytes | None) -> str: + fields: Final = ( + b"provider-cache-exact-v1", method.encode(), url.encode(), + *(part.encode() for pair in sorted(headers.items()) for part in pair), + b"no-body" if body is None else b"body", b"" if body is None else body, + ) + encoded: Final = b"".join(len(part).to_bytes(8, "big") + part for part in fields) + return hmac.new(secret, encoded, hashlib.sha256).hexdigest() + + +def cacheable_endpoint(method: str, url: str, body: bytes | None) -> bool: + return ( + method == "POST" + and urlsplit(url).path in {"/v1/chat/completions", "/v1/messages"} + and body is not None + and len(body) <= MAX_REQUEST_BYTES + ) + + +def successful_response(url: str, status: int, headers: Mapping[str, str], body: bytes) -> bool: + if not 200 <= status < 300 or len(body) > MAX_RESPONSE_BYTES: + return False + streaming: Final = "text/event-stream" in headers.get("content-type", "").lower() + if streaming: + try: + text: Final = body.decode("utf-8").replace("\r\n", "\n") + if not text.endswith("\n\n"): + return False + events: Final = tuple( + "\n".join(line[5:].removeprefix(" ") for line in event.split("\n") if line.startswith("data:")) + for event in text.split("\n\n") if any(line.startswith("data:") for line in event.split("\n")) + ) + values: Final = tuple(JSON_VALUE.validate_json(event) for event in events if event != "[DONE]") + except (UnicodeDecodeError, ValidationError): + return False + if not values or any(not isinstance(value, dict) or "error" in value or value.get("type") == "error" for value in values): + return False + if urlsplit(url).path == "/v1/chat/completions": + return events[-1] == "[DONE]" and "[DONE]" not in events[:-1] and complete_chat_stream(values) + return ( + "[DONE]" not in events + and isinstance(values[0], dict) and values[0].get("type") == "message_start" + and isinstance(values[-1], dict) and values[-1].get("type") == "message_stop" + and any( + isinstance(value, dict) and value.get("type") == "message_delta" + and isinstance(delta := value.get("delta"), dict) and isinstance(delta.get("stop_reason"), str) + for value in values + ) + ) + try: + value: Final = JSON_VALUE.validate_json(body) + except ValidationError: + return False + if not isinstance(value, dict) or "error" in value: + return False + if urlsplit(url).path == "/v1/messages": + return value.get("type") == "message" and isinstance(value.get("content"), list) and isinstance(value.get("stop_reason"), str) + choices: Final = value.get("choices") + return isinstance(choices, list) and bool(choices) and all( + isinstance(choice, dict) and isinstance(choice.get("message"), dict) and isinstance(choice.get("finish_reason"), str) + for choice in choices + ) + + +def complete_chat_stream(values: tuple[JsonValue, ...]) -> bool: + if any(not isinstance(value, dict) or not isinstance(value.get("choices"), list) for value in values): + return False + choices: Final = tuple( + choice for value in values if isinstance(value, dict) + if isinstance(items := value.get("choices"), list) for choice in items + ) + if not choices or any( + not isinstance(choice, dict) or type(choice.get("index")) is not int + or not isinstance(choice.get("delta"), dict) + for choice in choices + ): + return False + indices: Final = frozenset(choice["index"] for choice in choices if isinstance(choice, dict)) + return all( + isinstance(tuple(choice for choice in choices if isinstance(choice, dict) and choice["index"] == index)[-1].get("finish_reason"), str) + for index in indices + ) + + +def encode_response(secret: bytes, response: CachedResponse) -> bytes: + raw: Final = response.model_dump_json() + return SignedResponse(response=raw, signature=hmac.new(secret, raw.encode(), hashlib.sha256).hexdigest()).model_dump_json().encode() + + +def decode_response(secret: bytes, key: str, payload: bytes, url: str) -> CachedResponse | None: + if len(payload) > 2 * MAX_RESPONSE_BYTES: + return None + try: + signed: Final = SignedResponse.model_validate_json(payload) + if not hmac.compare_digest(signed.signature.encode(), hmac.new(secret, signed.response.encode(), hashlib.sha256).hexdigest().encode()): + return None + response: Final = CachedResponse.model_validate_json(signed.response) + chunks: Final = tuple(base64.b64decode(chunk, validate=True) for chunk in response.chunks) + except (ValidationError, ValueError): + return None + if response.request_key != key or not successful_response(url, response.status_code, response.headers, b"".join(chunks)): + return None + return response + + +@dataclass(slots=True) +class CacheCounters: + counts: tuple[tuple[str, int], ...] = () + lock: threading.Lock = field(default_factory=threading.Lock) + + def increment(self, name: str) -> None: + with self.lock: + current: Final = dict(self.counts) + self.counts = tuple((current | {name: current.get(name, 0) + 1}).items()) + + +@dataclass(slots=True) +class ResponseCapture: + buffer: io.BytesIO = field(default_factory=io.BytesIO) + size: int = 0 + eligible: bool = True + + def observe(self, step: StreamStep) -> None: + if not self.eligible: + return + if isinstance(step, StreamTruncation) or self.size + len(step.data) + 8 > MAX_RESPONSE_BYTES: + self.eligible = False + self.buffer.close() + return + self.buffer.write(len(step.data).to_bytes(8, "big")) + self.buffer.write(step.data) + self.size += len(step.data) + 8 + + def chunks(self) -> tuple[bytes, ...]: + self.buffer.seek(0) + return tuple(self.buffer.read(int.from_bytes(size, "big")) for size in iter(lambda: self.buffer.read(8), b"")) + + +def response_steps(response: CachedResponse) -> Generator[StreamStep, None, None]: + for chunk in response.chunks: + yield StreamChunk(base64.b64decode(chunk, validate=True)) + + +@dataclass(frozen=True, slots=True) +class CacheEdge: + store: ResponseStore + secret: bytes = field(repr=False) + counters: CacheCounters = field(default_factory=CacheCounters) + wait_seconds: float = 2.0 + clock: Callable[[], float] = time.monotonic + sleep: Callable[[float], None] = time.sleep + + def lookup(self, key: str) -> CacheLookup: + deadline: Final = self.clock() + self.wait_seconds + while isinstance(result := self.store.lookup(key), CacheBusy) and self.clock() < deadline: + self.sleep(min(0.05, max(0, deadline - self.clock()))) + return result + + def forward(self, method: str, url: str, headers: dict[str, str], body: bytes | None, timeout: float) -> StreamHead | NetworkError: + if not cacheable_endpoint(method, url, body): + self.counters.increment("bypass") + self.counters.increment("upstream_attempts") + return forward_stream(method, url, headers=headers, body=body, timeout=timeout) + prepared: Final = prepare_forward(method, url, headers, body) + if isinstance(prepared, NetworkError): + self.counters.increment("rejected") + return prepared + key: Final = exact_key(self.secret, method, url, prepared.headers, body) + found: Final = self.lookup(key) + if isinstance(found, CacheHit): + response: Final = decode_response(self.secret, key, found.payload, url) + if response is not None and self.clock() < found.valid_until: + self.counters.increment("hits") + return StreamHead(response.status_code, response.headers, response_steps(response)) + self.counters.increment("corrupt" if response is None else "expired") + self.store.discard(key, found.payload) + capture_slot: Final = self.lookup(key) if isinstance(found, CacheHit) else found + self.counters.increment("misses") + if isinstance(capture_slot, CacheUnavailable): + self.counters.increment("cache_errors") + self.counters.increment("upstream_attempts") + head: Final = forward_prepared_stream(prepared, timeout) + if not isinstance(capture_slot, CaptureLease): + return head + if isinstance(head, NetworkError): + self.store.release(key, capture_slot) + self.counters.increment("rejected") + return head + return StreamHead(head.status_code, head.headers, primed_steps(self.capture(key, capture_slot, url, head))) + + def capture(self, key: str, lease: CaptureLease, url: str, head: StreamHead) -> Generator[StreamStep, None, None]: + capture: Final = ResponseCapture() + try: + with closing(head.steps): + yield StreamChunk(b"") + for step in head.steps: + yield step + capture.observe(step) + chunks: Final = capture.chunks() if capture.eligible else () + headers: Final = { + name: value for name, value in head.headers.items() if name.lower() not in UNRECORDED_RESPONSE_HEADERS + } + if not capture.eligible or not successful_response(url, head.status_code, headers, b"".join(chunks)): + self.counters.increment("rejected") + return + response: Final = CachedResponse( + request_key=key, status_code=head.status_code, headers=headers, + chunks=tuple(base64.b64encode(chunk).decode("ascii") for chunk in chunks), + ) + published: Final = self.store.publish(key, lease, encode_response(self.secret, response)) + self.counters.increment("writes" if published else "write_failures") + finally: + self.store.release(key, lease) + capture.buffer.close() diff --git a/tests/e2e/provider_cache_redis.py b/tests/e2e/provider_cache_redis.py new file mode 100644 index 00000000000..be4e31b2c49 --- /dev/null +++ b/tests/e2e/provider_cache_redis.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import atexit +import functools +import json +import logging +import os +import re +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Protocol, cast + +from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease +from pydantic import TypeAdapter, ValidationError +from redis import Redis +from redis.exceptions import RedisError + +REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes]) + +LOOKUP: Final = """ +local clock = redis.call('TIME') +local now = clock[1] * 1000 + math.floor(clock[2] / 1000) +local row = redis.call('HMGET', KEYS[1], 'captured', 'expires', 'payload') +if row[3] then + local captured = tonumber(row[1]) + local expires = tonumber(row[2]) + if captured and expires and captured <= now and expires > now + and expires - captured == tonumber(ARGV[2]) then + return {'hit', row[3], tostring(expires - now)} + end + redis.call('DEL', KEYS[1]) +end +if redis.call('SET', KEYS[2], ARGV[1], 'NX', 'PX', ARGV[3]) then + return {'lease', tostring(now), tostring(now + tonumber(ARGV[2]))} +end +return {'busy'} +""" + +PUBLISH: Final = """ +if redis.call('GET', KEYS[2]) ~= ARGV[1] then return 0 end +local clock = redis.call('TIME') +local now = clock[1] * 1000 + math.floor(clock[2] / 1000) +local captured = tonumber(ARGV[2]) +local expires = tonumber(ARGV[3]) +if captured > now or expires <= now or expires - captured ~= tonumber(ARGV[5]) then return 0 end +if redis.call('EXISTS', KEYS[1]) == 1 then return 0 end +redis.call('HSET', KEYS[1], 'captured', ARGV[2], 'expires', ARGV[3], 'payload', ARGV[4]) +redis.call('PEXPIREAT', KEYS[1], expires) +redis.call('DEL', KEYS[2]) +return 1 +""" + +RELEASE: Final = """ +if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end +return redis.call('DEL', KEYS[1]) +""" + +DISCARD: Final = """ +if redis.call('HGET', KEYS[1], 'payload') ~= ARGV[1] then return 0 end +return redis.call('DEL', KEYS[1]) +""" + + +class RedisCommands(Protocol): + def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object: ... + + +@dataclass(frozen=True, slots=True) +class RedisResponseStore: + client: RedisCommands + namespace: str + lifetime_ms: int = LIFETIME_SECONDS * 1000 + lease_ms: int = 120_000 + + def keys(self, key: str) -> tuple[str, str]: + prefix: Final = f"e2e-provider-cache:v1:{self.namespace}:{{{key}}}" + return prefix + ":response", prefix + ":lease" + + def lookup(self, key: str) -> CacheLookup: + token: Final = uuid.uuid4().hex + started: Final = time.monotonic() + try: + result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms) + except (RedisError, OSError): + return CacheUnavailable() + try: + parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True)) + except ValidationError: + return CacheUnavailable() + if len(parts) == 3 and parts[0] == b"hit" and parts[2].isdigit(): + return CacheHit(parts[1], started + int(parts[2]) / 1000) + if len(parts) == 3 and parts[0] == b"lease" and parts[1].isdigit() and parts[2].isdigit(): + return CaptureLease(token, int(parts[1]), int(parts[2])) + if parts == (b"busy",): + return CacheBusy() + return CacheUnavailable() + + def publish(self, key: str, lease: CaptureLease, payload: bytes) -> bool: + try: + result: Final = self.client.eval( + PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms, + ) + except (RedisError, OSError): + return False + return result == 1 + + def release(self, key: str, lease: CaptureLease) -> bool: + try: + result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token) + except (RedisError, OSError): + return False + return result == 1 + + def discard(self, key: str, payload: bytes) -> bool: + try: + result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload) + except (RedisError, OSError): + return False + return result == 1 + + +def redis_store(url: str, namespace: str) -> RedisResponseStore: + client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False) + return RedisResponseStore(cast(RedisCommands, client), namespace) + + +def write_metrics(cache: CacheEdge) -> None: + report: Final = json.dumps({"provider_cache": dict(cache.counters.counts)}) + directory: Final = os.environ.get("E2E_PROVIDER_CACHE_METRICS_DIR") + if directory: + try: + root: Final = Path(directory) + root.mkdir(parents=True, exist_ok=True) + (root / f"{os.getpid()}.json").write_text(report + "\n") + except OSError: + logging.getLogger(__name__).warning("provider cache metrics artifact unavailable") + logging.getLogger(__name__).info("%s", report) + + +@functools.lru_cache(maxsize=1) +def configured_cache() -> CacheEdge | None: + if os.environ.get("E2E_PROVIDER_CACHE", "0") == "0": + return None + if os.environ.get("E2E_PROVIDER_CACHE") != "1": + raise ValueError("E2E_PROVIDER_CACHE must be 0 or 1") + secret: Final = os.environ.get("E2E_PROVIDER_CACHE_HMAC_KEY", "").encode() + namespace: Final = os.environ.get("E2E_PROVIDER_CACHE_NAMESPACE", "") + if len(secret) < 32 or re.fullmatch(r"[a-zA-Z0-9_-]{1,64}", namespace) is None: + raise ValueError("provider cache requires a dedicated key and namespace") + cache: Final = CacheEdge(redis_store(os.environ["E2E_PROVIDER_CACHE_REDIS_URL"], namespace), secret) + atexit.register(write_metrics, cache) + return cache diff --git a/tests/e2e/provider_cache_routing.py b/tests/e2e/provider_cache_routing.py new file mode 100644 index 00000000000..24599b5a313 --- /dev/null +++ b/tests/e2e/provider_cache_routing.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from collections.abc import Callable +from contextvars import ContextVar +from typing import Final + +from models import LiteLLMParamsBody, ModelMode + +LIVE_PROVIDER_REQUIRED: Final[ContextVar[bool]] = ContextVar("live_provider_required", default=False) + + +def route_cache_model( + params: LiteLLMParamsBody, base_for: Callable[[str], str | None], *, enabled: bool, mode: ModelMode | None = None, +) -> LiteLLMParamsBody: + if not enabled or mode == "realtime" or LIVE_PROVIDER_REQUIRED.get() or params.api_base is not None or params.mock_response is not None: + return params + provider: Final = params.model.partition("/")[0] + if provider not in {"openai", "anthropic"} or params.litellm_credential_name is not None: + return params + base: Final = base_for(provider) + if base is None: + return params + return params.model_copy(update={"api_base": f"{base}/v1" if provider == "openai" else base}) diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index de36895ebb6..dda9e6f8e4f 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -42,6 +42,7 @@ import base64 import difflib import functools import hashlib +import os import re import threading from collections import deque @@ -93,6 +94,8 @@ from fixture_mode import ( parse_fixture_mode, ) from fixture_profile import IneligibleRequest, MatchProfile, match_profile, strict_identity +from provider_cache import CacheEdge +from provider_cache_routing import LIVE_PROVIDER_REQUIRED from pydantic import JsonValue, TypeAdapter EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( @@ -506,7 +509,7 @@ class LiveEdge: pass -type EdgeBackend = RecordEdge | ReplayEdge | LiveEdge +type EdgeBackend = RecordEdge | ReplayEdge | LiveEdge | CacheEdge @dataclass(slots=True) @@ -750,12 +753,16 @@ def _handle_record( def _handle_live( - method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float + method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float, + cache: CacheEdge | None = None, ) -> EdgeOutcome: forwarded: Final = { name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS } - head: Final = forward_stream(method, url, headers=forwarded, body=body, timeout=timeout) + head: Final = ( + forward_stream(method, url, headers=forwarded, body=body, timeout=timeout) + if cache is None else cache.forward(method, url, forwarded, body, timeout) + ) match head: case NetworkError(message=message): return _recorded_outcome(_network_error_response(message)) @@ -821,6 +828,10 @@ def handle_edge_request( else edge_request(method, split.path, split.query, body, _header_value(headers, "content-type")) ) match backend: + case CacheEdge(): + return _handle_live( + method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout, backend, + ) case LiveEdge(): return _handle_live( method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout @@ -871,11 +882,19 @@ class _EdgeHandler(BaseHTTPRequestHandler): or isinstance(edge_server.backend, ReplayEdge) and edge_server.backend.source.bundle.manifest.match_profile == "stateless_v1" ) - if strict and len({name.lower() for name in self.headers.keys()}) != len(self.headers): + if strict and len({name.lower() for name in self.headers}) != len(self.headers): self._write_reply(_text_reply(REPLAY_MISS_STATUS, "stateless_v1 eligibility error: duplicate headers")) return + duplicate_headers: Final = len({name.lower() for name in self.headers}) != len(self.headers) + selected_backend: Final = ( + LiveEdge() if isinstance(edge_server.backend, CacheEdge) and duplicate_headers else edge_server.backend + ) + if isinstance(edge_server.backend, CacheEdge) and duplicate_headers: + edge_server.backend.counters.increment("duplicate_header_bypass") + if urlsplit(self.path).path.lstrip("/").partition("/")[0] in edge_server.mounts: + edge_server.backend.counters.increment("upstream_attempts") outcome: Final = handle_edge_request( - edge_server.backend, + selected_backend, edge_server.mounts, self.command, self.path, @@ -908,12 +927,12 @@ class _EdgeHandler(BaseHTTPRequestHandler): shuts down write-side first: the proxy sees a graceful close mid-message, which is the incomplete chunked read a provider hanging up produces, and not the reset that could discard the chunks already in flight.""" - self.send_response(stream.status_code) - for name, value in stream.headers.items(): - self.send_header(name, value) - self.send_header("transfer-encoding", "chunked") - self.end_headers() with closing(stream.steps) as steps: + self.send_response(stream.status_code) + for name, value in stream.headers.items(): + self.send_header(name, value) + self.send_header("transfer-encoding", "chunked") + self.end_headers() for step in steps: match step: case StreamChunk(data=data): @@ -923,7 +942,7 @@ class _EdgeHandler(BaseHTTPRequestHandler): return case _: assert_never(step) - self.wfile.write(b"0\r\n\r\n") + self.wfile.write(b"0\r\n\r\n") def log_message(self, format: str, *args: object) -> None: """Silence the per-request stderr line BaseHTTPRequestHandler emits.""" @@ -1056,6 +1075,8 @@ def provider_edge_api_base( case InvalidFixtureMode(value=value): raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") case "live": + if configured_cache_backend() is not None: + return _shared_cache_edge(bind_host, advertise_host, forward_timeout).api_base(mount) return None case "record" | "replay": if mount not in EDGE_MOUNTS: @@ -1073,7 +1094,7 @@ def _observed_backend(mode_raw: str, bundle_dir: Path) -> EdgeBackend: case InvalidFixtureMode(value=value): raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") case "live": - return LiveEdge() + return configured_cache_backend() or LiveEdge() case "record": return RecordEdge(_shared_recorder(bundle_dir, match_profile()), threading.Lock()) case "replay": @@ -1082,6 +1103,24 @@ def _observed_backend(mode_raw: str, bundle_dir: Path) -> EdgeBackend: assert_never(mode) +def configured_cache_backend() -> CacheEdge | None: + if LIVE_PROVIDER_REQUIRED.get() or os.environ.get("E2E_PROVIDER_CACHE", "0") == "0": + return None + from provider_cache_redis import configured_cache + + return configured_cache() + + +@functools.lru_cache(maxsize=8) +def _shared_cache_edge(bind_host: str, advertise_host: str, forward_timeout: float) -> ProviderEdge: + backend: Final = configured_cache_backend() + assert backend is not None + return start_provider_edge( + backend, mounts=EDGE_MOUNTS, bind_host=bind_host, + advertise_host=advertise_host, forward_timeout=forward_timeout, + ).edge + + @contextmanager def observed_provider_edge( observation: ProviderRequestObservation, diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 3f7fba5ffec..f8ed8843461 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -8,6 +8,7 @@ ProxyClient's key/customer methods for cleanup. Read-backs are eventually consis from __future__ import annotations +import os import time import warnings from collections.abc import Callable, Mapping @@ -26,6 +27,7 @@ from e2e_config import ( PROXY_REPLICA_URLS, REQUEST_TIMEOUT, SLOW_PROVIDER_TIMEOUT_SECONDS, + provider_edge_base, settle_propagation, ) from e2e_http import ( @@ -93,6 +95,7 @@ from models import ( UserDeleteBody, UserDeleteResponse, ) +from provider_cache_routing import route_cache_model from pydantic import BaseModel from transport import HttpTransport, SplitTransport, Transport, is_control_plane_path @@ -645,7 +648,10 @@ class ProxyClient: self.transport.post( "/model/new", headers=self.management_headers(), - json=body, + json=body.model_copy(update={"litellm_params": route_cache_model( + body.litellm_params, provider_edge_base, + enabled=os.environ.get("E2E_PROVIDER_CACHE", "0") == "1", mode=body.model_info.mode, + )}), response_type=ModelNewResponse, ) ).model_id diff --git a/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py b/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py index b0bc6b3508c..33d869ee80e 100644 --- a/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py @@ -26,7 +26,7 @@ from models import ( ) from quota_client import QuotaClient -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] # Anthropic prompt caching (host has ANTHROPIC_API_KEY; Bedrock was "Operation not allowed"). ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" diff --git a/tests/e2e/test_provider_edge.py b/tests/e2e/test_provider_edge.py index 81be81e7b59..5d0c79f26f6 100644 --- a/tests/e2e/test_provider_edge.py +++ b/tests/e2e/test_provider_edge.py @@ -1254,7 +1254,8 @@ class TestHandleEdgeRequestPure: class TestApiBaseSeam: - def test_live_mode_returns_none(self, tmp_path: Path) -> None: + def test_live_mode_returns_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("E2E_PROVIDER_CACHE", raising=False) for mode_raw in ("live", ""): assert ( provider_edge_api_base( diff --git a/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts b/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts index 5e2c80b5845..736c352e3ee 100644 --- a/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts +++ b/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts @@ -7,6 +7,7 @@ import { import { E2E_TEAM_CRUD_ALIAS, E2E_TEAM_ORG_ALIAS, + E2E_TEAM_ORG_ID, INTERNAL_USER_STORAGE_PATH, } from "../../constants"; import { Page } from "../../fixtures/pages"; @@ -179,18 +180,28 @@ test.describe("Models and Endpoints for an internal user", () => { `switching to ${ALL_MODELS_VIEW} leaves the table populated rather than blanking it`, ).toHaveCount(1, { timeout: 15_000 }); + await expect(page).toHaveURL((url) => + url.searchParams.get("filter_team") === E2E_TEAM_ORG_ID && + url.searchParams.get("view_mode") === "all", + ); await page.reload(); await expect( teamSelector(page), - "the team selection is not persisted across a reload, so the table returns to the personal view", - ).toContainText(PERSONAL_TEAM, { timeout: 15_000 }); + "the selected team is restored from the URL after a reload", + ).toContainText(E2E_TEAM_ORG_ALIAS, { timeout: 15_000 }); await expect( viewSelector(page), - "the view selection is not persisted across a reload either", - ).toContainText(CURRENT_TEAM_VIEW, { timeout: 15_000 }); + "the selected view is restored from the URL after a reload", + ).toContainText(ALL_MODELS_VIEW, { timeout: 15_000 }); + await expect(modelRow(page, CHAT_MODEL_A)).toHaveCount(1, { timeout: 15_000 }); + await expect(page.getByTestId("pagination-range")).toHaveText("Showing 1-1 of 1"); + await expect(modelRow(page, CHAT_MODEL_B)).toHaveCount(0); + await expect(modelRow(page, ungrantedModelName)).toHaveCount(0); + + await chooseOption(page, teamSelector(page), PERSONAL_TEAM); await expect( modelRow(page, ungrantedModelName), - "the personal view still renders models after a reload rather than coming back empty", + "switching back to the personal team restores models outside the selected team", ).toHaveCount(1, { timeout: 30_000 }); }); }); diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index 4103536950d..fe3c38a771f 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -163,6 +163,9 @@ async def test_reset_budget_keys_partial_failure(): key1, key2, key3, key4, key5, key6 = ( _attrify(k) for k in [key1, key2, key3, key4, key5, key6] ) + pre_reset_spend = { + k["token"]: k["spend"] for k in [key2, key3, key4, key5, key6] + } prisma_client.get_data = AsyncMock( return_value=[key1, key2, key3, key4, key5, key6] ) @@ -201,7 +204,7 @@ async def test_reset_budget_keys_partial_failure(): # And every write must carry only {spend, budget_reset_at} — never the full row. for c in key_writes: assert set(c["data"].keys()) == {"spend", "budget_reset_at"} - assert c["data"]["spend"] == 0 + assert c["data"]["spend"] == {"decrement": pre_reset_spend[c["where"]["token"]]} # Verify that the failure logging hook was scheduled (due to the failure for key1) failure_hook_calls = ( @@ -252,6 +255,9 @@ async def test_reset_budget_users_partial_failure(): user1, user2, user3, user4, user5, user6 = ( _attrify(u) for u in [user1, user2, user3, user4, user5, user6] ) + pre_reset_spend = { + u["user_id"]: u["spend"] for u in [user2, user3, user4, user5, user6] + } prisma_client.get_data = AsyncMock( return_value=[user1, user2, user3, user4, user5, user6] ) @@ -280,7 +286,9 @@ async def test_reset_budget_users_partial_failure(): assert written_ids == ["user2", "user3", "user4", "user5", "user6"] for c in user_writes: assert set(c["data"].keys()) == {"spend", "budget_reset_at"} - assert c["data"]["spend"] == 0 + assert c["data"]["spend"] == { + "decrement": pre_reset_spend[c["where"]["user_id"]] + } failure_hook_calls = ( proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args_list @@ -441,6 +449,7 @@ async def test_reset_budget_teams_partial_failure(): for t in [team1, team2]: t.setdefault("team_id", t["id"]) team1, team2 = _attrify(team1), _attrify(team2) + pre_reset_spend = team2["spend"] prisma_client.get_data = AsyncMock(return_value=[team1, team2]) async def fake_reset_team(team, current_time, reset_settings=None): @@ -465,7 +474,7 @@ async def test_reset_budget_teams_partial_failure(): assert len(team_writes) == 1 assert team_writes[0]["where"] == {"team_id": "team2"} assert set(team_writes[0]["data"].keys()) == {"spend", "budget_reset_at"} - assert team_writes[0]["data"]["spend"] == 0 + assert team_writes[0]["data"]["spend"] == {"decrement": pre_reset_spend} failure_hook_calls = ( proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args_list @@ -542,6 +551,11 @@ async def test_reset_budget_continues_other_categories_on_failure(): user1, user2 = _attrify(user1), _attrify(user2) team1, team2 = _attrify(team1), _attrify(team2) enduser1 = _attrify(enduser1) + pre_reset_spend = { + **{k["token"]: k["spend"] for k in [key1, key2]}, + **{u["user_id"]: u["spend"] for u in [user2]}, + **{t["team_id"]: t["spend"] for t in [team1, team2]}, + } _wire_cascade_reads_for_test(prisma_client) proxy_logging_obj = MagicMock() @@ -618,7 +632,9 @@ async def test_reset_budget_continues_other_categories_on_failure(): # Every batched write must carry only the two reset fields, never the full row. for c in key_writes + user_writes + team_writes: assert set(c["data"].keys()) == {"spend", "budget_reset_at"} - assert c["data"]["spend"] == 0 + assert c["data"]["spend"] == { + "decrement": pre_reset_spend[next(iter(c["where"].values()))] + } # --------------------------------------------------------------------------- diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 5535a62bb81..228457f4d55 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -75,6 +75,9 @@ _VCR_INCOMPATIBLE_FILES = frozenset( "test_router_caching.py", # Hits the local fake OpenAI endpoint on 127.0.0.1; nothing to record. "test_fake_openai_endpoint.py", + # Needs the real connection pool a collected handler tears down; vcrpy + # patches the transport that pool lives in. + "test_handler_gc_does_not_close_client.py", } ) diff --git a/tests/local_testing/test_handler_gc_does_not_close_client.py b/tests/local_testing/test_handler_gc_does_not_close_client.py new file mode 100644 index 00000000000..1a6ab1b1827 --- /dev/null +++ b/tests/local_testing/test_handler_gc_does_not_close_client.py @@ -0,0 +1,315 @@ +""" +Collecting an HTTP handler must not abort a response that is still on the wire. + +``HTTPHandler`` and ``AsyncHTTPHandler`` close their client from ``__del__``. +Closing a client tears down the connection pool, which aborts every response +still streaming through it. ``_handler_may_close_client`` already withholds the +close from a client someone else holds, but a streaming response holds the +connection it is reading from and never the client, so the refcount it reads +says "sole referrer" for exactly the client that is busiest. The handler is +routinely collectable at that moment: a provider's streaming call returns the +response and drops the handler, and ``get_async_httpx_client`` caches handlers +behind a one-hour TTL and then lets them go. + +The fix anchors the handler to the streaming response, so these tests turn on +*when* the handler is collected rather than on whether it is: pinned while the +body can still arrive, released once the caller is done with the response. + +Nothing here re-tests the shapes ``_handler_may_close_client`` covers -- a +borrowed ``handler.client``, a caller-supplied client, an evicted-but-held +client. Those are pinned in ``tests/test_litellm/llms/custom_httpx/ +test_http_handler.py``. What is uncovered there is the in-flight response, so no +test here may keep the client in a local: that inflates the very refcount under +test, and the test then passes on a broken handler. They hold weak references +instead, which the refcount does not count. + +These live here rather than under ``tests/test_litellm/`` because they need a +real connection pool: a mocked transport goes on yielding chunks after its +client is closed, so the very teardown under test is what a mock cannot +reproduce. The server is a hermetic, credential-free ``ThreadingHTTPServer`` on +an ephemeral loopback port, and needs no network access beyond it. + +Related: https://github.com/BerriAI/litellm/issues/24929 +""" + +import asyncio +import gc +import threading +import time +import weakref +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import httpx +import pytest + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) +from litellm.types.utils import LlmProviders + +FRAME_COUNT = 6 +# Generous: the server emits all frames in ~0.3s. A client whose pool was torn +# down mid-stream can stall silently instead of raising, so reads are bounded. +READ_TIMEOUT_SECONDS = 15.0 +RELEASE_TIMEOUT_SECONDS = 3.0 + +BOTH_TRANSPORTS = pytest.mark.parametrize("disable_aiohttp_transport", [False, True], ids=["aiohttp", "httpcore"]) + +STILL_PINNED = "the handler was released while its response could still read" +NOT_RELEASED = "the handler outlived the response that was holding it" + + +class _ChunkedSSEServer: + """In-process HTTP/1.1 server that answers every request with chunked SSE frames.""" + + def __init__(self, frame_count: int = FRAME_COUNT, frame_delay: float = 0.05) -> None: + self.frame_count = frame_count + self.frame_delay = frame_delay + parent = self + + class _Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _stream(self): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + try: + for index in range(parent.frame_count): + frame = f"data: frame-{index}\n\n".encode() + self.wfile.write(b"%x\r\n" % len(frame) + frame + b"\r\n") + self.wfile.flush() + time.sleep(parent.frame_delay) + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + pass + + do_GET = _stream + do_POST = _stream + + def log_message(self, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + self.url = f"http://127.0.0.1:{self._server.server_address[1]}/stream" + + def __enter__(self): + threading.Thread(target=self._server.serve_forever, daemon=True).start() + return self + + def __exit__(self, *exc_info): + self._server.shutdown() + self._server.server_close() + + +def _select_transport(monkeypatch, disable_aiohttp_transport: bool) -> None: + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", disable_aiohttp_transport) + monkeypatch.setattr(litellm, "force_ipv4", False) + + +async def _read_frames(response: httpx.Response) -> int: + """Count SSE frames, collecting garbage between chunks so a finalizer has every chance to fire. + + The body is joined before counting: a chunk boundary can fall inside the + marker, which a per-chunk count would miss. + """ + chunks = [] + async for chunk in response.aiter_bytes(): + chunks.append(chunk) + gc.collect() + return b"".join(chunks).count(b"data: frame-") + + +async def _wait_until(is_done, failure: str) -> None: + deadline = time.monotonic() + RELEASE_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if is_done(): + return + await asyncio.sleep(0.05) + pytest.fail(failure) + + +@pytest.mark.asyncio +@BOTH_TRANSPORTS +async def test_async_stream_survives_handler_collection(monkeypatch, disable_aiohttp_transport): + """A response still streaming keeps working after its handler goes out of scope. + + The caller holds the response and nothing else, which is what a provider's + streaming path is left with once ``post(..., stream=True)`` has returned. + """ + _select_transport(monkeypatch, disable_aiohttp_transport) + + with _ChunkedSSEServer() as server: + handler = AsyncHTTPHandler(timeout=httpx.Timeout(10.0, connect=5.0)) + response = await handler.post(server.url, stream=True) + + ref = weakref.ref(handler) + del handler + gc.collect() + await asyncio.sleep(0) # let any close the finalizer scheduled run + + assert ref() is not None, STILL_PINNED + assert await asyncio.wait_for(_read_frames(response), timeout=READ_TIMEOUT_SECONDS) == FRAME_COUNT + + del response + gc.collect() + assert ref() is None, NOT_RELEASED + + +def test_sync_stream_survives_handler_collection(monkeypatch): + """The sync handler closes inline from its finalizer, so a stream must hold it off. + + litellm/main.py builds a sync handler only for non-streaming calls, commented + "Keep this here, otherwise, the httpx.client closes and streaming is + impossible" -- a workaround for this finalizer rather than a fix for it. + """ + monkeypatch.setattr(litellm, "force_ipv4", False) + + with _ChunkedSSEServer() as server: + handler = HTTPHandler(timeout=httpx.Timeout(10.0, connect=5.0)) + response = handler.post(server.url, stream=True) + + ref = weakref.ref(handler) + del handler + gc.collect() + assert ref() is not None, STILL_PINNED + + # Joined before counting, as in ``_read_frames``. + chunks = [] + for chunk in response.iter_bytes(): + chunks.append(chunk) + gc.collect() + assert b"".join(chunks).count(b"data: frame-") == FRAME_COUNT + + del response + gc.collect() + assert ref() is None, NOT_RELEASED + + +@pytest.mark.asyncio +@BOTH_TRANSPORTS +async def test_an_abandoned_stream_still_releases_its_handler(monkeypatch, disable_aiohttp_transport): + """A caller that drops a stream unread must not pin the handler for good. + + Tying the handler to the response's own lifetime is what bounds this. No + deadline, and no poll of the connection's state, can tell an abandoned body + from one the upstream is merely slow to finish: httpx leaves the connection + checked out until the response is read or closed, and a legitimate stream is + bounded only by how long the upstream keeps sending. + """ + _select_transport(monkeypatch, disable_aiohttp_transport) + + with _ChunkedSSEServer() as server: + handler = AsyncHTTPHandler(timeout=httpx.Timeout(10.0, connect=5.0)) + client_ref = weakref.ref(handler.client) + response = await handler.post(server.url, stream=True) + + ref = weakref.ref(handler) + del handler, response + gc.collect() + + assert ref() is None, NOT_RELEASED + await _wait_until( + lambda: client_ref() is None or client_ref().is_closed, + "the client outlived the abandoned stream without being closed", + ) + + +@pytest.mark.asyncio +@BOTH_TRANSPORTS +async def test_the_pool_is_released_once_the_stream_it_carried_ends(monkeypatch, disable_aiohttp_transport): + """Holding the finalizer off must defer the close, not drop it. + + Otherwise a collected handler leaks its pool for every streaming request it + was carrying, and on aiohttp warns "Unclosed client session" when the + collector eventually takes it. The pool and the session are children of the + client, so keeping one here does not inflate the refcount the finalizer + reads, the way keeping the client would. + """ + _select_transport(monkeypatch, disable_aiohttp_transport) + + with _ChunkedSSEServer() as server: + handler = AsyncHTTPHandler(timeout=httpx.Timeout(10.0, connect=5.0)) + transport = handler.client._transport + if disable_aiohttp_transport: + pool = transport._pool + + def is_released() -> bool: + return pool.connections == [] + else: + session = transport._get_valid_client_session() + + def is_released() -> bool: + return session.closed + + response = await handler.post(server.url, stream=True) + + del handler, transport + gc.collect() + assert not is_released(), "the pool was torn down while it was still carrying a body" + + assert await asyncio.wait_for(_read_frames(response), timeout=READ_TIMEOUT_SECONDS) == FRAME_COUNT + del response + gc.collect() + + await _wait_until(is_released, "the pool outlived the stream it carried, unclosed") + + +@pytest.mark.asyncio +@BOTH_TRANSPORTS +async def test_a_non_streaming_response_does_not_pin_its_handler(monkeypatch, disable_aiohttp_transport): + """Only a body that can still arrive holds the handler. + + A non-streaming response has been read in full by the time ``post`` returns, + so pinning the handler to it would delay every client close behind whatever + the caller goes on to do with the response. + """ + _select_transport(monkeypatch, disable_aiohttp_transport) + + with _ChunkedSSEServer(frame_count=1, frame_delay=0.0) as server: + handler = AsyncHTTPHandler(timeout=httpx.Timeout(10.0, connect=5.0)) + response = await handler.post(server.url) + assert response.status_code == 200 + + ref = weakref.ref(handler) + del handler + gc.collect() + + assert ref() is None, "a fully-read response pinned its handler" + + +@pytest.mark.asyncio +@BOTH_TRANSPORTS +async def test_cached_handler_eviction_does_not_abort_an_in_flight_stream(monkeypatch, disable_aiohttp_transport): + """Evicting a cached handler mid-stream leaves the stream alone. + + ``get_async_httpx_client`` caches handlers for an hour. When that TTL + expires the cache drops the only reference to a handler whose client is + still streaming -- the production shape of #24929. + """ + _select_transport(monkeypatch, disable_aiohttp_transport) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + with _ChunkedSSEServer() as server: + handler = get_async_httpx_client(llm_provider=LlmProviders.OPENAI) + response = await handler.post(server.url, stream=True) + + # An hour passes: the TTL expires and the cache lets the handler go. + ref = weakref.ref(handler) + litellm.in_memory_llm_clients_cache.flush_cache() + del handler + gc.collect() + + assert ref() is not None, STILL_PINNED + assert await asyncio.wait_for(_read_frames(response), timeout=READ_TIMEOUT_SECONDS) == FRAME_COUNT + + del response + gc.collect() + assert ref() is None, NOT_RELEASED diff --git a/tests/local_testing/test_router_debug_logs.py b/tests/local_testing/test_router_debug_logs.py index 0fce5c824c7..b3c26a7689f 100644 --- a/tests/local_testing/test_router_debug_logs.py +++ b/tests/local_testing/test_router_debug_logs.py @@ -86,6 +86,7 @@ def test_async_fallbacks(caplog): if "Task exception was never retrieved" not in log and "Task was destroyed but it is pending" not in log and "get_available_deployment" not in log + and "Selected deployment for model" not in log and "in the Langfuse queue" not in log and "Unclosed client session" not in log and "Unclosed connector" not in log diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index e5e93c0b179..6017a820299 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -101,7 +101,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success): assert _error_body["type"] == "key_model_access_denied" assert _error_body["param"] == "model" assert _error_body["code"] == "403" - assert "key not allowed to access model" in _error_body["message"] + assert "is not available for this API key" in _error_body["message"] @pytest.mark.asyncio @@ -299,7 +299,5 @@ def _validate_model_access_exception( assert _error_body["type"] == expected_type assert _error_body["param"] == "model" assert _error_body["code"] == "403" - if expected_type == "key_model_access_denied": - assert "key not allowed to access model" in _error_body["message"] - elif expected_type == "team_model_access_denied": - assert "eam not allowed to access model" in _error_body["message"] + assert "is not available for this API key" in _error_body["message"] + assert "not allowed to access model" not in _error_body["message"] diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index d436c99cd20..2538556d3b5 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -163,7 +163,7 @@ async def test_can_key_call_model(model, expect_to_work): if expect_to_work: await can_key_call_model(**args) else: - with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: + with pytest.raises(Exception, match='is not available for this API key') as e: await can_key_call_model(**args) print(e) @@ -943,7 +943,7 @@ async def test_can_key_call_model_with_aliases(model, alias_map, expect_to_work) llm_router=router, ) else: - with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: + with pytest.raises(Exception, match='is not available for this API key') as e: await can_key_call_model( model=model, llm_model_list=llm_model_list, diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index 0cdf3500d50..a8fce58c60b 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -1069,6 +1069,7 @@ async def test_jwt_non_admin_team_route_access(monkeypatch): mock_jwt_response = { "is_proxy_admin": False, + "jwt_claims": {}, "team_id": None, "team_object": None, "user_id": None, diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md index 71e17541fd2..b66eaaeda9b 100644 --- a/tests/rust-python-harness/AGENTS.md +++ b/tests/rust-python-harness/AGENTS.md @@ -25,17 +25,6 @@ tests/rust-python-harness/ │ │ ├── ocr/ │ │ └── transcription/ │ │ -│ ├── unit_tests_mapping/ -│ │ ├── __init__.py -│ │ ├── contracts.py -│ │ ├── cases/ -│ │ │ └── ocr.py -│ │ ├── mapping_report.py -│ │ ├── mappings.py -│ │ ├── mapping_validator.py -│ │ ├── reporting.py -│ │ └── runner.py -│ │ │ ├── unit_tests_parity/ │ │ ├── __init__.py │ │ ├── reporting.py @@ -52,6 +41,7 @@ tests/rust-python-harness/ ├── reporting/ │ └── strategy.py └── unit_runners/ + ├── contracts.py └── suite_runner.py ``` @@ -63,10 +53,9 @@ tests/rust-python-harness/ - Examples: `run e2e_parity --surface sdk --function ocr`, `run unit_tests_parity --function ocr --pytest-arg=-x`, or `run all --function ocr` - `cli/catalog.py` discovers strategies, validates their Python definitions, and orders them; `cli/__init__.py` builds the Click command tree; `cli/commands.py` runs selected cases - `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses -- `trace_parity/` prints every collected Python call under `litellm/` and every Rust span without comparing them; mappings only filter the separate unit-test mapping strategy. Before running it rebuilds the native bridge with the `trace-parity` feature whenever `litellm-rust` sources are newer than the installed extension (`shared/native_build.py`) +- `trace_parity/` profiles the Python call stack and prints every collected Python call under `litellm/`; it never collects Rust spans and never rebuilds the native extension - E2E and trace strategies load their registered module cases and run surface-specific execution from their folders -- `unit_tests_mapping/contracts.py` owns typed harness-side mapping contracts, per-function contracts live below `cases/`, and `mappings.py` exports the registry; live test discovery derives unmapped Python and Rust-only tests without an exhaustive manifest -- `unit_tests_mapping/runner.py` validates confirmed mappings against the live Python and Rust inventories and attaches the derived status report +- `shared/unit_runners/contracts.py` owns the typed per-function unit contracts consumed by `unit_tests_parity` and `unit_tests_rust` - `unit_tests_parity/runner.py` runs each contract's `unit_parity_scope` with `LITELLM_RUST=0` and `LITELLM_RUST=1` in separate processes and requires matching outcomes, including failures; exclusions require a reason in the contract - `unit_tests_rust/runner.py` runs each contract's focused Cargo test suite; native Rust unit tests stay beside their implementation - `shared/unit_runners/suite_runner.py` runs typed suites registered in code with nodeids of the form `suite:::` @@ -74,4 +63,4 @@ tests/rust-python-harness/ - `shared/` contains reusable parity, tracing, reporting primitives, and unit-runner machinery - Keep fixtures with their owning API and existing Python tests in their current locations - Each strategy folder carries an `AGENTS.md` one-liner stating what it should be doing -- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/unit_tests_mapping tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust tests/test_rust_python_harness.py -q` +- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust tests/test_rust_python_harness.py -q` diff --git a/tests/rust-python-harness/cli/__init__.py b/tests/rust-python-harness/cli/__init__.py index d2bfdc55b19..13b995825dd 100644 --- a/tests/rust-python-harness/cli/__init__.py +++ b/tests/rust-python-harness/cli/__init__.py @@ -58,29 +58,16 @@ def _strategy_command(strategy: Strategy) -> click.Command: help=runner_argument.help, ) ) - for runner_option in strategy.definition.runner_options: - name: Final = runner_option.option.removeprefix("--").replace("-", "_") - params.append( - click.Option( - (runner_option.option, name), - type=click.Choice(runner_option.choices), - help=runner_option.help, - ) - ) def run_strategy( sdk_functions: tuple[str, ...], surface: str | None = None, runner_args: tuple[str, ...] = (), - **runner_options: str | None, ) -> int: selected_functions: Final = cast(frozenset[SdkFunction], frozenset(sdk_functions)) selected_surface: Final = cast(Surface | None, surface) cases: Final = select_cases((strategy,), selected_functions, selected_surface) - option_args: Final = tuple( - f"--{name.replace('_', '-')}={value}" for name, value in runner_options.items() if value is not None - ) - return run_command((strategy,), cases, (*runner_args, *option_args)) + return run_command((strategy,), cases, runner_args) return click.Command( strategy.id, diff --git a/tests/rust-python-harness/cli/test_cli.py b/tests/rust-python-harness/cli/test_cli.py index 5641aa8a539..219e1b0c6b7 100644 --- a/tests/rust-python-harness/cli/test_cli.py +++ b/tests/rust-python-harness/cli/test_cli.py @@ -19,7 +19,6 @@ from ..shared.reporting.models import ( ) from ..shared.reporting.strategy import NotImplementedCaseSpec, SkippedCaseSpec, StrategyDefinition from ..shared.reporting.ui import PlainDashboard, final_report, make_dashboard -from ..strategies.unit_tests_mapping.mappings import UNIT_TEST_CONTRACTS from ..strategies.unit_tests_parity import UNIT_PARITY_SUITES from ..strategies.unit_tests_rust import RUST_SUITES from . import main @@ -91,7 +90,6 @@ def test_should_load_surface_aware_and_function_only_strategies() -> None: assert [strategy.id for strategy in strategies] == [ "e2e_parity", "trace_parity", - "unit_tests_mapping", "unit_tests_parity", "unit_tests_rust", ] @@ -104,29 +102,23 @@ def test_should_load_surface_aware_and_function_only_strategies() -> None: def test_unit_strategies_use_function_only_cases() -> None: strategies: Final = { - strategy.id: strategy - for strategy in load_catalog() - if strategy.id in {"unit_tests_mapping", "unit_tests_parity", "unit_tests_rust"} + strategy.id: strategy for strategy in load_catalog() if strategy.id in {"unit_tests_parity", "unit_tests_rust"} } for sdk_function in SDK_FUNCTIONS: cases: Final = tuple( case for strategy in strategies.values() for case in strategy.cases if case.sdk_function == sdk_function ) - assert len(cases) == 3 + assert len(cases) == 2 assert all(case.surface is None for case in cases) - expected_mapping: Final = ( - CaseDisposition.RUNNABLE if sdk_function in UNIT_TEST_CONTRACTS else CaseDisposition.NOT_IMPLEMENTED - ) - assert cases[0].spec.disposition is expected_mapping expected_parity: Final = ( CaseDisposition.RUNNABLE if sdk_function in UNIT_PARITY_SUITES else CaseDisposition.NOT_IMPLEMENTED ) expected_rust: Final = ( CaseDisposition.RUNNABLE if sdk_function in RUST_SUITES else CaseDisposition.NOT_IMPLEMENTED ) - assert cases[1].spec.disposition is expected_parity - assert cases[2].spec.disposition is expected_rust + assert cases[0].spec.disposition is expected_parity + assert cases[1].spec.disposition is expected_rust def test_raw_dashboard_is_always_the_default() -> None: @@ -243,7 +235,6 @@ def test_every_unavailable_case_finishes_and_explains_itself() -> None: section_titles: Final = { "e2e_parity": "End-to-end parity outcomes", "trace_parity": "traces", - "unit_tests_mapping": "Python/Rust unit-test mappings", "unit_tests_parity": "Python backend parity outcomes", "unit_tests_rust": "Native Rust unit-test outcomes", } @@ -264,7 +255,6 @@ def test_every_unavailable_case_finishes_and_explains_itself() -> None: ("e2e_parity", "--surface", "--pytest-arg"), ("trace_parity", "--surface", "--pytest-arg"), ("unit_tests_parity", "--pytest-arg", "--surface"), - ("unit_tests_mapping", "--detail", "--surface"), ("unit_tests_rust", "--function", "--surface"), ), ) @@ -291,7 +281,6 @@ def test_run_help_lists_all_and_every_strategy(capsys: pytest.CaptureFixture[str "all", "e2e_parity", "trace_parity", - "unit_tests_mapping", "unit_tests_parity", "unit_tests_rust", ): @@ -359,7 +348,7 @@ def test_strategy_command_forwards_repeated_filters_and_runner_arguments( ] -def test_trace_command_forwards_engine_and_scenario(monkeypatch: pytest.MonkeyPatch) -> None: +def test_trace_command_forwards_scenario(monkeypatch: pytest.MonkeyPatch) -> None: cli: Final = importlib.import_module("tests.rust-python-harness.cli") captured: list[tuple[str, ...]] = [] @@ -374,8 +363,8 @@ def test_trace_command_forwards_engine_and_scenario(monkeypatch: pytest.MonkeyPa monkeypatch.setattr(cli, "run_command", capture_run) - assert main(["run", "trace_parity", "--scenario", "async-mistral", "--engine", "python"]) == 0 - assert captured == [("async-mistral", "--engine=python")] + assert main(["run", "trace_parity", "--scenario", "async-mistral"]) == 0 + assert captured == [("async-mistral",)] def test_omitted_surface_selects_every_strategy_surface(monkeypatch: pytest.MonkeyPatch) -> None: @@ -413,8 +402,8 @@ def test_run_all_selects_every_declared_case_once(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(cli, "run_command", capture_run) assert main(["run", "all", "--function", "ocr"]) == 0 - assert len(selected) == 7 - assert sum(case.surface is None for case in selected) == 3 + assert len(selected) == 6 + assert sum(case.surface is None for case in selected) == 2 assert sum(case.surface is not None for case in selected) == 4 diff --git a/tests/rust-python-harness/shared/native_build.py b/tests/rust-python-harness/shared/native_build.py deleted file mode 100644 index f67488cecb4..00000000000 --- a/tests/rust-python-harness/shared/native_build.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import importlib.util -import os -import subprocess -import sys -from collections.abc import Iterator -from pathlib import Path -from typing import Final - -from litellm.rust_bridge import get_native_bridge, reset_native_bridge_cache - -MATURIN_SPEC: Final = "maturin==1.15.0" -BRIDGE_FEATURE: Final = "trace-parity" -_RUST_ROOT: Final = "litellm-rust" -_LOCKFILE: Final = "Cargo.lock" -_SOURCE_SUFFIXES: Final = frozenset({".rs", ".toml"}) -_FAILURE_OUTPUT_LINES: Final = 15 -_TRACE_CHECK: Final = ( - "from litellm.rust_bridge import get_native_bridge; " - "bridge = get_native_bridge(); " - "raise SystemExit(0 if bridge is not None and getattr(bridge, '_trace', None) is not None else 1)" -) - - -def needs_rebuild(native_mtime: float | None, newest_source_mtime: float | None) -> bool: - if native_mtime is None: - return True - if newest_source_mtime is None: - return False - return newest_source_mtime > native_mtime - - -def _source_files(rust_root: Path) -> Iterator[Path]: - for path in rust_root.rglob("*"): - relative: Final = path.relative_to(rust_root) - if "target" in relative.parts or not path.is_file(): - continue - if path.name == _LOCKFILE or path.suffix in _SOURCE_SUFFIXES: - yield path - - -def _newest_source_mtime(repo_root: Path) -> float | None: - rust_root: Final = repo_root / _RUST_ROOT - if not rust_root.is_dir(): - return None - return max((path.stat().st_mtime for path in _source_files(rust_root)), default=None) - - -def _native_module_path() -> Path | None: - try: - spec: Final = importlib.util.find_spec("litellm.rust_bridge._native") - except (ImportError, ValueError): - return None - origin: Final = getattr(spec, "origin", None) - return Path(origin) if origin else None - - -def _drop_imported_bridge() -> None: - reset_native_bridge_cache() - for name in tuple(sys.modules): - if name.startswith("litellm.rust_bridge._native"): - del sys.modules[name] - - -def _rebuild(repo_root: Path) -> tuple[bool, str]: - command: Final = ("uvx", "--from", MATURIN_SPEC, "maturin", "develop", "--features", BRIDGE_FEATURE) - completed: Final = subprocess.run( - command, - cwd=repo_root, - env={**os.environ, "VIRTUAL_ENV": sys.prefix}, - capture_output=True, - text=True, - check=False, - ) - output: Final = f"{completed.stdout}\n{completed.stderr}".strip() - lines: Final = tuple(output.splitlines()) - return completed.returncode == 0, "\n".join(lines[-_FAILURE_OUTPUT_LINES:]) - - -def _installed_bridge_has_trace(repo_root: Path) -> bool: - completed: Final = subprocess.run( - (sys.executable, "-c", _TRACE_CHECK), - cwd=repo_root, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - return completed.returncode == 0 - - -def trace_bridge_error() -> str | None: - bridge: Final = get_native_bridge() - if bridge is None: - return "native Rust bridge is not importable" - if getattr(bridge, "_trace", None) is None: - return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature" - return None - - -def ensure_trace_bridge(repo_root: Path) -> str | None: - native_path: Final = _native_module_path() - native_mtime: Final = native_path.stat().st_mtime if native_path is not None and native_path.exists() else None - rebuild_required: Final = needs_rebuild( - native_mtime, _newest_source_mtime(repo_root) - ) or not _installed_bridge_has_trace(repo_root) - if rebuild_required: - print(f"Rebuilding native Rust bridge ({BRIDGE_FEATURE} feature)...", flush=True) - succeeded: Final - output: Final - succeeded, output = _rebuild(repo_root) - if not succeeded: - return f"native Rust bridge rebuild failed:\n{output}" - _drop_imported_bridge() - return trace_bridge_error() diff --git a/tests/rust-python-harness/shared/reporting/strategy.py b/tests/rust-python-harness/shared/reporting/strategy.py index d8e9d9e5ba9..7e76f035e20 100644 --- a/tests/rust-python-harness/shared/reporting/strategy.py +++ b/tests/rust-python-harness/shared/reporting/strategy.py @@ -67,13 +67,6 @@ class RunnerArgumentDefinition: metavar: str = "ARG" -@dataclass(frozen=True, slots=True) -class RunnerOptionDefinition: - option: str - help: str - choices: tuple[str, ...] - - class StrategyRunner(Protocol): def __call__( self, @@ -97,4 +90,3 @@ class StrategyDefinition: render: StrategyRenderer surfaces: tuple[Surface, ...] = () runner_argument: RunnerArgumentDefinition | None = None - runner_options: tuple[RunnerOptionDefinition, ...] = () diff --git a/tests/rust-python-harness/shared/test_native_build.py b/tests/rust-python-harness/shared/test_native_build.py deleted file mode 100644 index dc08bc1a2b6..00000000000 --- a/tests/rust-python-harness/shared/test_native_build.py +++ /dev/null @@ -1,121 +0,0 @@ -from __future__ import annotations - -import os -from types import SimpleNamespace -from typing import Final - -import pytest - -from . import native_build - - -def test_needs_rebuild_when_bridge_is_missing() -> None: - assert native_build.needs_rebuild(None, 1.0) - - -def test_needs_rebuild_when_sources_are_newer_than_bridge() -> None: - assert native_build.needs_rebuild(1.0, 2.0) - - -def test_fresh_bridge_with_older_sources_needs_no_rebuild() -> None: - assert not native_build.needs_rebuild(2.0, 1.0) - - -def test_bridge_without_rust_sources_needs_no_rebuild() -> None: - assert not native_build.needs_rebuild(2.0, None) - - -def test_newest_source_mtime_tracks_rust_sources_and_skips_target(tmp_path: Final) -> None: - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" - source.mkdir(parents=True) - (source / "lib.rs").write_text("fn main() {}\n") - os.utime(source / "lib.rs", (1_000, 1_000)) - manifest: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "Cargo.toml" - manifest.write_text("[package]\n") - os.utime(manifest, (2_000, 2_000)) - lockfile: Final = tmp_path / "litellm-rust" / "Cargo.lock" - lockfile.write_text("") - os.utime(lockfile, (1_500, 1_500)) - target: Final = tmp_path / "litellm-rust" / "target" / "debug" / "junk.rs" - target.parent.mkdir(parents=True) - target.write_text("fn main() {}\n") - os.utime(target, (9_999, 9_999)) - - assert native_build._newest_source_mtime(tmp_path) == 2_000.0 - - -def test_newest_source_mtime_is_none_without_rust_workspace(tmp_path: Final) -> None: - assert native_build._newest_source_mtime(tmp_path) is None - - -def test_ensure_trace_bridge_rebuilds_when_stale( - tmp_path: Final, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - native: Final = tmp_path / "_native.abi3.so" - native.write_bytes(b"") - os.utime(native, (1_000, 1_000)) - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" / "lib.rs" - source.parent.mkdir(parents=True) - source.write_text("fn main() {}\n") - os.utime(source, (2_000, 2_000)) - state: Final = SimpleNamespace(rebuilt=False) - - def fake_rebuild(repo_root: object) -> tuple[bool, str]: - state.rebuilt = True - return True, "" - - monkeypatch.setattr(native_build, "_native_module_path", lambda: native) - monkeypatch.setattr(native_build, "_rebuild", fake_rebuild) - monkeypatch.setattr(native_build, "_drop_imported_bridge", lambda: None) - monkeypatch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=object())) - - assert native_build.ensure_trace_bridge(tmp_path) is None - assert state.rebuilt is True - assert "Rebuilding native Rust bridge" in capsys.readouterr().out - - -def test_ensure_trace_bridge_reports_failed_rebuild(tmp_path: Final, monkeypatch: pytest.MonkeyPatch) -> None: - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" / "lib.rs" - source.parent.mkdir(parents=True) - source.write_text("fn main() {}\n") - - monkeypatch.setattr(native_build, "_native_module_path", lambda: None) - monkeypatch.setattr(native_build, "_rebuild", lambda repo_root: (False, "boom")) - - message: Final = native_build.ensure_trace_bridge(tmp_path) - - assert message is not None - assert "rebuild failed" in message - assert "boom" in message - - -def test_ensure_trace_bridge_rebuilds_when_trace_feature_is_missing( - tmp_path: Final, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - native: Final = tmp_path / "_native.abi3.so" - native.write_bytes(b"") - os.utime(native, (9_999, 9_999)) - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" / "lib.rs" - source.parent.mkdir(parents=True) - source.write_text("fn main() {}\n") - os.utime(source, (1_000, 1_000)) - state: Final = SimpleNamespace(rebuilt=False) - - def fake_rebuild(repo_root: object) -> tuple[bool, str]: - state.rebuilt = True - return True, "" - - def fake_get_native_bridge() -> SimpleNamespace: - assert state.rebuilt - return SimpleNamespace(_trace=object()) - - monkeypatch.setattr(native_build, "_native_module_path", lambda: native) - monkeypatch.setattr(native_build, "_rebuild", fake_rebuild) - monkeypatch.setattr(native_build, "_installed_bridge_has_trace", lambda repo_root: False) - monkeypatch.setattr(native_build, "get_native_bridge", fake_get_native_bridge) - - message: Final = native_build.ensure_trace_bridge(tmp_path) - - assert message is None - assert state.rebuilt is True - assert "Rebuilding native Rust bridge" in capsys.readouterr().out diff --git a/tests/rust-python-harness/shared/tracing/native.py b/tests/rust-python-harness/shared/tracing/native.py deleted file mode 100644 index 688995cbc4b..00000000000 --- a/tests/rust-python-harness/shared/tracing/native.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from pydantic import BaseModel, ConfigDict - -from .profiler import FunctionTraceEvent - - -class _TraceEventPayload(BaseModel): - model_config = ConfigDict(strict=True, extra="forbid") - id: int - parent_id: int | None - function: str - module_path: str | None = None - file: str | None = None - line: int | None = None - - -class TraceResponsePayload(BaseModel): - model_config = ConfigDict(strict=True, extra="forbid") - response: object = None - error: str | None = None - trace: tuple[_TraceEventPayload, ...] | list[_TraceEventPayload] - - -def native_trace_events(payload: object) -> tuple[FunctionTraceEvent, ...]: - response: Final = TraceResponsePayload.model_validate(payload) - return tuple( - FunctionTraceEvent( - event.id, - event.parent_id, - event.function, - event.module_path, - event.file, - event.line, - ) - for event in response.trace - ) diff --git a/tests/rust-python-harness/shared/tracing/steps.py b/tests/rust-python-harness/shared/tracing/steps.py index 492ffab64e5..415e3f02efc 100644 --- a/tests/rust-python-harness/shared/tracing/steps.py +++ b/tests/rust-python-harness/shared/tracing/steps.py @@ -1,46 +1,10 @@ from __future__ import annotations -import re -from collections import Counter from collections.abc import Sequence from dataclasses import dataclass -from typing import Final, Literal from .profiler import FunctionTraceEvent -Engine = Literal["python", "rust"] - - -@dataclass(frozen=True, slots=True) -class TraceMapping: - span: str - python: re.Pattern[str] | None - rust: str | None - - -def mapping( - *, - python_frame: str | None = None, - rust_span: str | None = None, - span: str | None = None, -) -> TraceMapping: - if rust_span is None: - if python_frame is None: - raise ValueError("mapping needs a python_frame pattern, a rust_span name, or both") - if span is None: - raise ValueError("a python-only mapping needs an explicit span to compare under") - return TraceMapping(span, re.compile(python_frame), None) - if python_frame is None: - return TraceMapping(rust_span, None, rust_span) - if span is not None and span != rust_span: - raise ValueError(f"span {span!r} disagrees with rust_span {rust_span!r}") - return TraceMapping(rust_span, re.compile(python_frame), rust_span) - - -@dataclass(frozen=True, slots=True) -class TraceContract: - unordered_children_of: frozenset[str] = frozenset() - @dataclass(frozen=True, slots=True) class PipelineStep: @@ -50,61 +14,22 @@ class PipelineStep: raw: str -@dataclass(frozen=True, slots=True) -class PipelineProjection: - steps: tuple[PipelineStep, ...] = () - unmatched: int = 0 - - -def _span_for(engine: Engine, function: str, mappings: Sequence[TraceMapping]) -> str | None: - matches: Final = tuple( - item.span - for item in mappings - if ( - engine == "python" - and item.python is not None - and item.python.search(function) - or engine == "rust" - and item.rust == function - ) - ) - if len(matches) > 1: - raise ValueError(f"{engine} event {function!r} matches multiple trace mappings: {matches}") - if matches: - return matches[0] - return function if engine == "rust" else None - - -def pipeline_projection( - engine: Engine, events: Sequence[FunctionTraceEvent], mappings: Sequence[TraceMapping] | None = None -) -> PipelineProjection: +def pipeline_projection(events: Sequence[FunctionTraceEvent]) -> tuple[PipelineStep, ...]: raw_parents: dict[int, int | None] = {} projected_ids: set[int] = set() shown: list[PipelineStep] = [] - unmatched: int = 0 for event in events: if event.id in raw_parents: raise ValueError(f"duplicate trace event id {event.id}") if event.parent_id is not None and event.parent_id not in raw_parents: raise ValueError(f"trace event {event.id} references unknown or later parent {event.parent_id}") raw_parents[event.id] = event.parent_id - span = event.function if mappings is None else _span_for(engine, event.function, mappings) - if span is None: - unmatched += 1 - continue parent_id: int | None = event.parent_id while parent_id is not None and parent_id not in projected_ids: parent_id = raw_parents[parent_id] - shown.append(PipelineStep(event.id, parent_id, span, event.raw)) + shown.append(PipelineStep(event.id, parent_id, event.function, event.raw)) projected_ids.add(event.id) - return PipelineProjection(tuple(shown), unmatched) - - -@dataclass(frozen=True, slots=True) -class TraceNode: - id: int - span: str - children: tuple[TraceNode, ...] + return tuple(shown) def trace_depths(steps: Sequence[PipelineStep]) -> dict[int, int]: @@ -112,153 +37,3 @@ def trace_depths(steps: Sequence[PipelineStep]) -> dict[int, int]: for step in steps: depths[step.id] = 0 if step.parent_id is None else depths[step.parent_id] + 1 return depths - - -def _forest(steps: Sequence[PipelineStep]) -> tuple[TraceNode, ...]: - children: dict[int | None, list[PipelineStep]] = {} - known: set[int] = set() - for step in steps: - if step.id in known: - raise ValueError(f"duplicate projected event id {step.id}") - if step.parent_id is not None and step.parent_id not in known: - raise ValueError(f"projected event {step.id} references unknown or later parent {step.parent_id}") - known.add(step.id) - children.setdefault(step.parent_id, []).append(step) - - def node(step: PipelineStep) -> TraceNode: - return TraceNode(step.id, step.span, tuple(node(child) for child in children.get(step.id, ()))) - - return tuple(node(step) for step in children.get(None, ())) - - -def _exclusive_spans(engine: Engine, mappings: Sequence[TraceMapping]) -> frozenset[str]: - return frozenset( - item.span - for item in mappings - if (engine == "python" and item.rust is None) or (engine == "rust" and item.python is None) - ) - - -def _comparable_steps( - engine: Engine, steps: Sequence[PipelineStep], mappings: Sequence[TraceMapping] -) -> tuple[PipelineStep, ...]: - exclusive: Final = _exclusive_spans(engine, mappings) - raw_parents: Final = {step.id: step.parent_id for step in steps} - included: Final = {step.id for step in steps if step.span not in exclusive} - comparable: list[PipelineStep] = [] - for step in steps: - if step.id not in included: - continue - parent_id: int | None = step.parent_id - while parent_id is not None and parent_id not in included: - parent_id = raw_parents[parent_id] - comparable.append(PipelineStep(step.id, parent_id, step.span, step.raw)) - return tuple(comparable) - - -def _signature(node: TraceNode, contract: TraceContract) -> tuple[object, ...]: - children: tuple[tuple[object, ...], ...] = tuple(_signature(child, contract) for child in node.children) - normalized: Final = tuple(sorted(children, key=repr)) if node.span in contract.unordered_children_of else children - return (node.span, normalized) - - -def trace_signature( - engine: Engine, - steps: Sequence[PipelineStep], - mappings: Sequence[TraceMapping], - contract: TraceContract, -) -> tuple[tuple[object, ...], ...]: - return tuple(_signature(root, contract) for root in _forest(_comparable_steps(engine, steps, mappings))) - - -@dataclass(frozen=True, slots=True) -class TraceDiff: - python_only: tuple[str, ...] - rust_only: tuple[str, ...] - shared_order_matches: bool - missing_mappings: tuple[str, ...] = () - first_difference: str | None = None - - @property - def matches(self) -> bool: - return not self.python_only and not self.rust_only and not self.missing_mappings and self.shared_order_matches - - -def _missing_mappings( - python: Sequence[PipelineStep], rust: Sequence[PipelineStep], mappings: Sequence[TraceMapping] -) -> tuple[str, ...]: - python_seen: Final = frozenset(step.span for step in python) - rust_seen: Final = frozenset(step.span for step in rust) - return tuple( - item.span - for item in mappings - if (item.python is not None and item.span not in python_seen) - or (item.rust is not None and item.span not in rust_seen) - ) - - -def _first_difference( - python: Sequence[PipelineStep], - rust: Sequence[PipelineStep], - mappings: Sequence[TraceMapping], - contract: TraceContract, -) -> str | None: - python_forest: Final = _forest(_comparable_steps("python", python, mappings)) - rust_forest: Final = _forest(_comparable_steps("rust", rust, mappings)) - - def compare_children( - python_nodes: Sequence[TraceNode], rust_nodes: Sequence[TraceNode], path: str, *, unordered: bool - ) -> str | None: - if unordered: - python_signatures: Final = Counter(_signature(node, contract) for node in python_nodes) - rust_signatures: Final = Counter(_signature(node, contract) for node in rust_nodes) - if python_signatures != rust_signatures: - return f"{path}: unordered child subtree multiset differs" - return None - for index in range(max(len(python_nodes), len(rust_nodes))): - child_path = f"{path}/child[{index + 1}]" - if index >= len(python_nodes): - return f"{child_path}: Rust has extra {rust_nodes[index].span!r}" - if index >= len(rust_nodes): - return f"{child_path}: Python has extra {python_nodes[index].span!r}" - python_node = python_nodes[index] - rust_node = rust_nodes[index] - if python_node.span != rust_node.span: - return f"{child_path}: Python={python_node.span!r}, Rust={rust_node.span!r}" - difference = compare_children( - python_node.children, - rust_node.children, - f"{child_path}/{python_node.span}", - unordered=python_node.span in contract.unordered_children_of, - ) - if difference is not None: - return difference - return None - - return compare_children(python_forest, rust_forest, "root", unordered=False) - - -def trace_diff( - python: Sequence[PipelineStep], - rust: Sequence[PipelineStep], - mappings: Sequence[TraceMapping] = (), - contract: TraceContract = TraceContract(), -) -> TraceDiff: - python_comparable: Final = _comparable_steps("python", python, mappings) - rust_comparable: Final = _comparable_steps("rust", rust, mappings) - python_spans: Final = tuple(step.span for step in python_comparable) - rust_spans: Final = tuple(step.span for step in rust_comparable) - python_counts: Final = Counter(python_spans) - rust_counts: Final = Counter(rust_spans) - python_only_counts: Final = python_counts - rust_counts - rust_only_counts: Final = rust_counts - python_counts - python_only: Final = tuple(span for span, count in python_only_counts.items() for _ in range(count)) - rust_only: Final = tuple(span for span, count in rust_only_counts.items() for _ in range(count)) - first_difference: Final = _first_difference(python, rust, mappings, contract) - return TraceDiff( - python_only=python_only, - rust_only=rust_only, - shared_order_matches=bool(python_comparable or rust_comparable) and first_difference is None, - missing_mappings=_missing_mappings(python, rust, mappings), - first_difference=first_difference, - ) diff --git a/tests/rust-python-harness/shared/tracing/test_steps.py b/tests/rust-python-harness/shared/tracing/test_steps.py index ee5e0bafd28..cad9bf1aab5 100644 --- a/tests/rust-python-harness/shared/tracing/test_steps.py +++ b/tests/rust-python-harness/shared/tracing/test_steps.py @@ -5,43 +5,14 @@ from typing import Final import pytest from .profiler import FunctionTraceEvent -from .steps import Engine, TraceContract, mapping, pipeline_projection, trace_depths, trace_diff - -MAPPINGS: Final = ( - mapping(rust_span="route", python_frame=r"entry$"), - mapping(rust_span="provider", python_frame=r"provider$"), - mapping(rust_span="request", python_frame=r"request$"), - mapping(rust_span="http", python_frame=r"post$"), - mapping(rust_span="response", python_frame=r"response$"), -) +from .steps import pipeline_projection, trace_depths def event(event_id: int, function: str, parent_id: int | None = None) -> FunctionTraceEvent: return FunctionTraceEvent(event_id, parent_id, function) -def test_python_projection_collapses_unmapped_parents_and_counts_noise() -> None: - events: Final = ( - event(0, "module.py:1 entry"), - event(1, "noise", 0), - event(2, "module.py:2 provider", 1), - event(3, "module.py:3 request", 0), - event(4, "client.py:4 post", 3), - event(5, "module.py:5 response", 0), - ) - projection: Final = pipeline_projection("python", events, MAPPINGS) - assert projection.unmatched == 1 - assert [(step.id, step.parent_id, step.span, step.raw) for step in projection.steps] == [ - (0, None, "route", "module.py:1 entry"), - (2, 0, "provider", "module.py:2 provider"), - (3, 0, "request", "module.py:3 request"), - (4, 3, "http", "client.py:4 post"), - (5, 0, "response", "module.py:5 response"), - ] - - -@pytest.mark.parametrize("engine", ("python", "rust")) -def test_projection_without_mappings_keeps_every_call_and_parent(engine: Engine) -> None: +def test_projection_keeps_every_call_and_parent() -> None: events: Final = ( event(0, "module.py:1 entry"), event(1, "module.py:2 internal_helper", 0), @@ -49,124 +20,25 @@ def test_projection_without_mappings_keeps_every_call_and_parent(engine: Engine) event(3, "module.py:2 internal_helper", 0), ) - projection: Final = pipeline_projection(engine, events) + steps: Final = pipeline_projection(events) - assert projection.unmatched == 0 - assert tuple((step.id, step.parent_id, step.span, step.raw) for step in projection.steps) == tuple( + assert tuple((step.id, step.parent_id, step.span, step.raw) for step in steps) == tuple( (item.id, item.parent_id, item.function, item.raw) for item in events ) -def test_rust_projection_keeps_unknown_spans() -> None: - projection: Final = pipeline_projection("rust", (event(0, "route"), event(1, "new_span", 0)), MAPPINGS) - assert [(step.span, step.parent_id) for step in projection.steps] == [("route", None), ("new_span", 0)] - - def test_projection_preserves_repeated_occurrences() -> None: - projection: Final = pipeline_projection( - "rust", - (event(0, "route"), event(1, "http", 0), event(2, "http", 0)), - MAPPINGS, - ) - assert [step.span for step in projection.steps] == ["route", "http", "http"] + steps: Final = pipeline_projection((event(0, "route"), event(1, "http", 0), event(2, "http", 0))) + assert [step.span for step in steps] == ["route", "http", "http"] def test_projection_preserves_multiple_roots() -> None: - projection: Final = pipeline_projection("rust", (event(0, "route"), event(1, "request")), MAPPINGS) - assert trace_depths(projection.steps) == {0: 0, 1: 0} + steps: Final = pipeline_projection((event(0, "route"), event(1, "request"))) + assert trace_depths(steps) == {0: 0, 1: 0} def test_projection_rejects_duplicate_and_unknown_parent_ids() -> None: with pytest.raises(ValueError, match="duplicate trace event id"): - pipeline_projection("rust", (event(0, "route"), event(0, "request")), MAPPINGS) + pipeline_projection((event(0, "route"), event(0, "request"))) with pytest.raises(ValueError, match="unknown or later parent"): - pipeline_projection("rust", (event(1, "request", 0),), MAPPINGS) - - -@pytest.mark.parametrize("engine", ("python", "rust")) -def test_rust_only_mappings_do_not_swallow_python_frames(engine: Engine) -> None: - projection: Final = pipeline_projection( - engine, - (event(0, "anything"),), - (mapping(rust_span="rust_only_span"),), - ) - if engine == "python": - assert projection.unmatched == 1 - assert projection.steps == () - else: - assert projection.unmatched == 0 - assert projection.steps[0].span == "anything" - - -def test_mapping_builder_rejects_empty_and_ambiguous_declarations() -> None: - with pytest.raises(ValueError, match="mapping needs"): - mapping() - with pytest.raises(ValueError, match="python-only mapping needs"): - mapping(python_frame=r"frame$") - with pytest.raises(ValueError, match="disagrees with"): - mapping(rust_span="span_a", python_frame=r"frame$", span="span_b") - - -def test_projection_rejects_ambiguous_python_mapping() -> None: - mappings: Final = ( - mapping(rust_span="first", python_frame=r"same$"), - mapping(rust_span="second", python_frame=r"same$"), - ) - with pytest.raises(ValueError, match="multiple trace mappings"): - pipeline_projection("python", (event(0, "module.py:1 same"),), mappings) - - -def test_trace_diff_matches_identical_occurrence_trees() -> None: - mappings: Final = (MAPPINGS[0], MAPPINGS[2]) - steps: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "request", 0)), mappings - ).steps - assert trace_diff(steps, steps, mappings).matches - - -def test_trace_diff_rejects_missing_occurrence_and_parent_drift() -> None: - python: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "request", 0)), MAPPINGS - ).steps - missing: Final = pipeline_projection("rust", (event(0, "route"), event(1, "request", 0)), MAPPINGS).steps - reparented: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "request", 1)), MAPPINGS - ).steps - assert trace_diff(python, missing, MAPPINGS).python_only == ("request",) - assert not trace_diff(python, reparented, MAPPINGS).matches - - -def test_trace_diff_rejects_sequential_reorder() -> None: - first: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "response", 0)), MAPPINGS - ).steps - second: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "response", 0), event(2, "request", 0)), MAPPINGS - ).steps - diff: Final = trace_diff(first, second, MAPPINGS) - assert not diff.matches - assert diff.first_difference == "root/child[1]/route/child[1]: Python='request', Rust='response'" - - -def test_trace_diff_allows_reordered_concurrent_children() -> None: - mappings: Final = (MAPPINGS[0], MAPPINGS[2], MAPPINGS[4]) - first: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "response", 0)), mappings - ).steps - second: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "response", 0), event(2, "request", 0)), mappings - ).steps - contract: Final = TraceContract(frozenset({"route"})) - assert trace_diff(first, second, mappings, contract).matches - - -def test_trace_diff_prunes_declared_engine_only_nodes_but_requires_them() -> None: - mappings: Final = (MAPPINGS[0], mapping(rust_span="rust_prepare")) - python: Final = pipeline_projection("python", (event(0, "module.py:1 entry"),), mappings).steps - rust: Final = pipeline_projection("rust", (event(0, "route"), event(1, "rust_prepare", 0)), mappings).steps - assert trace_diff(python, rust, mappings).matches - assert trace_diff(python, rust[:1], mappings).missing_mappings == ("rust_prepare",) - - -def test_trace_diff_does_not_claim_empty_traces_match() -> None: - assert not trace_diff((), ()).matches + pipeline_projection((event(1, "request", 0),)) diff --git a/tests/rust-python-harness/shared/unit_runners/contracts.py b/tests/rust-python-harness/shared/unit_runners/contracts.py new file mode 100644 index 00000000000..e121ee515e6 --- /dev/null +++ b/tests/rust-python-harness/shared/unit_runners/contracts.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from typing_extensions import Self + +from ..reporting.models import SdkFunction + + +class _ContractModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +def _clean_unique(values: tuple[str, ...], field: str) -> tuple[str, ...]: + cleaned: Final = tuple(value.strip().rstrip("/") for value in values) + if not cleaned or any(not value for value in cleaned): + raise ValueError(f"{field} must contain non-empty paths") + duplicates: Final = tuple(value for value, count in Counter(cleaned).items() if count > 1) + if duplicates: + raise ValueError(f"{field} contains duplicates: {sorted(duplicates)}") + return cleaned + + +class UnitParityExclusionSpec(_ContractModel): + nodeid: str + reason: str + + @field_validator("nodeid", "reason") + @classmethod + def validate_fields(cls, value: str) -> str: + stripped: Final = value.strip() + if not stripped: + raise ValueError("must be a non-empty string") + return stripped + + +class UnitParitySpec(_ContractModel): + python_selectors: tuple[str, ...] + exclusions: tuple[UnitParityExclusionSpec, ...] = () + + @field_validator("python_selectors") + @classmethod + def validate_python_selectors(cls, value: tuple[str, ...]) -> tuple[str, ...]: + return _clean_unique(value, "unit parity python_selectors") + + @model_validator(mode="after") + def validate_exclusions(self) -> Self: + nodeids: Final = tuple(exclusion.nodeid for exclusion in self.exclusions) + duplicates: Final = tuple(nodeid for nodeid, count in Counter(nodeids).items() if count > 1) + if duplicates: + raise ValueError(f"unit parity exclusions contain duplicate nodeids: {sorted(duplicates)}") + return self + + +class RustUnitSpec(_ContractModel): + cargo_manifest: str + cargo_filter: str + cargo_package: str | None = None + + @field_validator("cargo_manifest", "cargo_filter") + @classmethod + def validate_required_fields(cls, value: str) -> str: + stripped: Final = value.strip() + if not stripped: + raise ValueError("must be a non-empty string") + return stripped + + @field_validator("cargo_package") + @classmethod + def validate_package(cls, value: str | None) -> str | None: + if value is None: + return None + stripped: Final = value.strip() + if not stripped: + raise ValueError("must be a non-empty string when provided") + return stripped + + +class UnitTestContract(_ContractModel): + unit_parity: UnitParitySpec + rust: RustUnitSpec + + +OCR_CONTRACT: Final = UnitTestContract( + unit_parity=UnitParitySpec( + python_selectors=( + "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "tests/test_litellm/llms/mistral/ocr", + "tests/test_litellm/llms/ocr", + "tests/test_litellm/ocr", + ), + exclusions=( + UnitParityExclusionSpec( + nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_rust_toggles_flag", + reason="This test asserts the process-level backend flag selected by the parity runner.", + ), + ), + ), + rust=RustUnitSpec( + cargo_manifest="litellm-rust/Cargo.toml", + cargo_filter="ocr", + ), +) + +UNIT_TEST_CONTRACTS: Final[Mapping[SdkFunction, UnitTestContract]] = MappingProxyType({"ocr": OCR_CONTRACT}) diff --git a/tests/rust-python-harness/strategies/trace_parity/AGENTS.md b/tests/rust-python-harness/strategies/trace_parity/AGENTS.md index 19861aea2b4..030caa557c8 100644 --- a/tests/rust-python-harness/strategies/trace_parity/AGENTS.md +++ b/tests/rust-python-harness/strategies/trace_parity/AGENTS.md @@ -1 +1 @@ -Prints every collected Python call under litellm/ and every feature-gated Rust span from live traces against replayed HTTP responses. The two traces are independent and are not compared. API-key and Vertex credentials scenarios exercise separate authentication paths; credentials scenarios replay the token exchange locally. +Prints every collected Python call under litellm/ from live traces against replayed HTTP responses. API-key and Vertex credentials scenarios exercise separate authentication paths; credentials scenarios replay the token exchange locally. diff --git a/tests/rust-python-harness/strategies/trace_parity/__init__.py b/tests/rust-python-harness/strategies/trace_parity/__init__.py index 710bdaa3d39..9aa4f46e4df 100644 --- a/tests/rust-python-harness/strategies/trace_parity/__init__.py +++ b/tests/rust-python-harness/strategies/trace_parity/__init__.py @@ -7,7 +7,6 @@ from ...shared.reporting.strategy import ( ModuleCaseSpec, NotImplementedCaseSpec, RunnerArgumentDefinition, - RunnerOptionDefinition, StrategyDefinition, ) from .reporting import render_trace_results @@ -72,20 +71,12 @@ CASES: Final[tuple[CaseDefinition, ...]] = ( ), CaseDefinition( "messages", - ModuleCaseSpec( - coverage=Coverage.PARTIAL, - module="tests.rust-python-harness.strategies.trace_parity.gateway.messages.case", - note="Anthropic/Azure provider routes plus a fully consumed downstream streaming path.", - ), + NotImplementedCaseSpec(reason="No gateway Messages trace-parity case is registered."), surface="gateway", ), CaseDefinition( "responses", - ModuleCaseSpec( - coverage=Coverage.PARTIAL, - module="tests.rust-python-harness.strategies.trace_parity.gateway.responses.case", - note="Native OpenAI non-streaming and fully consumed downstream streaming paths.", - ), + NotImplementedCaseSpec(reason="No gateway Responses trace-parity case is registered."), surface="gateway", ), CaseDefinition( @@ -95,11 +86,7 @@ CASES: Final[tuple[CaseDefinition, ...]] = ( ), CaseDefinition( "chat_completions", - ModuleCaseSpec( - coverage=Coverage.PARTIAL, - module="tests.rust-python-harness.strategies.trace_parity.gateway.chat_completions.case", - note="Anthropic non-streaming and fully consumed downstream streaming paths.", - ), + NotImplementedCaseSpec(reason="No gateway chat trace-parity case is registered."), surface="gateway", ), CaseDefinition( @@ -113,7 +100,7 @@ STRATEGY: Final = StrategyDefinition( id="trace_parity", order=20, label="Traces", - description="Print Python profiler frames and Rust spans for representative pipeline scenarios.", + description="Print Python profiler frames for representative pipeline scenarios.", directory=Path(__file__).parent, runnable_spec=ModuleCaseSpec, cases=CASES, @@ -125,11 +112,4 @@ STRATEGY: Final = StrategyDefinition( metavar="NAME", help="run only this named trace scenario; repeat to select more than one", ), - runner_options=( - RunnerOptionDefinition( - option="--engine", - choices=("python", "rust"), - help="show only this engine's trace; omit to print both engines", - ), - ), ) diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py b/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py deleted file mode 100644 index f999dfecfc6..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""In-process gateway trace adapters.""" diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py b/tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py deleted file mode 100644 index 3dc6d731b4b..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from .....shared.tracing.steps import Engine, mapping -from ...fixtures import anthropic_response_body, anthropic_stream_events, json_response, sse_response -from ...models import GatewayRouteSpec, RouteFixture, TraceScenario, TraceSuite - -MAPPINGS: Final = ( - mapping(span="python_chat_gateway_route", python_frame=r"proxy_server\.py:\d+ chat_completion$"), - mapping(span="python_gateway_service", python_frame=r"ProxyBaseLLMRequestProcessing\.base_process_llm_request$"), - mapping(span="python_chat_entrypoint", python_frame=r"main\.py:\d+ a?completion$"), - mapping(span="python_provider_config", python_frame=r"ProviderConfigManager\.get_provider_chat_config$"), - mapping(rust_span="validate_environment", python_frame=r"(? RouteFixture: - return RouteFixture( - kwargs={ - "model_alias": "trace-model", - "provider_model": "anthropic/claude-sonnet-5", - "body": { - "model": "trace-model", - "messages": [{"role": "user", "content": "hello"}], - "max_tokens": 16, - }, - }, - provider_responses=(json_response(anthropic_response_body()),), - ) - - -def _stream_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, base_url) - return fixture.with_body(stream=True).derive( - provider_responses=(sse_response(anthropic_stream_events()),), - ) - - -TRACE_SUITE: Final = TraceSuite( - route=GatewayRouteSpec("chat_completions", rust_supported=False), - scenarios=( - TraceScenario(name="async-anthropic", fixture=_fixture, mappings=MAPPINGS, asynchronous=True), - TraceScenario( - name="async-anthropic-downstream-stream", - fixture=_stream_fixture, - mappings=(*MAPPINGS, *STREAM_MAPPINGS), - asynchronous=True, - ), - ), -) diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/execution.py b/tests/rust-python-harness/strategies/trace_parity/gateway/execution.py deleted file mode 100644 index 94d6be7cebf..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/execution.py +++ /dev/null @@ -1,197 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from functools import cache -from pathlib import Path -from typing import Final, Protocol, cast - -import httpx -from pydantic import BaseModel, ConfigDict - -from ....shared.parity.replay import replay_server -from ....shared.tracing.native import TraceResponsePayload, native_trace_events -from ....shared.tracing.profiler import FunctionTraceEvent, profile_python -from ....shared.tracing.steps import Engine, PipelineProjection, pipeline_projection -from ..models import GatewayRouteSpec, RouteFixture, TraceEngine, TraceExecutionFailure, TraceScenario -from ..reporting import TraceArtifact - - -class _GatewayResponsePayload(BaseModel): - model_config = ConfigDict(strict=True, extra="forbid") - - status: int - body: object - - -class _GatewayClient(Protocol): - def post(self, url: str, *, json: object, headers: dict[str, str]) -> httpx.Response: ... - - -_ROUTE_PATHS: Final = { - "messages": "/v1/messages", - "chat_completions": "/v1/chat/completions", - "responses": "/v1/responses", -} - - -def _collect_python(fixture: RouteFixture, route: GatewayRouteSpec) -> tuple[FunctionTraceEvent, ...]: - from fastapi.testclient import TestClient - - import litellm - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.anthropic_endpoints.endpoints import user_api_key_auth - - provider_model: Final = cast(str, fixture.kwargs["provider_model"]) - model_alias: Final = cast(str, fixture.kwargs["model_alias"]) - old_router: Final = proxy_server.llm_router - old_override: Final = proxy_server.app.dependency_overrides.get(user_api_key_auth) - - async def authorize() -> UserAPIKeyAuth: - return UserAPIKeyAuth(api_key="trace-key") - - proxy_server.llm_router = litellm.Router( - model_list=[ - { - "model_name": model_alias, - "litellm_params": { - "model": provider_model, - "api_key": "trace-provider-key", - "api_base": fixture.kwargs["api_base"], - }, - } - ] - ) - proxy_server.app.dependency_overrides[user_api_key_auth] = authorize - try: - with profile_python(Path(litellm.__file__).parent, threads=True) as profiler: - client: Final = cast(_GatewayClient, TestClient(proxy_server.app)) - response: Final = client.post( - _ROUTE_PATHS[route.route], - json=fixture.kwargs["body"], - headers={"authorization": "Bearer trace-key"}, - ) - if response.status_code != 200: - raise RuntimeError(f"Python gateway returned {response.status_code}: {response.text}") - return tuple(profiler.events) - finally: - proxy_server.llm_router = old_router - if old_override is None: - proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) - else: - proxy_server.app.dependency_overrides[user_api_key_auth] = old_override - - -def _collect_rust(fixture: RouteFixture, route: GatewayRouteSpec) -> tuple[FunctionTraceEvent, ...]: - payload: Final = json.dumps( - { - "path": _ROUTE_PATHS[route.route], - "model_alias": fixture.kwargs["model_alias"], - "provider_model": fixture.kwargs["provider_model"], - "api_base": fixture.kwargs["api_base"], - "body": fixture.kwargs["body"], - } - ) - completed: Final = subprocess.run( - (_gateway_trace_binary(),), - input=payload, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError(f"Rust gateway trace failed: {completed.stderr.strip()}") - result: Final = json.loads(completed.stdout) - payload: Final = TraceResponsePayload.model_validate(result) - response: Final = _GatewayResponsePayload.model_validate(payload.response) - if response.status != 200: - raise RuntimeError(f"Rust gateway returned {response.status}: {response.body}") - return native_trace_events(payload) - - -@cache -def _gateway_trace_binary() -> Path: - repo_root: Final = next(parent for parent in Path(__file__).resolve().parents if (parent / "litellm-rust").is_dir()) - rust_root: Final = repo_root / "litellm-rust" - completed: Final = subprocess.run( - ( - "cargo", - "build", - "--quiet", - "--package", - "litellm-ai-gateway", - "--features", - "trace-parity", - "--bin", - "trace-parity-gateway", - "--target-dir", - rust_root / "target", - ), - cwd=rust_root, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError(f"Rust gateway trace build failed: {completed.stderr.strip()}") - return rust_root / "target" / "debug" / "trace-parity-gateway" - - -def _collect( - route: GatewayRouteSpec, scenario: TraceScenario, engine: Engine -) -> tuple[FunctionTraceEvent, ...] | TraceExecutionFailure: - try: - with replay_server() as provider: - base_fixture: Final = scenario.fixture(engine, provider.url) - fixture: Final = RouteFixture( - kwargs={**base_fixture.kwargs, "api_base": provider.url}, - provider_responses=base_fixture.provider_responses, - ) - for response in fixture.provider_responses: - provider.enqueue_response(response) - events: Final = _collect_python(fixture, route) if engine == "python" else _collect_rust(fixture, route) - provider.take_requests(len(fixture.provider_responses)) - return events - except Exception as error: - return TraceExecutionFailure(engine, f"{type(error).__name__}: {error}") - - -def _projections( - python_events: tuple[FunctionTraceEvent, ...], - rust_events: tuple[FunctionTraceEvent, ...], -) -> tuple[PipelineProjection, PipelineProjection, str | None]: - try: - return ( - pipeline_projection("python", python_events), - pipeline_projection("rust", rust_events), - None, - ) - except ValueError as error: - return PipelineProjection(), PipelineProjection(), f"harness: {error}" - - -def execute_gateway_trace( - route: GatewayRouteSpec, - scenario: TraceScenario, - engine: TraceEngine = "both", -) -> TraceArtifact: - effective_engine: Final[TraceEngine] = "python" if engine == "both" and not route.rust_supported else engine - python_trace: Final = _collect(route, scenario, "python") if effective_engine != "rust" else () - rust_trace: Final = _collect(route, scenario, "rust") if effective_engine != "python" else () - collection_python_error: Final = None if isinstance(python_trace, tuple) else f"python: {python_trace.message}" - rust_error: Final = None if isinstance(rust_trace, tuple) else f"rust: {rust_trace.message}" - python_events: Final = python_trace if isinstance(python_trace, tuple) else () - rust_events: Final = rust_trace if isinstance(rust_trace, tuple) else () - python, rust, projection_error = _projections(python_events, rust_events) - python_error: Final = projection_error or collection_python_error - return TraceArtifact.from_traces( - engine=effective_engine, - surface="gateway", - sdk_function=route.route, - scenario=scenario.name, - python=python.steps, - rust=rust.steps, - python_error=python_error, - rust_error=rust_error, - ) diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py b/tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py deleted file mode 100644 index bd9195b7c22..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Messages gateway trace cases.""" diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py b/tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py deleted file mode 100644 index ca9c858f6b7..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from .....shared.tracing.steps import Engine, mapping -from ...fixtures import anthropic_response_body, anthropic_stream_events, json_response, sse_response -from ...models import GatewayRouteSpec, RouteFixture, TraceScenario, TraceSuite - - -GATEWAY_MAPPINGS: Final = ( - mapping( - span="python_messages_gateway_route", - python_frame=r"anthropic_endpoints/endpoints\.py:\d+ anthropic_response$", - ), - mapping(rust_span="messages_gateway_route"), - mapping( - span="python_messages_gateway_service", - python_frame=r"ProxyBaseLLMRequestProcessing\.base_process_llm_request$", - ), - mapping(rust_span="messages_gateway_service"), - mapping(rust_span="messages"), - mapping( - span="python_messages_provider_config", - python_frame=r"ProviderConfigManager\.get_provider_anthropic_messages_config$", - ), - mapping(rust_span="messages_provider_config"), - mapping(rust_span="validate_environment", python_frame=r"validate_anthropic_messages_environment$"), - mapping(rust_span="complete_url", python_frame=r"get_complete_url$"), - mapping(span="python_messages_entry_handler", python_frame=r"messages/handler\.py:\d+ anthropic_messages_handler$"), - mapping(span="python_messages_handler_wrapper", python_frame=r"BaseLLMHTTPHandler\.anthropic_messages_handler$"), - mapping( - rust_span="execute_messages_provider_call", - python_frame=r"BaseLLMHTTPHandler\.async_anthropic_messages_handler$", - ), - mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"), - mapping(rust_span="transform_response", python_frame=r"(? RouteFixture: - return RouteFixture( - kwargs={ - "model_alias": "trace-model", - "provider_model": f"{provider}/claude-sonnet-5", - "body": { - "model": "trace-model", - "messages": [{"role": "user", "content": "hello"}], - "max_tokens": 16, - }, - }, - provider_responses=(json_response(anthropic_response_body()),), - ) - - -def _anthropic_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "anthropic") - - -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "azure_ai") - - -def _stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, _base_url) - return fixture.with_body(stream=True).derive( - provider_responses=(sse_response(anthropic_stream_events()),), - ) - - -ANTHROPIC_MAPPINGS: Final = ( - *GATEWAY_MAPPINGS, - mapping( - rust_span="transform_request", - python_frame=r"(? RouteFixture: - return RouteFixture( - kwargs={ - "model_alias": "trace-model", - "provider_model": "openai/gpt-5", - "body": {"model": "trace-model", "input": "hello"}, - }, - provider_responses=(json_response(responses_body()),), - ) - - -def _stream_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, base_url) - return fixture.with_body(stream=True).derive( - provider_responses=(sse_response(responses_stream_events()),), - ) - - -TRACE_SUITE: Final = TraceSuite( - route=GatewayRouteSpec("responses", rust_supported=False), - scenarios=( - TraceScenario(name="async-openai", fixture=_fixture, mappings=MAPPINGS, asynchronous=True), - TraceScenario( - name="async-openai-downstream-stream", - fixture=_stream_fixture, - mappings=(*MAPPINGS, *STREAM_MAPPINGS), - asynchronous=True, - ), - ), -) diff --git a/tests/rust-python-harness/strategies/trace_parity/models.py b/tests/rust-python-harness/strategies/trace_parity/models.py index d6ed42250c4..7e6fc321d93 100644 --- a/tests/rust-python-harness/strategies/trace_parity/models.py +++ b/tests/rust-python-harness/strategies/trace_parity/models.py @@ -2,14 +2,12 @@ from __future__ import annotations from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import Final, Literal, TypeAlias, cast +from typing import Final, Literal, cast from ...shared.parity.recorded_http import RecordedResponse from ...shared.reporting.models import SdkFunction -from ...shared.tracing.steps import Engine, TraceMapping -TraceEngine = Literal["python", "rust", "both"] -TraceFailureSource = Literal["python", "rust", "harness"] +TraceFailureSource = Literal["python", "harness"] @dataclass(frozen=True, slots=True) @@ -48,30 +46,19 @@ class RouteFixture: class RouteSpec: route: SdkFunction python_entrypoints: tuple[str, str] - rust_entrypoints: tuple[str, str] | None - fixture: Callable[[Engine, str], RouteFixture] - - -@dataclass(frozen=True, slots=True) -class GatewayRouteSpec: - route: SdkFunction - rust_supported: bool = True - - -TraceRouteSpec: TypeAlias = RouteSpec | GatewayRouteSpec + fixture: Callable[[str], RouteFixture] @dataclass(frozen=True, slots=True) class TraceScenario: name: str - fixture: Callable[[Engine, str], RouteFixture] - mappings: tuple[TraceMapping, ...] + fixture: Callable[[str], RouteFixture] asynchronous: bool @dataclass(frozen=True, slots=True) class TraceSuite: - route: TraceRouteSpec + route: RouteSpec scenarios: tuple[TraceScenario, ...] diff --git a/tests/rust-python-harness/strategies/trace_parity/reporting.py b/tests/rust-python-harness/strategies/trace_parity/reporting.py index e7c07ef9c0f..086cf2d03c7 100644 --- a/tests/rust-python-harness/strategies/trace_parity/reporting.py +++ b/tests/rust-python-harness/strategies/trace_parity/reporting.py @@ -11,14 +11,10 @@ from ...shared.reporting.models import SURFACES, CaseResult, RunStatus, SdkFunct from ...shared.reporting.rendering import ReportSection from ...shared.reporting.strategy import NotImplementedCaseSpec, SkippedCaseSpec from ...shared.tracing.steps import PipelineStep, trace_depths -from .models import TraceEngine TRACE_ARTIFACT: Final = "trace" -TRACE_PARITY_HINT: Final = ( - "rebuild the native bridge with the trace-parity feature, e.g. `uvx maturin develop --features trace-parity`" -) -_COLORS: Final[dict[str, str]] = {"yellow": "33", "red": "31", "cyan": "36"} +_COLORS: Final[dict[str, str]] = {"red": "31", "cyan": "36"} _RESET: Final = "\033[0m" @@ -43,30 +39,23 @@ class TraceEventArtifact(BaseModel): class TraceArtifact(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - engine: TraceEngine = "both" surface: Surface sdk_function: SdkFunction scenario: str python: tuple[TraceEventArtifact, ...] - rust: tuple[TraceEventArtifact, ...] python_error: str | None = None - rust_error: str | None = None @classmethod def from_traces( cls, *, - engine: TraceEngine = "both", surface: Surface, sdk_function: SdkFunction, scenario: str, python: Sequence[PipelineStep], - rust: Sequence[PipelineStep], python_error: str | None = None, - rust_error: str | None = None, ) -> TraceArtifact: return cls( - engine=engine, surface=surface, sdk_function=sdk_function, scenario=scenario, @@ -74,21 +63,14 @@ class TraceArtifact(BaseModel): TraceEventArtifact(id=step.id, parent_id=step.parent_id, span=step.span, raw=step.raw) for step in python ), - rust=tuple( - TraceEventArtifact(id=step.id, parent_id=step.parent_id, span=step.span, raw=step.raw) for step in rust - ), python_error=python_error, - rust_error=rust_error, ) def python_steps(self) -> tuple[PipelineStep, ...]: return tuple(event.step() for event in self.python) - def rust_steps(self) -> tuple[PipelineStep, ...]: - return tuple(event.step() for event in self.rust) - def has_errors(self) -> bool: - return self.python_error is not None or self.rust_error is not None + return self.python_error is not None def _split_raw(raw: str) -> tuple[str, str]: @@ -111,34 +93,14 @@ def _python_lines(steps: tuple[PipelineStep, ...]) -> str: return f"{_paint('PYTHON', 'cyan')} ({len(steps)} steps)\n" + ("\n".join(lines) if lines else "(empty)") -def _rust_lines(steps: tuple[PipelineStep, ...]) -> str: - depths: Final = trace_depths(steps) - lines: Final = tuple( - _paint(f"{index} {' ' * depths[step.id]}{step.span}", "yellow") for index, step in enumerate(steps, 1) - ) - return f"{_paint('RUST', 'yellow')} ({len(steps)} steps)\n" + ("\n".join(lines) if lines else "(empty)") - - def _error_lines(artifact: TraceArtifact) -> tuple[str, ...]: - lines: list[str] = [] - for engine, error in (("Python", artifact.python_error), ("Rust", artifact.rust_error)): - if error is None: - continue - lines.append(_paint(f"{engine} error: {error}", "red")) - if "trace-parity feature" in error: - lines.append(f"hint: {TRACE_PARITY_HINT}") - return tuple(lines) + if artifact.python_error is None: + return () + return (_paint(f"Python error: {artifact.python_error}", "red"),) def _render_trace(artifact: TraceArtifact) -> str: - traces: tuple[str, ...] - if artifact.engine == "python": - traces = (_python_lines(artifact.python_steps()),) - elif artifact.engine == "rust": - traces = (_rust_lines(artifact.rust_steps()),) - else: - traces = (_python_lines(artifact.python_steps()), _rust_lines(artifact.rust_steps())) - return "\n\n".join((*traces, *_error_lines(artifact))) + return "\n\n".join((_python_lines(artifact.python_steps()), *_error_lines(artifact))) def _scenario(nodeid: str) -> str: diff --git a/tests/rust-python-harness/strategies/trace_parity/runner.py b/tests/rust-python-harness/strategies/trace_parity/runner.py index 706e054bf52..9147bcfe8b3 100644 --- a/tests/rust-python-harness/strategies/trace_parity/runner.py +++ b/tests/rust-python-harness/strategies/trace_parity/runner.py @@ -4,15 +4,11 @@ import importlib from collections.abc import Sequence from pathlib import Path from time import monotonic -from typing import Final, cast +from typing import Final -from ...shared.native_build import ensure_trace_bridge from ...shared.reporting.models import CaseResult, HarnessCase, HarnessRun, ResultArtifact, RunStatus, Surface from ...shared.reporting.strategy import ModuleCaseSpec, UpdateCallback from .models import ( - GatewayRouteSpec, - RouteSpec, - TraceEngine, TraceExecutionFailure, TraceScenario, TraceSuite, @@ -47,12 +43,8 @@ def validate_trace_suite(suite: TraceSuite, harness_case: HarnessCase) -> str | if invalid_names: return f"scenario names must start with sync- or async-: {', '.join(invalid_names)}" surface: Final = harness_case.surface - if surface == "sdk" and not isinstance(suite.route, RouteSpec): - return "must use RouteSpec for the sdk surface" - if surface == "gateway" and not isinstance(suite.route, GatewayRouteSpec): - return "must use GatewayRouteSpec for the gateway surface" - if surface is None: - return "requires an sdk or gateway surface" + if surface != "sdk": + return "requires the sdk surface" if suite.route.route != harness_case.sdk_function: return f"route {suite.route.route} does not match case function {harness_case.sdk_function}" return None @@ -89,10 +81,9 @@ def run_trace_scenario( surface: Surface, nodeid: str, on_update: UpdateCallback, - engine: TraceEngine = "both", ) -> None: started_at: Final = monotonic() - trace: Final = _execute_scenario(trace_suite, scenario, surface, engine) + trace: Final = _execute_scenario(trace_suite, scenario, surface) duration: Final = monotonic() - started_at if isinstance(trace, TraceExecutionFailure): result.record(nodeid, RunStatus.ERROR, duration) @@ -102,7 +93,7 @@ def run_trace_scenario( artifact: Final = ResultArtifact(TRACE_ARTIFACT, trace.model_dump_json()) if trace.has_errors(): result.record(nodeid, RunStatus.ERROR, duration, (artifact,)) - run.failures.append((nodeid, "\n".join(error for error in (trace.python_error, trace.rust_error) if error))) + run.failures.append((nodeid, trace.python_error or "")) else: result.record(nodeid, RunStatus.PASSED, duration, (artifact,)) on_update(run) @@ -112,18 +103,10 @@ def _execute_scenario( trace_suite: TraceSuite, scenario: TraceScenario, surface: Surface, - engine: TraceEngine, ) -> TraceArtifact | TraceExecutionFailure: - route: Final = trace_suite.route - if isinstance(route, GatewayRouteSpec): - if surface != "gateway": - return TraceExecutionFailure("harness", "gateway route cannot run on the sdk surface") - from .gateway.execution import execute_gateway_trace - - return execute_gateway_trace(route, scenario, engine) if surface != "sdk": - return TraceExecutionFailure("harness", "sdk route cannot run on the gateway surface") - return execute_trace(route, scenario, surface, engine) + return TraceExecutionFailure("harness", "trace scenarios only run on the sdk surface") + return execute_trace(trace_suite.route, scenario, surface) def _run_case( @@ -131,7 +114,6 @@ def _run_case( harness_case: HarnessCase, selected_scenarios: frozenset[str], on_update: UpdateCallback, - engine: TraceEngine, ) -> None: result: Final = run.results[harness_case.key] spec: Final = harness_case.spec @@ -154,21 +136,7 @@ def _run_case( result.status = RunStatus.RUNNING on_update(run) for scenario, nodeid in nodeids: - run_trace_scenario(run, result, trace_suite, scenario, surface, nodeid, on_update, engine) - - -def runner_selection(runner_args: Sequence[str]) -> tuple[frozenset[str], TraceEngine]: - engine: TraceEngine = "both" - scenarios: list[str] = [] - for argument in runner_args: - if argument.startswith("--engine="): - value = argument.removeprefix("--engine=") - if value not in {"python", "rust"}: - raise ValueError(f"invalid trace engine: {value}") - engine = cast(TraceEngine, value) - else: - scenarios.append(argument) - return frozenset(scenarios), engine + run_trace_scenario(run, result, trace_suite, scenario, surface, nodeid, on_update) def run_trace_cases( @@ -177,18 +145,11 @@ def run_trace_cases( on_update: UpdateCallback, runner_args: Sequence[str] = (), ) -> tuple[int, HarnessRun]: - selected_scenarios, engine = runner_selection(runner_args) + del repo_root + selected_scenarios: Final = frozenset(runner_args) run: Final = HarnessRun.from_cases(cases) - runnable_cases: Final = tuple(case for case in cases if isinstance(case.spec, ModuleCaseSpec)) - bridge_error: Final = ensure_trace_bridge(repo_root) if runnable_cases and engine != "python" else None - if bridge_error is not None: - for harness_case in runnable_cases: - _record_setup_failure(run, harness_case, bridge_error, "bridge") - run.finished_at = monotonic() - on_update(run) - return 1, run for harness_case in cases: - _run_case(run, harness_case, selected_scenarios, on_update, engine) + _run_case(run, harness_case, selected_scenarios, on_update) run.finished_at = monotonic() on_update(run) failed: Final = any( diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py index 1221f237570..016a3683079 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py @@ -2,7 +2,6 @@ from __future__ import annotations from typing import Final -from .....shared.tracing.steps import Engine, mapping from ...fixtures import ( anthropic_response_body, anthropic_stream_events, @@ -12,70 +11,19 @@ from ...fixtures import ( ) from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -COMMON_MAPPINGS: Final = ( - mapping(span="python_provider_config", python_frame=r"ProviderConfigManager\.get_provider_chat_config$"), - mapping(rust_span="chat_completions_provider_config"), - mapping( - span="python_supported_openai_params", - python_frame=r"litellm_core_utils/get_supported_openai_params\.py:\d+ get_supported_openai_params$", - ), - mapping( - span="python_provider_supported_openai_params", - python_frame=r"AnthropicConfig\.get_supported_openai_params$", - ), - mapping(rust_span="supported_openai_params"), - mapping(rust_span="validate_environment", python_frame=r"(? RouteFixture: +def _anthropic_fixture(_base_url: str) -> RouteFixture: return RouteFixture( kwargs={ "model": "anthropic/claude-sonnet-5", "messages": [{"role": "user", "content": "hello"}], - **({"optional_params": {"max_tokens": 16}} if engine == "rust" else {"max_tokens": 16}), + "max_tokens": 16, }, provider_responses=(json_response(anthropic_response_body()),), ) -def _bedrock_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _bedrock_fixture(_base_url: str) -> RouteFixture: response: Final[dict[str, object]] = { "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}}, "stopReason": "end_turn", @@ -91,18 +39,15 @@ def _bedrock_fixture(engine: Engine, _base_url: str) -> RouteFixture: kwargs={ "model": "bedrock/us-east-1/anthropic.claude-v2", "messages": [{"role": "user", "content": "hello"}], - **( - {"optional_params": {**credentials, "maxTokens": 16}} - if engine == "rust" - else {**credentials, "max_tokens": 16} - ), + **credentials, + "max_tokens": 16, }, provider_responses=(json_response(response),), ) -def _anthropic_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, _base_url) +def _anthropic_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _anthropic_fixture(_base_url) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(anthropic_stream_events()),), @@ -110,8 +55,8 @@ def _anthropic_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _bedrock_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _bedrock_fixture(engine, _base_url) +def _bedrock_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _bedrock_fixture(_base_url) events: Final[tuple[dict[str, object], ...]] = ( {"messageStart": {"role": "assistant"}}, {"contentBlockStart": {"contentBlockIndex": 0, "start": {}}}, @@ -127,8 +72,8 @@ def _bedrock_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, _base_url) +def _provider_error_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _anthropic_fixture(_base_url) return fixture.derive( provider_responses=( json_response( @@ -140,8 +85,8 @@ def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, base_url) +def _stream_error_fixture(base_url: str) -> RouteFixture: + fixture: Final = _anthropic_fixture(base_url) events: Final = ( anthropic_stream_events()[0], ("error", {"type": "error", "error": {"type": "overloaded_error", "message": "overloaded"}}), @@ -157,90 +102,59 @@ def _stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture: SPEC: Final = RouteSpec( "chat_completions", ("completion", "acompletion"), - ("chat_completions", "achat_completions"), _anthropic_fixture, ) -BEDROCK_COMMON_MAPPINGS: Final = ( - mapping(rust_span="chat_completions_provider_config"), - mapping(rust_span="supported_openai_params"), - mapping(rust_span="execute_chat_completions_provider_call"), - mapping(rust_span="validate_environment"), - mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"), - mapping(span="python_transform_response", python_frame=r"AmazonConverseConfig\._transform_response$"), -) -BEDROCK_SYNC_MAPPINGS: Final = ( - mapping(span="python_chat_completions", python_frame=r"main\.py:\d+ completion$"), - mapping(rust_span="chat_completions"), - mapping(span="python_transform_request", python_frame=r"AmazonConverseConfig\._transform_request$"), - *BEDROCK_COMMON_MAPPINGS, -) -BEDROCK_ASYNC_MAPPINGS: Final = ( - mapping(span="python_chat_completions", python_frame=r"main\.py:\d+ acompletion$"), - mapping(span="python_completion_wrapper", python_frame=r"main\.py:\d+ completion$"), - mapping(rust_span="chat_completions"), - *BEDROCK_COMMON_MAPPINGS, -) TRACE_SUITE: Final = TraceSuite( route=SPEC, scenarios=( TraceScenario( name="sync-anthropic", fixture=_anthropic_fixture, - mappings=SYNC_MAPPINGS, asynchronous=False, ), TraceScenario( name="async-anthropic", fixture=_anthropic_fixture, - mappings=ASYNC_MAPPINGS, asynchronous=True, ), TraceScenario( name="sync-anthropic-stream", fixture=_anthropic_stream_fixture, - mappings=(*SYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=False, ), TraceScenario( name="async-anthropic-stream", fixture=_anthropic_stream_fixture, - mappings=(*ASYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-anthropic-provider-error", fixture=_provider_error_fixture, - mappings=(*ASYNC_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-anthropic-stream-error", fixture=_stream_error_fixture, - mappings=(*ASYNC_MAPPINGS, *STREAM_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="sync-bedrock", fixture=_bedrock_fixture, - mappings=BEDROCK_SYNC_MAPPINGS, asynchronous=False, ), TraceScenario( name="async-bedrock", fixture=_bedrock_fixture, - mappings=BEDROCK_ASYNC_MAPPINGS, asynchronous=True, ), TraceScenario( name="sync-bedrock-event-stream", fixture=_bedrock_stream_fixture, - mappings=(*BEDROCK_SYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=False, ), TraceScenario( name="async-bedrock-event-stream", fixture=_bedrock_stream_fixture, - mappings=(*BEDROCK_ASYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py b/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py index 783c22a0dc0..ed98e550f4e 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py @@ -10,10 +10,9 @@ from unittest.mock import patch from ....shared.parity.replay import replay_server from ....shared.reporting.models import Surface -from ....shared.tracing.native import TraceResponsePayload, native_trace_events from ....shared.tracing.profiler import FunctionTraceEvent, profile_python -from ....shared.tracing.steps import Engine, pipeline_projection -from ..models import RouteFixture, RouteSpec, TraceEngine, TraceExecutionFailure, TraceScenario +from ....shared.tracing.steps import pipeline_projection +from ..models import RouteFixture, RouteSpec, TraceExecutionFailure, TraceScenario from ..reporting import TraceArtifact @@ -56,25 +55,10 @@ def _invoke( return response -def _entrypoint(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> SdkCall | TraceExecutionFailure: +def _entrypoint(spec: RouteSpec, *, asynchronous: bool) -> SdkCall: import litellm from litellm.anthropic_interface import messages as sdk_messages - from litellm.rust_bridge import get_native_bridge - if engine == "rust": - if spec.rust_entrypoints is None: - return TraceExecutionFailure("rust", f"{spec.route} has no native Rust trace entrypoint") - bridge: Final = cast(object | None, get_native_bridge()) - if bridge is None: - return TraceExecutionFailure("rust", "native Rust bridge is required for trace parity") - trace_bridge: Final[object | None] = getattr(bridge, "_trace", None) - if trace_bridge is None: - return TraceExecutionFailure("rust", "native Rust bridge must include the trace-parity feature") - entrypoint: Final = spec.rust_entrypoints[int(asynchronous)] - function: Final[object | None] = getattr(trace_bridge, entrypoint, None) - if function is None: - return TraceExecutionFailure("rust", f"native Rust trace bridge does not expose {entrypoint}") - return cast(SdkCall, function) owner: Final = sdk_messages if spec.route == "messages" else litellm return cast(SdkCall, getattr(owner, spec.python_entrypoints[int(asynchronous)])) @@ -82,14 +66,9 @@ def _entrypoint(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> SdkCa def _collect( function: SdkCall, fixture: RouteFixture, - engine: Engine, *, asynchronous: bool, ) -> _CollectedTrace: - kwargs: Final = fixture.kwargs - if engine == "rust": - payload: Final = TraceResponsePayload.model_validate(_invoke(function, kwargs, asynchronous=asynchronous)) - return _CollectedTrace(native_trace_events(payload), payload.error) import litellm previous_suppress_debug_info: Final = litellm.suppress_debug_info @@ -99,7 +78,7 @@ def _collect( with profile_python(Path(litellm.__file__).parent, threads=True) as profiler: error: str | None try: - _invoke(function, kwargs, asynchronous=asynchronous, consume_stream=fixture.consume_stream) + _invoke(function, fixture.kwargs, asynchronous=asynchronous, consume_stream=fixture.consume_stream) error = None except Exception as caught: error = f"{type(caught).__name__}: {caught}" @@ -108,13 +87,11 @@ def _collect( return _CollectedTrace(tuple(profiler.events), error) -def collect_trace(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> tuple[FunctionTraceEvent, ...] | TraceExecutionFailure: - function: Final = _entrypoint(spec, engine, asynchronous=asynchronous) - if isinstance(function, TraceExecutionFailure): - return function +def collect_trace(spec: RouteSpec, *, asynchronous: bool) -> tuple[FunctionTraceEvent, ...] | TraceExecutionFailure: + function: Final = _entrypoint(spec, asynchronous=asynchronous) try: with replay_server() as provider: - base_fixture: Final = spec.fixture(engine, provider.url) + base_fixture: Final = spec.fixture(provider.url) for response in base_fixture.provider_responses: provider.enqueue_response(response) fixture: Final = RouteFixture( @@ -122,7 +99,7 @@ def collect_trace(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> tup "api_key": "test-key", **base_fixture.kwargs, "api_base": provider.url, - **({"timeout_seconds": 5} if engine == "rust" else {"timeout": 5}), + "timeout": 5, }, provider_responses=base_fixture.provider_responses, expected_failure=base_fixture.expected_failure, @@ -130,76 +107,42 @@ def collect_trace(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> tup environment=base_fixture.environment, ) with patch.dict(os.environ, fixture.environment): - collected: Final = _collect(function, fixture, engine, asynchronous=asynchronous) + collected: Final = _collect(function, fixture, asynchronous=asynchronous) provider.take_requests(len(fixture.provider_responses)) except Exception as error: - return TraceExecutionFailure(engine, f"{type(error).__name__}: {error}") + return TraceExecutionFailure("python", f"{type(error).__name__}: {error}") if fixture.expected_failure and collected.error is None: - return TraceExecutionFailure(engine, "call succeeded but the scenario expects failure") + return TraceExecutionFailure("python", "call succeeded but the scenario expects failure") if not fixture.expected_failure and collected.error is not None: - return TraceExecutionFailure(engine, collected.error) + return TraceExecutionFailure("python", collected.error) if not collected.events: - return TraceExecutionFailure(engine, "trace is empty") + return TraceExecutionFailure("python", "trace is empty") return collected.events -def _failure_message(result: tuple[FunctionTraceEvent, ...] | TraceExecutionFailure) -> str | None: - if isinstance(result, tuple): - return None - return f"{result.engine}: {result.message}" - - -def execute_trace( - route: RouteSpec, - scenario: TraceScenario, - surface: Surface, - engine: TraceEngine = "both", -) -> TraceArtifact: - effective_engine: Final[TraceEngine] = "python" if engine == "both" and route.rust_entrypoints is None else engine +def execute_trace(route: RouteSpec, scenario: TraceScenario, surface: Surface) -> TraceArtifact: scenario_route: Final = RouteSpec( route=route.route, python_entrypoints=route.python_entrypoints, - rust_entrypoints=route.rust_entrypoints, fixture=scenario.fixture, ) - python_trace: Final = ( - collect_trace( - scenario_route, - "python", - asynchronous=scenario.asynchronous, - ) - if effective_engine != "rust" - else () - ) - rust_trace: Final = ( - collect_trace(scenario_route, "rust", asynchronous=scenario.asynchronous) - if effective_engine != "python" - else () - ) - python_error: Final = _failure_message(python_trace) - rust_error: Final = _failure_message(rust_trace) + python_trace: Final = collect_trace(scenario_route, asynchronous=scenario.asynchronous) + python_error: Final = None if isinstance(python_trace, tuple) else f"{python_trace.engine}: {python_trace.message}" python_events: Final = python_trace if isinstance(python_trace, tuple) else () - rust_events: Final = rust_trace if isinstance(rust_trace, tuple) else () try: - python: Final = pipeline_projection("python", python_events) - rust: Final = pipeline_projection("rust", rust_events) + python: Final = pipeline_projection(python_events) except ValueError as error: return TraceArtifact.from_traces( - engine=effective_engine, surface=surface, sdk_function=route.route, scenario=scenario.name, python=(), - rust=(), python_error=f"harness: {error}", ) return TraceArtifact.from_traces( - engine=effective_engine, surface=surface, sdk_function=route.route, scenario=scenario.name, - python=python.steps, - rust=rust.steps, + python=python, python_error=python_error, - rust_error=rust_error, ) diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py index 211c454eadf..4e6e50c7e37 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py @@ -2,7 +2,6 @@ from __future__ import annotations from typing import Final -from .....shared.tracing.steps import Engine, mapping from ...fixtures import ( anthropic_response_body, anthropic_stream_events, @@ -12,172 +11,44 @@ from ...fixtures import ( ) from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -COMMON_MAPPINGS: Final = ( - mapping(rust_span="messages", python_frame=r"anthropic_interface/messages/__init__\.py:\d+ a?create$"), - mapping(span="python_sanitize_empty_content", python_frame=r"strip_empty_content_blocks_from_anthropic_messages$"), - mapping(span="python_sanitize_tool_ids", python_frame=r"sanitize_tool_use_ids_in_anthropic_messages$"), - mapping( - span="python_flatten_web_search", python_frame=r"flatten_unencrypted_web_search_results_in_anthropic_messages$" - ), - mapping(span="python_cache_control", python_frame=r"AnthropicCacheControlHook\.maybe_inject_cache_control$"), - mapping(span="python_pre_request_hooks", python_frame=r"_execute_pre_request_hooks$"), - mapping( - span="python_messages_provider_config", - python_frame=r"ProviderConfigManager\.get_provider_anthropic_messages_config$", - ), - mapping(rust_span="messages_provider_config"), - mapping(rust_span="validate_environment", python_frame=r"validate_anthropic_messages_environment$"), - mapping(rust_span="complete_url", python_frame=r"get_complete_url$"), - mapping( - span="python_messages_entry_handler", - python_frame=r"messages/handler\.py:\d+ anthropic_messages_handler$", - ), - mapping( - span="python_messages_handler_wrapper", - python_frame=r"BaseLLMHTTPHandler\.anthropic_messages_handler$", - ), - mapping( - rust_span="execute_messages_provider_call", - python_frame=r"BaseLLMHTTPHandler\.async_anthropic_messages_handler$", - ), - mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"), - mapping(rust_span="transform_response", python_frame=r"(? RouteFixture: +def _fixture(provider: str) -> RouteFixture: conversation: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 16} return RouteFixture( kwargs={ "model": f"{provider}/claude-sonnet-5", - **({"body": {**conversation, "model": "claude-sonnet-5"}} if engine == "rust" else conversation), + **conversation, }, provider_responses=(json_response(anthropic_response_body()),), ) -def _anthropic_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "anthropic") +def _anthropic_fixture(_base_url: str) -> RouteFixture: + return _fixture("anthropic") -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "azure_ai") +def _azure_fixture(_base_url: str) -> RouteFixture: + return _fixture("azure_ai") -def _bedrock_kwargs(engine: Engine) -> dict[str, object]: +def _bedrock_kwargs() -> dict[str, object]: conversation: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 16} return { "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - **( - {"body": {**conversation, "model": "anthropic.claude-3-sonnet-20240229-v1:0"}} - if engine == "rust" - else conversation - ), + **conversation, "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "aws_region_name": "us-east-1", } -def _bedrock_fixture(engine: Engine, _base_url: str) -> RouteFixture: - response_fixture: Final = _fixture(engine, "anthropic") - return RouteFixture(kwargs=_bedrock_kwargs(engine), provider_responses=response_fixture.provider_responses) +def _bedrock_fixture(_base_url: str) -> RouteFixture: + response_fixture: Final = _fixture("anthropic") + return RouteFixture(kwargs=_bedrock_kwargs(), provider_responses=response_fixture.provider_responses) -def _bedrock_retry_fixture(engine: Engine, _base_url: str) -> RouteFixture: - success_fixture: Final = _bedrock_fixture(engine, _base_url) +def _bedrock_retry_fixture(_base_url: str) -> RouteFixture: + success_fixture: Final = _bedrock_fixture(_base_url) messages: Final = [ {"role": "user", "content": "hello"}, { @@ -189,14 +60,7 @@ def _bedrock_retry_fixture(engine: Engine, _base_url: str) -> RouteFixture: }, {"role": "user", "content": "continue"}, ] - kwargs: Final = { - **_bedrock_kwargs(engine), - **( - {"body": {"messages": messages, "max_tokens": 16, "model": "anthropic.claude-3-sonnet-20240229-v1:0"}} - if engine == "rust" - else {"messages": messages} - ), - } + kwargs: Final = {**_bedrock_kwargs(), "messages": messages} return success_fixture.derive( kwargs=kwargs, provider_responses=( @@ -206,13 +70,13 @@ def _bedrock_retry_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _mock_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, "anthropic") +def _mock_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _fixture("anthropic") return fixture.derive(kwargs={"mock_response": "hello from mock"}, provider_responses=()) -def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, "anthropic") +def _provider_error_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _fixture("anthropic") return fixture.derive( provider_responses=( json_response( @@ -224,15 +88,13 @@ def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _sync_unsupported_fixture(engine: Engine, base_url: str) -> RouteFixture: - if engine == "rust": - return _anthropic_fixture(engine, base_url) - fixture: Final = _fixture(engine, "anthropic") +def _sync_unsupported_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _fixture("anthropic") return fixture.derive(provider_responses=(), expected_failure=True) -def _stream_fixture_for(engine: Engine, provider: str) -> RouteFixture: - fixture: Final = _fixture(engine, provider) +def _stream_fixture_for(provider: str) -> RouteFixture: + fixture: Final = _fixture(provider) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(anthropic_stream_events()),), @@ -240,16 +102,16 @@ def _stream_fixture_for(engine: Engine, provider: str) -> RouteFixture: ) -def _stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _stream_fixture_for(engine, "anthropic") +def _stream_fixture(_base_url: str) -> RouteFixture: + return _stream_fixture_for("anthropic") -def _azure_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _stream_fixture_for(engine, "azure_ai") +def _azure_stream_fixture(_base_url: str) -> RouteFixture: + return _stream_fixture_for("azure_ai") -def _bedrock_stream_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _bedrock_fixture(engine, base_url) +def _bedrock_stream_fixture(base_url: str) -> RouteFixture: + fixture: Final = _bedrock_fixture(base_url) events: Final = tuple(payload for _, payload in anthropic_stream_events()) return fixture.derive( kwargs={"stream": True}, @@ -258,8 +120,8 @@ def _bedrock_stream_fixture(engine: Engine, base_url: str) -> RouteFixture: ) -def _bedrock_stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _bedrock_fixture(engine, base_url) +def _bedrock_stream_error_fixture(base_url: str) -> RouteFixture: + fixture: Final = _bedrock_fixture(base_url) start: Final = anthropic_stream_events(model="anthropic.claude-3-sonnet-20240229-v1:0")[0][1] return fixture.derive( kwargs={"stream": True}, @@ -269,56 +131,47 @@ def _bedrock_stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture ) -SPEC: Final = RouteSpec("messages", ("create", "acreate"), ("messages", "amessages"), _anthropic_fixture) +SPEC: Final = RouteSpec("messages", ("create", "acreate"), _anthropic_fixture) TRACE_SUITE: Final = TraceSuite( route=SPEC, scenarios=( - TraceScenario( - name="async-anthropic", fixture=_anthropic_fixture, mappings=ANTHROPIC_MAPPINGS, asynchronous=True - ), - TraceScenario(name="async-azure-ai", fixture=_azure_fixture, mappings=AZURE_MAPPINGS, asynchronous=True), - TraceScenario(name="async-bedrock", fixture=_bedrock_fixture, mappings=BEDROCK_MAPPINGS, asynchronous=True), + TraceScenario(name="async-anthropic", fixture=_anthropic_fixture, asynchronous=True), + TraceScenario(name="async-azure-ai", fixture=_azure_fixture, asynchronous=True), + TraceScenario(name="async-bedrock", fixture=_bedrock_fixture, asynchronous=True), TraceScenario( name="async-bedrock-invalid-thinking-retry", fixture=_bedrock_retry_fixture, - mappings=RETRY_MAPPINGS, asynchronous=True, ), - TraceScenario(name="async-mock-response", fixture=_mock_fixture, mappings=MOCK_MAPPINGS, asynchronous=True), + TraceScenario(name="async-mock-response", fixture=_mock_fixture, asynchronous=True), TraceScenario( name="async-anthropic-provider-error", fixture=_provider_error_fixture, - mappings=ANTHROPIC_FAILURE_MAPPINGS, asynchronous=True, ), TraceScenario( name="async-anthropic-stream", fixture=_stream_fixture, - mappings=(*ANTHROPIC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-azure-ai-stream", fixture=_azure_stream_fixture, - mappings=(*AZURE_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-bedrock-event-stream", fixture=_bedrock_stream_fixture, - mappings=(*BEDROCK_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-bedrock-event-stream-error", fixture=_bedrock_stream_error_fixture, - mappings=(*BEDROCK_MAPPINGS, *STREAM_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="sync-unsupported", fixture=_sync_unsupported_fixture, - mappings=ANTHROPIC_MAPPINGS, asynchronous=False, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py index bb21e8ab0c5..036e6b48026 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py @@ -4,122 +4,10 @@ import json from typing import Final from .....shared.parity.recorded_http import HttpHeader, RecordedHttpResponse -from .....shared.tracing.steps import Engine, mapping from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -COMMON_MAPPINGS: Final = ( - mapping(rust_span="ocr", python_frame=r"ocr/main\.py:\d+ a?ocr$"), - mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"), - mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"), - mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"), - mapping(rust_span="map_ocr_params", python_frame=r"(? RouteFixture: +def _fixture(model: str, document: dict[str, str] | None = None) -> RouteFixture: response: Final = json.dumps( { "pages": [{"index": 0, "markdown": "hello"}], @@ -131,7 +19,7 @@ def _fixture(engine: Engine, model: str, document: dict[str, str] | None = None) kwargs={ "model": model, "document": document or {"type": "document_url", "document_url": "https://example.com/document.pdf"}, - **({"optional_params": {"pages": [0]}} if engine == "rust" else {"pages": [0]}), + "pages": [0], }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -141,12 +29,12 @@ def _fixture(engine: Engine, model: str, document: dict[str, str] | None = None) ) -def _mistral_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "mistral/mistral-ocr-latest") +def _mistral_fixture(_base_url: str) -> RouteFixture: + return _fixture("mistral/mistral-ocr-latest") -def _callback_fixture(engine: Engine, *, failure: bool) -> RouteFixture: - fixture: Final = _fixture(engine, "mistral/mistral-ocr-latest") +def _callback_fixture(*, failure: bool) -> RouteFixture: + fixture: Final = _fixture("mistral/mistral-ocr-latest") provider_responses: Final = ( ( RecordedHttpResponse.from_bytes( @@ -165,29 +53,28 @@ def _callback_fixture(engine: Engine, *, failure: bool) -> RouteFixture: ) -def _mistral_callback_success_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _callback_fixture(engine, failure=False) +def _mistral_callback_success_fixture(_base_url: str) -> RouteFixture: + return _callback_fixture(failure=False) -def _mistral_callback_failure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _callback_fixture(engine, failure=True) +def _mistral_callback_failure_fixture(_base_url: str) -> RouteFixture: + return _callback_fixture(failure=True) -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _azure_fixture(_base_url: str) -> RouteFixture: return _fixture( - engine, "azure_ai/pixtral-12b-2409", {"type": "image_url", "image_url": "data:image/png;base64,aGVsbG8="}, ) -def _vertex_deepseek_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _vertex_deepseek_fixture(_base_url: str) -> RouteFixture: vertex: Final = {"vertex_project": "trace-project", "vertex_location": "us-central1"} return RouteFixture( kwargs={ "model": "vertex_ai/deepseek-ocr-maas", "document": {"type": "image_url", "image_url": "data:image/png;base64,aGVsbG8="}, - **({"optional_params": vertex} if engine == "rust" else vertex), + **vertex, }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -204,11 +91,11 @@ def _vertex_deepseek_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _vertex_deepseek_credentials_fixture(engine: Engine, base_url: str) -> RouteFixture: +def _vertex_deepseek_credentials_fixture(base_url: str) -> RouteFixture: from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa - fixture: Final = _vertex_deepseek_fixture(engine, base_url) + fixture: Final = _vertex_deepseek_fixture(base_url) private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) credentials: Final = json.dumps( { @@ -238,12 +125,12 @@ def _vertex_deepseek_credentials_fixture(engine: Engine, base_url: str) -> Route ) -def _cohere_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _cohere_fixture(_base_url: str) -> RouteFixture: return RouteFixture( kwargs={ "model": "cohere/parse-v5.0", "document": {"type": "image_url", "image_url": "data:image/png;base64,aGVsbG8="}, - **({"optional_params": {"output_format": "blocks"}} if engine == "rust" else {"output_format": "blocks"}), + "output_format": "blocks", }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -260,7 +147,7 @@ def _cohere_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> RouteFixture: +def _azure_document_intelligence_fixture(base_url: str) -> RouteFixture: completed: Final = json.dumps( { "status": "succeeded", @@ -285,7 +172,7 @@ def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> Route "type": "document_url", "document_url": "data:application/pdf;base64,aGVsbG8=", }, - **({"optional_params": {"pages": [0]}} if engine == "rust" else {"pages": [0]}), + "pages": [0], }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -305,204 +192,83 @@ def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> Route ) -DEEPSEEK_COMMON_MAPPINGS: Final = ( - mapping(rust_span="ocr", python_frame=r"ocr/main\.py:\d+ a?ocr$"), - mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"), - mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"), - mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"), - mapping(rust_span="map_ocr_params", python_frame=r"(? RouteFixture: +def _native_fixture(provider: str) -> RouteFixture: model: Final = "gpt-5" return RouteFixture( kwargs={ "model": f"{provider}/{model}", "input": "hello", - **({"body": {"model": model, "input": "hello"}} if engine == "rust" else {}), }, provider_responses=(json_response(responses_body(model=model)),), ) -def _openai_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _native_fixture(engine, "openai") +def _openai_fixture(_base_url: str) -> RouteFixture: + return _native_fixture("openai") -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _native_fixture(engine, "azure") +def _azure_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _native_fixture("azure") return fixture.derive(kwargs={"api_version": "2025-04-01-preview"}) -def _openai_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _openai_fixture(engine, _base_url) +def _openai_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _openai_fixture(_base_url) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(responses_stream_events()),), @@ -107,8 +42,8 @@ def _openai_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _openai_fixture(engine, _base_url) +def _provider_error_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _openai_fixture(_base_url) return fixture.derive( provider_responses=( json_response({"error": {"message": "bad request", "type": "invalid_request_error"}}, status=400), @@ -117,8 +52,8 @@ def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _stream_failed_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _openai_fixture(engine, base_url) +def _stream_failed_fixture(base_url: str) -> RouteFixture: + fixture: Final = _openai_fixture(base_url) failed_response: Final[dict[str, object]] = { **responses_body(), "status": "failed", @@ -140,20 +75,19 @@ def _stream_failed_fixture(engine: Engine, base_url: str) -> RouteFixture: ) -def _anthropic_bridge_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _anthropic_bridge_fixture(_base_url: str) -> RouteFixture: return RouteFixture( kwargs={ "model": "anthropic/claude-sonnet-5", "input": "hello", "max_output_tokens": 16, - **({"body": {"model": "claude-sonnet-5", "input": "hello"}} if engine == "rust" else {}), }, provider_responses=(json_response(anthropic_response_body()),), ) -def _anthropic_bridge_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_bridge_fixture(engine, _base_url) +def _anthropic_bridge_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _anthropic_bridge_fixture(_base_url) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(anthropic_stream_events()),), @@ -161,55 +95,41 @@ def _anthropic_bridge_stream_fixture(engine: Engine, _base_url: str) -> RouteFix ) -SPEC: Final = RouteSpec("responses", ("responses", "aresponses"), None, _openai_fixture) +SPEC: Final = RouteSpec("responses", ("responses", "aresponses"), _openai_fixture) TRACE_SUITE: Final = TraceSuite( route=SPEC, scenarios=( - TraceScenario(name="sync-openai", fixture=_openai_fixture, mappings=COMMON_MAPPINGS, asynchronous=False), - TraceScenario(name="async-openai", fixture=_openai_fixture, mappings=COMMON_MAPPINGS, asynchronous=True), + TraceScenario(name="sync-openai", fixture=_openai_fixture, asynchronous=False), + TraceScenario(name="async-openai", fixture=_openai_fixture, asynchronous=True), TraceScenario( name="sync-openai-stream", fixture=_openai_stream_fixture, - mappings=(*COMMON_MAPPINGS, *STREAM_MAPPINGS), asynchronous=False, ), TraceScenario( name="async-openai-stream", fixture=_openai_stream_fixture, - mappings=(*COMMON_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-openai-provider-error", fixture=_provider_error_fixture, - mappings=(*COMMON_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-openai-stream-failed", fixture=_stream_failed_fixture, - mappings=(*COMMON_MAPPINGS, *STREAM_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), - TraceScenario(name="async-azure", fixture=_azure_fixture, mappings=AZURE_MAPPINGS, asynchronous=True), + TraceScenario(name="async-azure", fixture=_azure_fixture, asynchronous=True), TraceScenario( name="async-anthropic-chat-bridge", fixture=_anthropic_bridge_fixture, - mappings=BRIDGE_MAPPINGS, asynchronous=True, ), TraceScenario( name="async-anthropic-chat-bridge-stream", fixture=_anthropic_bridge_stream_fixture, - mappings=( - *BRIDGE_MAPPINGS, - mapping(span="python_chat_stream_wrapper", python_frame=r"CustomStreamWrapper\.__init__$"), - mapping(span="python_chat_stream_next", python_frame=r"CustomStreamWrapper\.__anext__$"), - mapping( - span="python_responses_bridge_stream_iterator", - python_frame=r"LiteLLMCompletionStreamingIterator\.__init__$|LiteLLMCompletionStreamingIterator\.__anext__$", - ), - ), asynchronous=True, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py b/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py index d0dbd281a97..47c5948af75 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py @@ -41,8 +41,8 @@ def test_core_sdk_scenario_matrix_keeps_distinct_migration_paths() -> None: } assert {(scenario.name, scenario.asynchronous) for scenario in ocr.scenarios} >= { ("async-cohere", True), - ("sync-public-rust-dispatch", False), - ("async-public-rust-dispatch", True), + ("sync-vertex-deepseek", False), + ("async-vertex-deepseek", True), } assert {(scenario.name, scenario.asynchronous) for scenario in responses.scenarios} >= { ("sync-openai", False), @@ -55,15 +55,3 @@ def test_core_sdk_scenario_matrix_keeps_distinct_migration_paths() -> None: ("async-anthropic-chat-bridge", True), ("async-anthropic-chat-bridge-stream", True), } - - -def test_core_gateway_matrix_keeps_downstream_streams_separate() -> None: - modules: Final = ( - "tests.rust-python-harness.strategies.trace_parity.gateway.chat_completions.case", - "tests.rust-python-harness.strategies.trace_parity.gateway.messages.case", - "tests.rust-python-harness.strategies.trace_parity.gateway.responses.case", - ) - - for module in modules: - suite = _suite(module) - assert any("downstream-stream" in scenario.name for scenario in suite.scenarios) diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py index 3b4d2e1447d..2071e00d3d6 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py @@ -7,41 +7,8 @@ import wave from typing import Final from .....shared.parity.recorded_http import HttpHeader, RecordedHttpResponse -from .....shared.tracing.steps import Engine, mapping from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -MAPPINGS: Final = ( - mapping(rust_span="prepare_audio_transcription_provider_call"), - mapping(span="get_non_default_params", python_frame=r"get_non_default_transcription_params$"), - mapping(rust_span="map_transcription_params", python_frame=r"get_optional_params_transcription$"), - mapping( - span="python_provider_config", - python_frame=r"ProviderConfigManager\.get_provider_audio_transcription_config$", - ), - mapping(rust_span="provider_config"), - mapping(rust_span="supported_transcription_params"), - mapping(rust_span="transform_transcription_request"), - mapping( - rust_span="execute_audio_transcription_provider_call", - python_frame=r"BedrockAudioTranscriptionRustDispatch\.(?:async_)?audio_transcriptions$", - ), - mapping(rust_span="transform_transcription_response"), - mapping(rust_span="http_request"), -) - -SYNC_MAPPINGS: Final = ( - mapping(rust_span="audio_transcription", python_frame=r"main\.py:\d+ transcription$"), - *MAPPINGS, -) -ASYNC_MAPPINGS: Final = ( - mapping(rust_span="audio_transcription", python_frame=r"main\.py:\d+ atranscription$"), - mapping(span="python_transcription_wrapper", python_frame=r"main\.py:\d+ transcription$"), - *MAPPINGS[:2], - mapping(span="python_map_transcription_params", python_frame=r"get_optional_params_transcription$"), - mapping(rust_span="map_transcription_params"), - *MAPPINGS[3:], -) - def _audio_bytes() -> bytes: with io.BytesIO() as buffer: @@ -53,18 +20,14 @@ def _audio_bytes() -> bytes: return buffer.getvalue() -def _fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _fixture(_base_url: str) -> RouteFixture: credentials: Final = { "aws_access_key_id": "test-access", "aws_secret_access_key": "test-secret", "aws_region_name": "us-east-1", } audio: Final = _audio_bytes() - payload: Final = ( - {"audio": {"data": base64.b64encode(audio).decode(), "format": "wav"}, "optional_params": credentials} - if engine == "rust" - else {"file": ("sample.wav", audio, "audio/wav"), **credentials} - ) + payload: Final = {"file": ("sample.wav", audio, "audio/wav"), **credentials} response: Final = json.dumps( { "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}}, @@ -85,7 +48,6 @@ def _fixture(engine: Engine, _base_url: str) -> RouteFixture: SPEC: Final = RouteSpec( "transcription", ("transcription", "atranscription"), - ("transcription", "atranscription"), _fixture, ) TRACE_SUITE: Final = TraceSuite( @@ -94,13 +56,11 @@ TRACE_SUITE: Final = TraceSuite( TraceScenario( name="sync-bedrock", fixture=_fixture, - mappings=SYNC_MAPPINGS, asynchronous=False, ), TraceScenario( name="async-bedrock", fixture=_fixture, - mappings=ASYNC_MAPPINGS, asynchronous=True, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/test_reporting.py b/tests/rust-python-harness/strategies/trace_parity/test_reporting.py index 22cc87592b8..2d5ed14b6cd 100644 --- a/tests/rust-python-harness/strategies/trace_parity/test_reporting.py +++ b/tests/rust-python-harness/strategies/trace_parity/test_reporting.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Final, Literal +from typing import Final import pytest @@ -28,20 +28,16 @@ def _result(trace: TraceArtifact) -> CaseResult: def _trace( python: tuple[PipelineStep, ...], - rust: tuple[PipelineStep, ...], *, - rust_error: str | None = None, - engine: Literal["python", "rust", "both"] = "both", + python_error: str | None = None, scenario: str = "sync-default", ) -> TraceArtifact: return TraceArtifact.from_traces( - engine=engine, surface="sdk", sdk_function="ocr", scenario=scenario, python=python, - rust=rust, - rust_error=rust_error, + python_error=python_error, ) @@ -55,52 +51,29 @@ def _events(*items: tuple[str, int, str | None]) -> tuple[PipelineStep, ...]: return tuple(steps) -def test_renderer_prints_python_and_rust_traces_independently() -> None: +def test_renderer_prints_the_python_trace() -> None: python: Final = _events( ("ocr", 0, "ocr/main.py:88 aocr"), ("python_prepare", 1, "prep.py:1 python_prepare"), ) - rust: Final = _events(("ocr", 0, None), ("rust_prepare", 1, None)) - section: Final = render_trace_results((_result(_trace(python, rust)),))[0] + section: Final = render_trace_results((_result(_trace(python)),))[0] report: Final = "\n\n".join(section.blocks) assert section.title == "SDK traces" assert "PYTHON (2 steps)\n1 aocr (ocr/main.py:88)\n2 python_prepare (prep.py:1)" in report - assert "RUST (2 steps)\n1 ocr\n2 rust_prepare" in report - assert "python only" not in report - assert "rust only" not in report - assert " -> " not in report - assert "Trace: MATCH" not in report - assert "Trace: DRIFT" not in report - assert "Contract:" not in report + assert "RUST" not in report -@pytest.mark.parametrize( - ("engine", "present", "absent"), - (("python", "PYTHON (1 steps)", "RUST"), ("rust", "RUST (1 steps)", "PYTHON")), -) -def test_renderer_prints_only_selected_engine(engine: Literal["python", "rust"], present: str, absent: str) -> None: - events: Final = _events(("ocr", 0, None)) - - report: Final = "\n\n".join(render_trace_results((_result(_trace(events, events, engine=engine)),))[0].blocks) - - assert present in report - assert absent not in report - - -def test_renderer_keeps_collected_trace_when_one_engine_errors() -> None: +def test_renderer_keeps_collected_trace_when_python_errors() -> None: python: Final = _events(("ocr", 0, "ocr/main.py:88 aocr")) report: Final = "\n\n".join( - render_trace_results( - (_result(_trace(python, (), rust_error="rust: native Rust bridge must include the trace-parity feature")),) - )[0].blocks + render_trace_results((_result(_trace(python, python_error="python: replay server closed")),))[0].blocks ) assert "PYTHON (1 steps)\n1 aocr (ocr/main.py:88)" in report - assert "Rust error: rust: native Rust bridge must include the trace-parity feature" in report - assert "hint: rebuild the native bridge with the trace-parity feature" in report + assert "Python error: python: replay server closed" in report def test_unavailable_trace_reports_scenario_from_nodeid() -> None: @@ -122,8 +95,8 @@ def test_unavailable_trace_reports_scenario_from_nodeid() -> None: def test_renderer_groups_scenarios_under_one_case_header() -> None: - result: Final = _result(_trace(_events(("ocr", 0, None)), (), scenario="sync-default")) - async_trace: Final = _trace((), _events(("ocr", 0, None)), scenario="async-default") + result: Final = _result(_trace(_events(("ocr", 0, None)), scenario="sync-default")) + async_trace: Final = _trace(_events(("ocr", 0, None)), scenario="async-default") nodeid: Final = "trace:sdk:ocr:async-default" result.collected.add(nodeid) result.record(nodeid, RunStatus.PASSED, artifacts=(ResultArtifact(TRACE_ARTIFACT, async_trace.model_dump_json()),)) @@ -140,12 +113,10 @@ def test_renderer_colors_every_trace_line_in_a_terminal(monkeypatch: pytest.Monk monkeypatch.setattr(reporting.sys.stdout, "isatty", lambda: True) monkeypatch.delenv("NO_COLOR", raising=False) - report: Final = "\n\n".join(render_trace_results((_result(_trace(events, events)),))[0].blocks) + report: Final = "\n\n".join(render_trace_results((_result(_trace(events)),))[0].blocks) assert "\033[36mPYTHON\033[0m (1 steps)" in report assert "\033[36m1 aocr (ocr/main.py:88)\033[0m" in report - assert "\033[33mRUST\033[0m (1 steps)" in report - assert "\033[33m1 ocr\033[0m" in report def test_renderer_groups_unavailable_entries_by_surface() -> None: @@ -160,7 +131,7 @@ def test_renderer_groups_unavailable_entries_by_surface() -> None: status=RunStatus.NOT_IMPLEMENTED, ) - sections: Final = render_trace_results((_result(_trace((), ())), gateway_result)) + sections: Final = render_trace_results((_result(_trace(())), gateway_result)) assert tuple(section.title for section in sections) == ("SDK traces", "GATEWAY traces") assert "- messages: No messages case is registered." in "\n\n".join(sections[1].blocks) diff --git a/tests/rust-python-harness/strategies/trace_parity/test_runner.py b/tests/rust-python-harness/strategies/trace_parity/test_runner.py index be25dd53b02..a5b66ab0088 100644 --- a/tests/rust-python-harness/strategies/trace_parity/test_runner.py +++ b/tests/rust-python-harness/strategies/trace_parity/test_runner.py @@ -13,14 +13,14 @@ import litellm from ...shared.reporting.models import Coverage, HarnessCase, HarnessRun, RunStatus, SdkFunction, Surface from ...shared.reporting.strategy import ModuleCaseSpec from ...shared.tracing.profiler import FunctionTraceEvent -from ...shared.tracing.steps import Engine, PipelineStep, mapping -from .models import GatewayRouteSpec, RouteFixture, RouteSpec, TraceScenario, TraceSuite +from ...shared.tracing.steps import PipelineStep +from .models import RouteFixture, RouteSpec, TraceScenario, TraceSuite from .reporting import TraceArtifact -from .runner import run_trace_cases, run_trace_scenario, runner_selection, scenario_nodeids, validate_trace_suite +from .runner import run_trace_cases, run_trace_scenario, scenario_nodeids, validate_trace_suite from .sdk.execution import SdkCall, collect_trace, execute_trace -def _fixture(_engine: Engine, _base_url: str) -> RouteFixture: +def _fixture(_base_url: str) -> RouteFixture: return RouteFixture(kwargs={}, provider_responses=()) @@ -36,11 +36,11 @@ def _case(*, surface: Surface = "sdk", function: SdkFunction = "ocr") -> Harness def test_scenario_filtering_and_occurrence_node_ids() -> None: suite: Final = TraceSuite( - route=RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), scenarios=( - TraceScenario("sync-one", _fixture, (), asynchronous=False), - TraceScenario("async-one", _fixture, (), asynchronous=True), - TraceScenario("async-two", _fixture, (), asynchronous=True), + TraceScenario("sync-one", _fixture, asynchronous=False), + TraceScenario("async-one", _fixture, asynchronous=True), + TraceScenario("async-two", _fixture, asynchronous=True), ), ) case: Final = _case() @@ -50,45 +50,35 @@ def test_scenario_filtering_and_occurrence_node_ids() -> None: assert tuple(nodeid for _, nodeid in nodes) == ("trace:sdk:ocr:async-two",) -def test_python_engine_is_separate_from_scenario_selection() -> None: - assert runner_selection(("mistral", "--engine=python")) == (frozenset({"mistral"}), "python") - - -def test_python_engine_skips_native_bridge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_runner_arguments_select_scenarios(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: runner: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.runner") case: Final = _case() - selected: list[tuple[frozenset[str], str]] = [] - - def reject_bridge(_repo_root: Path) -> str | None: - raise AssertionError("Python-only tracing must not inspect or build the native bridge") + selected: list[frozenset[str]] = [] def capture_case( _run: HarnessRun, _case: HarnessCase, scenarios: frozenset[str], _on_update: object, - engine: str, ) -> None: - selected.append((scenarios, engine)) + selected.append(scenarios) - monkeypatch.setattr(runner, "ensure_trace_bridge", reject_bridge) monkeypatch.setattr(runner, "_run_case", capture_case) - exit_code, _ = run_trace_cases((case,), tmp_path, lambda _: None, ("mistral", "--engine=python")) + exit_code, _ = run_trace_cases((case,), tmp_path, lambda _: None, ("mistral",)) assert exit_code == 0 - assert selected == [(frozenset({"mistral"}), "python")] + assert selected == [frozenset({"mistral"})] def test_python_trace_preserves_native_ocr_dispatch_setting(monkeypatch: pytest.MonkeyPatch) -> None: execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.sdk.execution") - route: Final = RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture) + route: Final = RouteSpec("ocr", ("ocr", "aocr"), _fixture) observed: list[str | None] = [] def collect( _function: SdkCall, _fixture: RouteFixture, - _engine: Engine, *, asynchronous: bool, ) -> SimpleNamespace: @@ -101,9 +91,9 @@ def test_python_trace_preserves_native_ocr_dispatch_setting(monkeypatch: pytest. monkeypatch.setattr(execution, "_collect", collect) monkeypatch.setenv("LITELLM_RUST", "0") - collect_trace(route, "python", asynchronous=False) + collect_trace(route, asynchronous=False) monkeypatch.setenv("LITELLM_RUST", "1") - collect_trace(route, "python", asynchronous=True) + collect_trace(route, asynchronous=True) assert observed == ["0", "1"] assert os.environ["LITELLM_RUST"] == "1" @@ -116,9 +106,8 @@ def test_expected_provider_failure_omits_feedback_banner( suite: Final = cast(TraceSuite, loaded.TRACE_SUITE) scenario: Final = next(item for item in suite.scenarios if item.name == "async-openai-provider-error") monkeypatch.setattr(litellm, "suppress_debug_info", False) - assert isinstance(suite.route, RouteSpec) - result: Final = execute_trace(suite.route, scenario, "sdk", engine="python") + result: Final = execute_trace(suite.route, scenario, "sdk") assert result.python_error is None assert "Give Feedback / Get Help" not in capsys.readouterr().out @@ -131,9 +120,8 @@ def test_vertex_trace_keeps_unmapped_helpers_and_parents(asynchronous: bool) -> suite: Final = cast(TraceSuite, loaded.TRACE_SUITE) name: Final = f"{'async' if asynchronous else 'sync'}-vertex-deepseek" scenario: Final = next(item for item in suite.scenarios if item.name == name) - assert isinstance(suite.route, RouteSpec) - trace: Final = execute_trace(suite.route, scenario, "sdk", engine="python") + trace: Final = execute_trace(suite.route, scenario, "sdk") assert trace.python_error is None url: Final = next( @@ -157,9 +145,8 @@ def test_vertex_credentials_trace_runs_real_auth_helpers(asynchronous: bool, mon scenario: Final = next(item for item in suite.scenarios if item.name == name) monkeypatch.setenv("VERTEXAI_CREDENTIALS", "original-credentials") monkeypatch.setenv("VERTEX_AI_API_KEY", "original-api-key") - assert isinstance(suite.route, RouteSpec) - trace: Final = execute_trace(suite.route, scenario, "sdk", engine="python") + trace: Final = execute_trace(suite.route, scenario, "sdk") assert trace.python_error is None validate: Final = next( @@ -180,79 +167,16 @@ def test_vertex_credentials_trace_runs_real_auth_helpers(asynchronous: bool, mon assert os.environ["VERTEX_AI_API_KEY"] == "original-api-key" -def test_gateway_trace_keeps_calls_outside_scenario_mappings(monkeypatch: pytest.MonkeyPatch) -> None: - execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.gateway.execution") - events: Final = ( - FunctionTraceEvent(0, None, "route.py:1 entry"), - FunctionTraceEvent(1, 0, "auth.py:2 authenticate"), - FunctionTraceEvent(2, 1, "auth.py:3 credentials"), - ) - scenario: Final = TraceScenario( - "async-gateway", - _fixture, - (mapping(rust_span="entry", python_frame=r" entry$"),), - asynchronous=True, - ) - monkeypatch.setattr(execution, "_collect", lambda *_args: events) - - trace: Final = execution.execute_gateway_trace(GatewayRouteSpec("messages"), scenario, engine="python") - - assert trace.python_error is None - assert tuple((event.id, event.parent_id, event.raw) for event in trace.python) == tuple( - (event.id, event.parent_id, event.raw) for event in events - ) - - -def test_default_trace_skips_unavailable_rust_sdk_entrypoint(monkeypatch: pytest.MonkeyPatch) -> None: - execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.sdk.execution") - route: Final = RouteSpec("responses", ("responses", "aresponses"), None, _fixture) - scenario: Final = TraceScenario("sync-openai", _fixture, (), asynchronous=False) - engines: list[Engine] = [] - - def collect(_route: RouteSpec, engine: Engine, *, asynchronous: bool) -> tuple[FunctionTraceEvent, ...]: - engines.append(engine) - return (FunctionTraceEvent(0, None, "responses"),) - - monkeypatch.setattr(execution, "collect_trace", collect) - - trace: Final = execution.execute_trace(route, scenario, "sdk") - - assert engines == ["python"] - assert trace.engine == "python" - assert trace.rust_error is None - - -def test_default_trace_skips_unavailable_rust_gateway_route(monkeypatch: pytest.MonkeyPatch) -> None: - execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.gateway.execution") - route: Final = GatewayRouteSpec("responses", rust_supported=False) - scenario: Final = TraceScenario("async-openai", _fixture, (), asynchronous=True) - engines: list[Engine] = [] - - def collect(_route: GatewayRouteSpec, _scenario: TraceScenario, engine: Engine) -> tuple[FunctionTraceEvent, ...]: - engines.append(engine) - return (FunctionTraceEvent(0, None, "responses"),) - - monkeypatch.setattr(execution, "_collect", collect) - - trace: Final = execution.execute_gateway_trace(route, scenario) - - assert engines == ["python"] - assert trace.engine == "python" - assert trace.rust_error is None - - def test_scenario_validation_rejects_duplicate_and_unsafe_names() -> None: - route: Final = RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture) + route: Final = RouteSpec("ocr", ("ocr", "aocr"), _fixture) duplicate: Final = TraceSuite( route=route, scenarios=( - TraceScenario("sync-same", _fixture, (), asynchronous=False), - TraceScenario("sync-same", _fixture, (), asynchronous=False), + TraceScenario("sync-same", _fixture, asynchronous=False), + TraceScenario("sync-same", _fixture, asynchronous=False), ), ) - unsafe: Final = TraceSuite( - route=route, scenarios=(TraceScenario("sync-bad:name", _fixture, (), asynchronous=False),) - ) + unsafe: Final = TraceSuite(route=route, scenarios=(TraceScenario("sync-bad:name", _fixture, asynchronous=False),)) case: Final = _case() assert validate_trace_suite(duplicate, case) is not None @@ -261,22 +185,22 @@ def test_scenario_validation_rejects_duplicate_and_unsafe_names() -> None: def test_scenario_validation_rejects_invalid_names_and_route_registration() -> None: invalid_name: Final = TraceSuite( - route=RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture), - scenarios=(TraceScenario("bedrock", _fixture, (), asynchronous=True),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("bedrock", _fixture, asynchronous=True),), ) wrong_function: Final = TraceSuite( - route=RouteSpec("messages", ("create", "acreate"), ("messages", "amessages"), _fixture), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("messages", ("create", "acreate"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) wrong_surface: Final = TraceSuite( - route=GatewayRouteSpec("ocr"), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) case: Final = _case() assert "start with sync- or async-" in (validate_trace_suite(invalid_name, case) or "") assert "does not match case function" in (validate_trace_suite(wrong_function, case) or "") - assert "must use RouteSpec" in (validate_trace_suite(wrong_surface, case) or "") + assert "requires the sdk surface" in (validate_trace_suite(wrong_surface, _case(surface="gateway")) or "") def test_invalid_route_dispatch_records_harness_error() -> None: @@ -284,32 +208,31 @@ def test_invalid_route_dispatch_records_harness_error() -> None: run: Final = HarnessRun.from_cases((case,)) result: Final = run.results[case.key] suite: Final = TraceSuite( - route=GatewayRouteSpec("ocr"), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) - nodeid: Final = "trace:sdk:ocr:sync-one" + nodeid: Final = "trace:gateway:ocr:sync-one" - run_trace_scenario(run, result, suite, suite.scenarios[0], "sdk", nodeid, lambda _: None) + run_trace_scenario(run, result, suite, suite.scenarios[0], "gateway", nodeid, lambda _: None) assert result.outcomes[nodeid] is RunStatus.ERROR - assert run.failures == [(nodeid, "gateway route cannot run on the sdk surface")] + assert run.failures == [(nodeid, "trace scenarios only run on the sdk surface")] -def test_different_python_and_rust_traces_pass(monkeypatch: pytest.MonkeyPatch) -> None: +def test_python_trace_without_errors_passes(monkeypatch: pytest.MonkeyPatch) -> None: runner: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.runner") case: Final = _case() run: Final = HarnessRun.from_cases((case,)) result: Final = run.results[case.key] suite: Final = TraceSuite( - route=RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) trace: Final = TraceArtifact.from_traces( surface="sdk", sdk_function="ocr", scenario="sync-one", python=(PipelineStep(0, None, "python_step", "python.py:1 python_step"),), - rust=(PipelineStep(0, None, "rust_step", "rust_step"),), ) monkeypatch.setattr(runner, "_execute_scenario", lambda *_args: trace) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md b/tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md deleted file mode 100644 index 379d1443f33..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md +++ /dev/null @@ -1,13 +0,0 @@ -# What this is - -Validates that unit tests covering traced Python behavior have semantic counterparts among colocated Rust unit tests - -# How it works - -Trace parity runs representative public API scenarios and records the Python and Rust functions reached, including their source files and lines. The OCR contract selects the behavior-level trace spans that require parity and excludes shared infrastructure such as generic HTTP transport - -For Python, those traced functions define the denominator. Static references and explicit includes create a safe pytest discovery universe, then a pytest profiler keeps only tests that actually execute at least one selected function. Static matches do not count by themselves. Parametrized pytest cases are collapsed to one logical test function in the mapping report. Explicit includes and exclusions cover dynamic callers or intentional harness behavior that static discovery cannot express reliably - -For Rust, each traced function identifies its source file and module. If that source file has a colocated `#[cfg(test)] mod tests`, the harness inventories that module for the configured Rust target. Rust test names are therefore derived from traced implementation files, not from a hand-maintained list of OCR test modules - -The Python-to-Rust mappings remain explicit because equivalent behavior often has different test boundaries and names in each SDK. Host-only exclusions require a reason. The report validates both against the live inventories, then shows mapped, excluded, and unmapped Python tests plus Rust-only tests diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py b/tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py deleted file mode 100644 index 4d857c01ed0..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from functools import partial -from pathlib import Path -from typing import Final - -from ...shared.reporting.models import SDK_FUNCTIONS, Coverage -from ...shared.reporting.strategy import ( - CaseDefinition, - NotImplementedCaseSpec, - RunnerArgumentDefinition, - StrategyDefinition, - SuiteCaseSpec, -) -from ...shared.unit_runners.suite_runner import run_suites -from .mappings import UNIT_TEST_CONTRACTS -from .reporting import render_mapping_results -from .runner import run_suite - - -CASES: Final[tuple[CaseDefinition, ...]] = ( - *( - CaseDefinition( - sdk_function, - SuiteCaseSpec(coverage=Coverage.COMPLETE, suite=sdk_function) - if sdk_function in UNIT_TEST_CONTRACTS - else NotImplementedCaseSpec(reason=f"No {sdk_function} unit-test mapping is registered."), - ) - for sdk_function in SDK_FUNCTIONS - ), -) - -STRATEGY: Final = StrategyDefinition( - id="unit_tests_mapping", - order=30, - label="Unit test mapping", - description="Validate Python/Rust unit-test mappings against collected test inventories.", - directory=Path(__file__).parent, - runnable_spec=SuiteCaseSpec, - cases=CASES, - run=partial(run_suites, suites=UNIT_TEST_CONTRACTS, execute=run_suite), - render=render_mapping_results, - runner_argument=RunnerArgumentDefinition( - option="--detail", - metavar="MODE", - help="show individual test names; any value enables full detail", - ), -) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py b/tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py deleted file mode 100644 index 8b137891791..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py b/tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py deleted file mode 100644 index 0e771f0dc17..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py +++ /dev/null @@ -1,422 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from ....shared.unit_runners.rust_runner import RustTarget, RustTestIdentity -from ..contracts import ( - MappingExclusionSpec, - MappingSpec, - PythonFunctionDiscoverySpec, - RustTestFamily, - RustUnitSpec, - TestMapping, - UnitParityExclusionSpec, - UnitParitySpec, - UnitTestContract, -) - -_CORE_TARGET: Final = RustTarget(package="litellm-core", name="litellm_core", kind="lib") -_GATEWAY_TARGET: Final = RustTarget( - package="litellm-ai-gateway", - name="litellm_ai_gateway", - kind="lib", -) -_AZURE_OCR_TESTS: Final = "providers::azure_ai::ocr::transformation::tests" -_MISTRAL_OCR_TESTS: Final = "providers::mistral::ocr::transformation::tests" -_VERTEX_OCR_TESTS: Final = "providers::vertex_ai::ocr::transformation::tests" -_REDUCTO_OCR_TESTS: Final = "providers::reducto::ocr::tests" -_GATEWAY_OCR_TESTS: Final = "ocr::tests" -_GATEWAY_PREPARE_OCR_TESTS: Final = "ocr::prepare::tests" - - -def _rust_test(target: RustTarget, module: str, test: str) -> RustTestIdentity: - return RustTestIdentity(target=target, name=f"{module}::{test}") - - -def _rust_family(target: RustTarget, module: str, test: str) -> RustTestFamily: - return RustTestFamily(target=target, name=f"{module}::{test}") - - -def _test_mappings(target: RustTarget, module: str, pairs: tuple[tuple[str, str], ...]) -> tuple[TestMapping, ...]: - return tuple(TestMapping(python=python, rust=_rust_test(target, module, test)) for python, test in pairs) - - -_AZURE_TRANSFORM_FILE: Final = "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py" -_AZURE_PAGES_FILE: Final = "tests/ocr_tests/test_ocr_azure_document_intelligence.py" -_AZURE_BASE_FILE: Final = "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py" -_RUST_BRIDGE_FILE: Final = "tests/test_litellm/ocr/test_rust_bridge.py" - -_AZURE_PORT_MAPPINGS: Final = _test_mappings( - _CORE_TARGET, - _AZURE_OCR_TESTS, - ( - ( - f"{_AZURE_TRANSFORM_FILE}::test_should_encode_azure_document_intelligence_model_id", - "azure_document_intelligence_model_id_is_encoded", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_should_reject_dot_segment_azure_document_intelligence_model_id", - "azure_document_intelligence_dot_segment_model_id_is_rejected", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_async_transform_ocr_response_preserves_azure_native_fields", - "document_intelligence_async_response_preserves_normalized_fields", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_tolerates_missing_native_fields", - "document_intelligence_response_tolerates_missing_native_fields", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_non_succeeded_status_raises", - "document_intelligence_non_succeeded_status_is_rejected", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_get_supported_ocr_params_includes_features", - "document_intelligence_supported_params_include_features", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_native_format_carries_raw_operation", - "document_intelligence_native_format_carries_raw_operation", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_async_transform_ocr_response_native_format_carries_raw_operation", - "document_intelligence_async_native_format_carries_raw_operation", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_map_ocr_params_rejects_unknown_req_format_as_bad_request", - "document_intelligence_rejects_unknown_req_format", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_get_complete_url_omits_req_format_query_param", - "document_intelligence_url_omits_req_format", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_validate_environment_uses_subscription_key", - "document_intelligence_validate_environment_uses_subscription_key", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_validate_environment_falls_back_to_entra_token", - "document_intelligence_validate_environment_falls_back_to_entra_token", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_get_supported_ocr_params_includes_pages_and_features", - "document_intelligence_supported_params_include_pages_features_and_req_format", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_mistral_zero_based_int_list", - "document_intelligence_maps_zero_based_page_list", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_dedupes_and_sorts", - "document_intelligence_page_mapping_dedupes_and_sorts", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_empty_list_omits_pages", - "document_intelligence_page_mapping_omits_empty_list", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_azure_native_string_range", - "document_intelligence_page_mapping_accepts_native_range", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_azure_native_string_with_spaces_stripped", - "document_intelligence_page_mapping_strips_spaces", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_list_of_string_tokens", - "document_intelligence_page_mapping_accepts_string_tokens", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_invalid_string_raises", - "document_intelligence_page_mapping_rejects_invalid_string", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_negative_index_raises", - "document_intelligence_page_mapping_rejects_negative_index", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_bool_list_raises", - "document_intelligence_page_mapping_rejects_bool_list", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_unsupported_type_raises", - "document_intelligence_page_mapping_rejects_unsupported_type", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_get_complete_url_appends_pages_query", - "document_intelligence_url_appends_pages_query", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_get_complete_url_no_pages_when_optional_params_empty", - "document_intelligence_url_has_no_pages_when_params_are_empty", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_transform_ocr_request_does_not_put_pages_in_body", - "document_intelligence_request_keeps_pages_out_of_body", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_end_to_end_mistral_shape_to_azure_query", - "document_intelligence_mistral_pages_flow_to_query_only", - ), - ( - "tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py::test_ocr_authenticates_with_entra_token", - "azure_ai_ocr_authenticates_with_entra_token", - ), - ( - f"{_AZURE_BASE_FILE}::TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", - "document_intelligence_endpoint_ignores_generic_azure_ai_base", - ), - ( - f"{_AZURE_BASE_FILE}::TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", - "document_intelligence_endpoint_honors_explicit_api_base", - ), - ( - f"{_AZURE_BASE_FILE}::TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", - "azure_ai_mistral_ocr_uses_generic_api_base", - ), - ), -) - -_REDUCTO_PORT_MAPPINGS: Final = _test_mappings( - _CORE_TARGET, - _REDUCTO_OCR_TESTS, - ( - ( - "tests/test_litellm/llms/reducto/test_parse_v3.py::test_parse_v3_reducto_id_passthrough_skips_upload", - "test_parse_v3_reducto_id_passthrough_skips_upload", - ), - ( - "tests/test_litellm/llms/reducto/test_parse_legacy.py::test_parse_legacy_wraps_enhance_under_options", - "test_parse_legacy_wraps_enhance_under_options", - ), - ( - "tests/test_litellm/llms/reducto/test_upload.py::test_parse_v3_image_data_uri_upload_uses_image_mime", - "test_parse_v3_image_data_uri_upload_uses_image_mime", - ), - ( - "tests/test_litellm/llms/reducto/test_upload.py::test_parse_v3_uses_programmatic_api_key_over_env", - "test_parse_v3_uses_programmatic_api_key_over_env", - ), - ), -) - -_REDUCTO_GATEWAY_MAPPING: Final = TestMapping( - python="tests/test_litellm/llms/reducto/test_parse_v3.py::test_parse_v3_file_upload_and_response_mapping", - rust=_rust_test(_GATEWAY_TARGET, _GATEWAY_OCR_TESTS, "reducto_file_upload_then_parse_maps_response"), -) - -_GATEWAY_PORT_MAPPINGS: Final = _test_mappings( - _GATEWAY_TARGET, - _GATEWAY_PREPARE_OCR_TESTS, - ( - ( - "tests/test_litellm/ocr/test_ocr_native_format.py::test_native_format_rejected_for_provider_without_support_as_bad_request", - "native_format_rejected_for_provider_without_support_as_bad_request", - ), - ( - "tests/test_litellm/ocr/test_ocr_native_format.py::test_unknown_format_rejected_for_provider_without_support_as_bad_request", - "unknown_format_rejected_for_provider_without_support_as_bad_request", - ), - ), -) - -_HOST_ONLY_BRIDGE_EXCLUSIONS: Final = tuple( - MappingExclusionSpec(nodeid=f"{_RUST_BRIDGE_FILE}::{test}", reason=reason) - for test, reason in ( - ("test_ocr_routes_to_rust_when_enabled", "Python selects and invokes the native bridge."), - ("test_ocr_routes_azure_ai_to_rust_when_enabled", "Python resolves provider arguments before the bridge."), - ("test_ocr_rust_path_converts_file_document_before_bridge", "Python converts file inputs before the bridge."), - ( - "test_ocr_exception_type_uses_resolved_provider_context", - "Python wraps bridge exceptions into public errors.", - ), - ( - "test_rust_upstream_error_uses_ocr_provider_error_mapping", - "Python maps native upstream errors through the selected OCR provider config.", - ), - ("test_aocr_routes_to_async_rust_when_enabled", "Python selects and invokes the async native bridge."), - ("test_aocr_exception_type_uses_resolved_provider_context", "Python wraps async bridge exceptions."), - ("test_ocr_forwards_timeout_to_rust", "Python converts and forwards explicit timeouts."), - ("test_ocr_passes_default_request_timeout_to_rust", "Python supplies its process-level default timeout."), - ("test_ocr_falls_back_to_python_when_bridge_unavailable", "Python owns fallback when the extension is absent."), - ) -) - -_FAMILY_PORT_MAPPINGS: Final = ( - TestMapping( - python=f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_default_format_omits_raw_operation", - rust=_rust_family( - _CORE_TARGET, - _AZURE_OCR_TESTS, - "document_intelligence_default_format_omits_raw_operation", - ), - ), - TestMapping( - python=f"{_AZURE_TRANSFORM_FILE}::test_map_ocr_params_passes_through_req_format", - rust=_rust_family(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_maps_req_format"), - ), - TestMapping( - python="tests/ocr_tests/test_ocr_vertex_ai.py::test_deepseek_request_uses_single_provider_namespace", - rust=_rust_family( - _CORE_TARGET, - _VERTEX_OCR_TESTS, - "vertex_deepseek_request_uses_single_provider_namespace", - ), - ), - TestMapping( - python="tests/test_litellm/llms/reducto/test_upload.py::test_parse_v3_rejects_plain_http_urls", - rust=_rust_family(_CORE_TARGET, _REDUCTO_OCR_TESTS, "test_parse_v3_rejects_plain_http_urls"), - ), -) - - -OCR_CONTRACT: Final = UnitTestContract( - mapping=MappingSpec( - python_functions=PythonFunctionDiscoverySpec( - trace_module="tests.rust-python-harness.strategies.trace_parity.sdk.ocr.case", - trace_spans=( - "ocr", - "prepare_ocr_call", - "ocr_provider_config", - "supported_ocr_params", - "map_ocr_params", - "validate_environment", - "complete_url", - "transform_ocr_request", - "execute_ocr_provider_call", - "transform_ocr_response", - "poll_document_intelligence", - ), - search_roots=("tests",), - exclude_roots=( - "tests/e2e", - "tests/ocr_tests/test_ocr_mistral.py", - "tests/rust-python-harness", - ), - includes=( - "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "tests/test_litellm/llms/mistral/ocr", - "tests/test_litellm/llms/ocr", - "tests/test_litellm/ocr", - "tests/test_litellm/proxy/ocr_endpoints", - ), - exclusions=( - "tests/ocr_tests/test_ocr_azure_document_intelligence.py::TestAzureDocumentIntelligenceOCR", - "tests/ocr_tests/test_ocr_vertex_ai.py::TestVertexAIMistralOCR", - "tests/ocr_tests/test_ocr_vertex_ai.py::TestVertexAIDeepSeekOCR", - ), - ), - rust_targets=(_CORE_TARGET, _GATEWAY_TARGET), - mappings=( - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_transform_ocr_response_preserves_azure_native_fields", - rust=_rust_test(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_response_normalizes_pages"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_map_ocr_params_features", - rust=_rust_family(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_maps_features"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_map_ocr_params_empty_features_list_omitted", - rust=_rust_test(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_url_omits_empty_feature_list"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_map_ocr_params_invalid_features_raises", - rust=_rust_family( - _CORE_TARGET, - _AZURE_OCR_TESTS, - "document_intelligence_mapping_rejects_invalid_features", - ), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_get_complete_url_appends_features_query", - rust=_rust_test(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_url_normalizes_features"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_get_complete_url_combines_pages_and_features", - rust=_rust_test( - _CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_url_combines_pages_and_feature_list" - ), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestGetSupportedOcrParams::test_extract_header_in_supported_params", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "extract_header_is_a_supported_ocr_param"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestGetSupportedOcrParams::test_extract_footer_in_supported_params", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "extract_footer_is_a_supported_ocr_param"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestGetSupportedOcrParams::test_existing_params_still_present", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "existing_ocr_params_remain_supported"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_extract_header_passed_through", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_extract_header"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_extract_footer_passed_through", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_extract_footer"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_extract_header_and_footer_together", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_extract_header_and_footer"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_unknown_param_is_dropped", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_drops_unknown_params"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestNewSupportedParams::test_new_param_in_supported_list", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "new_ocr_params_are_supported"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestNewParamsMapOcr::test_new_param_passed_through", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_new_ocr_params"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrRequest::test_param_included_in_request_body", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_request_includes_each_optional_param"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrRequest::test_multiple_new_params_together", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_request_includes_multiple_new_params"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", - rust=_rust_test( - _CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_response_preserves_blocks_and_confidence_scores" - ), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_response_preserves_ocr4_page_fields"), - ), - *_AZURE_PORT_MAPPINGS, - *_REDUCTO_PORT_MAPPINGS, - _REDUCTO_GATEWAY_MAPPING, - *_GATEWAY_PORT_MAPPINGS, - *_FAMILY_PORT_MAPPINGS, - ), - exclusions=_HOST_ONLY_BRIDGE_EXCLUSIONS, - require_complete=True, - ), - unit_parity=UnitParitySpec( - python_selectors=( - "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "tests/test_litellm/llms/mistral/ocr", - "tests/test_litellm/llms/ocr", - "tests/test_litellm/ocr", - ), - exclusions=( - UnitParityExclusionSpec( - nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_rust_toggles_flag", - reason="This test asserts the process-level backend flag selected by the parity runner.", - ), - ), - ), - rust=RustUnitSpec( - cargo_manifest="litellm-rust/Cargo.toml", - cargo_filter="ocr", - ), -) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py b/tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py deleted file mode 100644 index a8f309cc8f3..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import annotations - -from collections import Counter -from typing import Final, Literal - -from pydantic import BaseModel, ConfigDict, field_validator, model_validator -from typing_extensions import Self - -from ...shared.tracing.pytest_usage import PythonFunctionReference -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope - - -class _ContractModel(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - -def _clean_unique(values: tuple[str, ...], field: str) -> tuple[str, ...]: - cleaned: Final = tuple(value.strip().rstrip("/") for value in values) - if not cleaned or any(not value for value in cleaned): - raise ValueError(f"{field} must contain non-empty paths") - duplicates: Final = tuple(value for value, count in Counter(cleaned).items() if count > 1) - if duplicates: - raise ValueError(f"{field} contains duplicates: {sorted(duplicates)}") - return cleaned - - -def _selector_contains(parent: str, child: str) -> bool: - return child == parent or child.startswith(f"{parent}/") - - -class RustTestFamily(_ContractModel): - kind: Literal["family"] = "family" - target: RustTarget - name: str - - @field_validator("name") - @classmethod - def validate_name(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped or stripped.endswith("::"): - raise ValueError("must be a non-empty Rust test base name") - return stripped - - @property - def key(self) -> str: - return f"{self.target.key}::{self.name}::case_*" - - def contains(self, identity: RustTestIdentity) -> bool: - return identity.target == self.target and identity.name.startswith(f"{self.name}::case_") - - -class TestMapping(_ContractModel): - python: str - rust: RustTestIdentity | RustTestFamily - - @field_validator("python") - @classmethod - def validate_python_nodeid(cls, value: str) -> str: - stripped: Final = value.strip() - if "::" not in stripped: - raise ValueError("must be a source path and test name separated by '::'") - return stripped - - -class PythonFunctionDiscoverySpec(_ContractModel): - functions: tuple[PythonFunctionReference, ...] = () - trace_module: str | None = None - trace_spans: tuple[str, ...] = () - search_roots: tuple[str, ...] - exclude_roots: tuple[str, ...] = () - includes: tuple[str, ...] = () - exclusions: tuple[str, ...] = () - - @field_validator("search_roots") - @classmethod - def validate_search_roots(cls, value: tuple[str, ...]) -> tuple[str, ...]: - return _clean_unique(value, "python function search_roots") - - @field_validator("exclude_roots") - @classmethod - def validate_exclude_roots(cls, value: tuple[str, ...]) -> tuple[str, ...]: - if not value: - return () - return _clean_unique(value, "python function exclude_roots") - - @model_validator(mode="after") - def validate_functions(self) -> Self: - if bool(self.functions) == bool(self.trace_module): - raise ValueError("python function discovery needs exactly one function list or trace module") - if self.trace_module is not None and not self.trace_spans: - raise ValueError("trace-derived Python function discovery needs trace_spans") - if not self.functions: - return self - keys: Final = tuple(f"{function.module}:{function.qualname}" for function in self.functions) - duplicates: Final = tuple(key for key, count in Counter(keys).items() if count > 1) - if duplicates: - raise ValueError(f"python function discovery contains duplicates: {sorted(duplicates)}") - return self - - -class UnitParityExclusionSpec(_ContractModel): - nodeid: str - reason: str - - @field_validator("nodeid", "reason") - @classmethod - def validate_fields(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string") - return stripped - - -class MappingExclusionSpec(_ContractModel): - nodeid: str - reason: str - - @field_validator("nodeid", "reason") - @classmethod - def validate_fields(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string") - return stripped - - -class MappingSpec(_ContractModel): - python_selectors: tuple[str, ...] = () - python_functions: PythonFunctionDiscoverySpec | None = None - rust_scope: tuple[RustTestScope, ...] = () - rust_targets: tuple[RustTarget, ...] = () - mappings: tuple[TestMapping, ...] - exclusions: tuple[MappingExclusionSpec, ...] = () - require_complete: bool = False - - @field_validator("python_selectors") - @classmethod - def validate_python_selectors(cls, value: tuple[str, ...]) -> tuple[str, ...]: - if not value: - return () - return _clean_unique(value, "python_selectors") - - @model_validator(mode="after") - def validate_rust_scope(self) -> Self: - if bool(self.python_selectors) == bool(self.python_functions): - raise ValueError("mapping needs exactly one Python selector or function-discovery scope") - targets: Final = tuple(scope.target.key for scope in self.rust_scope) - duplicates: Final = tuple(target for target, count in Counter(targets).items() if count > 1) - if duplicates: - raise ValueError(f"rust_scope contains duplicate targets: {sorted(duplicates)}") - target_names: Final = tuple(target.name for target in self.rust_targets) - duplicate_names: Final = tuple(name for name, count in Counter(target_names).items() if count > 1) - if duplicate_names: - raise ValueError(f"rust_targets contains duplicate names: {sorted(duplicate_names)}") - exclusion_nodeids: Final = tuple(exclusion.nodeid for exclusion in self.exclusions) - duplicate_exclusions: Final = tuple(nodeid for nodeid, count in Counter(exclusion_nodeids).items() if count > 1) - if duplicate_exclusions: - raise ValueError(f"mapping exclusions contain duplicate nodeids: {sorted(duplicate_exclusions)}") - return self - - -class UnitParitySpec(_ContractModel): - python_selectors: tuple[str, ...] - exclusions: tuple[UnitParityExclusionSpec, ...] = () - - @field_validator("python_selectors") - @classmethod - def validate_python_selectors(cls, value: tuple[str, ...]) -> tuple[str, ...]: - return _clean_unique(value, "unit parity python_selectors") - - @model_validator(mode="after") - def validate_exclusions(self) -> Self: - nodeids: Final = tuple(exclusion.nodeid for exclusion in self.exclusions) - duplicates: Final = tuple(nodeid for nodeid, count in Counter(nodeids).items() if count > 1) - if duplicates: - raise ValueError(f"unit parity exclusions contain duplicate nodeids: {sorted(duplicates)}") - return self - - -class RustUnitSpec(_ContractModel): - cargo_manifest: str - cargo_filter: str - cargo_package: str | None = None - - @field_validator("cargo_manifest", "cargo_filter") - @classmethod - def validate_required_fields(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string") - return stripped - - @field_validator("cargo_package") - @classmethod - def validate_package(cls, value: str | None) -> str | None: - if value is None: - return None - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string when provided") - return stripped - - -class UnitTestContract(_ContractModel): - mapping: MappingSpec - unit_parity: UnitParitySpec - rust: RustUnitSpec - - @model_validator(mode="after") - def validate_unit_parity_scope(self) -> Self: - if not self.mapping.python_selectors: - return self - unknown: Final = tuple( - selector - for selector in self.unit_parity.python_selectors - if not any(_selector_contains(parent, selector) for parent in self.mapping.python_selectors) - ) - if unknown: - raise ValueError(f"unit parity selectors must be contained in mapping selectors: {sorted(unknown)}") - return self diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py b/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py deleted file mode 100644 index a5fd92e449d..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -from collections import Counter -from collections.abc import Callable, Sequence -from typing import Final - -from pydantic import BaseModel, ConfigDict - -from .mapping_validator import MappingReport - - -class MappingReportArtifact(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - report: MappingReport - detailed: bool = False - - -def _group_counts(nodeids: Sequence[str], owner: Callable[[str], str]) -> tuple[str, ...]: - counts: Final = Counter(owner(nodeid) for nodeid in nodeids) - width: Final = max((len(str(count)) for count in counts.values()), default=1) - return tuple( - f" {count:>{width}} {name}" for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0])) - ) - - -def _python_file(nodeid: str) -> str: - return nodeid.partition("::")[0] - - -def _rust_module(nodeid: str) -> str: - return nodeid.rpartition("::")[0] - - -def _details(nodeids: Sequence[str], owner: Callable[[str], str]) -> tuple[str, ...]: - owners: Final = tuple(sorted(frozenset(owner(nodeid) for nodeid in nodeids))) - return tuple( - line - for name in owners - for line in ( - f" {name}", - *(f" {nodeid.removeprefix(f'{name}::')}" for nodeid in nodeids if owner(nodeid) == name), - ) - ) - - -def _contract_errors(report: MappingReport) -> tuple[str, ...]: - return ( - *(f" Missing Python test: {nodeid}" for nodeid in report.missing_python_tests), - *(f" Missing Rust test: {nodeid}" for nodeid in report.missing_rust_tests), - *(f" Python test mapped more than once: {nodeid}" for nodeid in report.duplicate_python_mappings), - *(f" Rust test mapped more than once: {nodeid}" for nodeid in report.duplicate_rust_mappings), - *(f" Missing mapping exclusion: {nodeid}" for nodeid in report.invalid_mapping_exclusions), - *(f" Python test is both mapped and excluded: {nodeid}" for nodeid in report.mapped_and_excluded_python_tests), - *(f" Missing unit-parity exclusion: {nodeid}" for nodeid in report.invalid_unit_parity_exclusions), - ) - - -def mapping_report_lines(report: MappingReport, *, detailed: bool = False) -> tuple[str, ...]: - unmapped_count: Final = len(report.unmapped_python_tests) - excluded_count: Final = len(report.excluded_python_tests) - excluded_percentage: Final = ( - 0.0 if not report.total_count else round(100.0 * excluded_count / report.total_count, 1) - ) - unmapped_percentage: Final = ( - 0.0 if not report.total_count else round(100.0 * unmapped_count / report.total_count, 1) - ) - rust_total: Final = len(report.rust_tests) - rust_only_count: Final = len(report.rust_only_tests) - rust_mapped_count: Final = rust_total - rust_only_count - contract_errors: Final = _contract_errors(report) - detail_lines: Final = ( - ( - "", - "Unmapped Python test details", - *_details(report.unmapped_python_tests, _python_file), - "", - "Excluded Python test details", - *_details(report.excluded_python_tests, _python_file), - "", - "Rust-only test details", - *_details(report.rust_only_tests, _rust_module), - ) - if detailed - else () - ) - return ( - f"Contract: {'PASS' if report.is_valid else 'FAIL'}", - "", - "Python coverage", - f" Mapped {report.mapped_count:>3} / {report.total_count} ({report.percentage}%)", - f" Excluded {excluded_count:>3} / {report.total_count} ({excluded_percentage}%)", - f" Unmapped {unmapped_count:>3} / {report.total_count} ({unmapped_percentage}%)", - "", - "Rust inventory", - f" Mapped {rust_mapped_count:>3} / {rust_total}", - f" Rust-only {rust_only_count:>3} / {rust_total}", - "", - f"Unmapped Python tests by file ({unmapped_count})", - *_group_counts(report.unmapped_python_tests, _python_file), - "", - f"Excluded Python tests by file ({excluded_count})", - *_group_counts(report.excluded_python_tests, _python_file), - "", - f"Rust-only tests by module ({rust_only_count})", - *_group_counts(report.rust_only_tests, _rust_module), - *(("", "Contract errors", *contract_errors) if contract_errors else ()), - *detail_lines, - ) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py deleted file mode 100644 index 9dd79e860e6..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py +++ /dev/null @@ -1,296 +0,0 @@ -from __future__ import annotations - -import importlib -from collections import Counter, defaultdict -from collections.abc import Callable, Sequence -from pathlib import Path -from typing import Final, TypeAlias - -from pydantic import BaseModel, ConfigDict - -from ...shared.tracing.pytest_usage import ( - PythonFunctionIdentity, - RustFunctionIdentity, - candidate_test_files, - collect_python_function_tests, -) -from ...shared.tracing.steps import pipeline_projection -from ...shared.unit_runners.python_runner import collect_python_tests, contract_nodeid -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope, enumerate_rust_tests -from .contracts import PythonFunctionDiscoverySpec, RustTestFamily, TestMapping, UnitTestContract - -PythonInventory: TypeAlias = Callable[[Sequence[str], Path], frozenset[str]] -RustInventory: TypeAlias = Callable[[Path, tuple[RustTestScope, ...]], frozenset[RustTestIdentity]] - - -def _trace_functions( - spec: PythonFunctionDiscoverySpec, -) -> tuple[tuple[PythonFunctionIdentity, ...], tuple[RustFunctionIdentity, ...]]: - from ..trace_parity.models import RouteSpec, TraceExecutionFailure, TraceSuite - from ..trace_parity.sdk.execution import collect_trace - - if spec.trace_module is None: - return () - module: Final = importlib.import_module(spec.trace_module) - suite: Final = getattr(module, "TRACE_SUITE", None) - if not isinstance(suite, TraceSuite) or not isinstance(suite.route, RouteSpec): - raise ValueError(f"{spec.trace_module} must export an SDK TRACE_SUITE") - python_functions: Final[dict[str, PythonFunctionIdentity]] = {} - rust_functions: Final[dict[str, RustFunctionIdentity]] = {} - for scenario in suite.scenarios: - route: Final = RouteSpec( - route=suite.route.route, - python_entrypoints=suite.route.python_entrypoints, - rust_entrypoints=suite.route.rust_entrypoints, - fixture=scenario.fixture, - ) - python_trace: Final = collect_trace(route, "python", asynchronous=scenario.asynchronous) - rust_trace: Final = collect_trace(route, "rust", asynchronous=scenario.asynchronous) - if isinstance(python_trace, TraceExecutionFailure): - raise ValueError(f"Python trace discovery failed for {scenario.name}: {python_trace.message}") - if isinstance(rust_trace, TraceExecutionFailure): - raise ValueError(f"Rust trace discovery failed for {scenario.name}: {rust_trace.message}") - python_projection: Final = pipeline_projection("python", python_trace, scenario.mappings) - rust_projection: Final = pipeline_projection("rust", rust_trace, scenario.mappings) - for step in python_projection.steps: - if step.span in spec.trace_spans: - function: Final = PythonFunctionIdentity.from_trace(step.raw) - python_functions[function.raw] = function - for step in rust_projection.steps: - if step.span in spec.trace_spans: - function: Final = RustFunctionIdentity.from_trace(step.raw) - rust_functions[step.raw] = function - if not python_functions or not rust_functions: - raise ValueError(f"Python trace discovery found no functions for spans: {', '.join(spec.trace_spans)}") - return ( - tuple(python_functions[key] for key in sorted(python_functions)), - tuple(rust_functions[key] for key in sorted(rust_functions)), - ) - - -def collect_python_function_inventory( - spec: PythonFunctionDiscoverySpec, - repo_root: Path, - traced_functions: Sequence[PythonFunctionIdentity] = (), -) -> frozenset[str]: - source_root: Final = repo_root / "litellm" - functions: Final = ( - tuple(reference.resolve(source_root) for reference in spec.functions) - if spec.functions - else tuple(traced_functions) - ) - discovered: Final = candidate_test_files( - functions, - spec.search_roots, - repo_root, - exclude_roots=spec.exclude_roots, - ) - selectors: Final = tuple(dict.fromkeys((*discovered, *spec.includes))) - if not selectors: - raise ValueError("Python function discovery found no candidate test files") - report: Final = collect_python_function_tests( - functions, - selectors, - repo_root, - source_root=source_root, - exclusions=spec.exclusions, - ) - if report.exit_code or report.problems: - details: Final = "\n".join(report.problems) or f"pytest exited with code {report.exit_code}" - raise ValueError(f"Python function test discovery failed:\n{details}") - return frozenset(contract_nodeid(nodeid) for usage in report.usages for nodeid in usage.tests) - - -def _colocated_rust_scope(mappings: Sequence[TestMapping]) -> tuple[RustTestScope, ...]: - modules_by_target: Final[dict[str, set[str]]] = defaultdict(set) - targets: Final[dict[str, RustTarget]] = {} - for item in mappings: - module, separator, _ = item.rust.name.partition("::tests::") - if not separator: - raise ValueError(f"Rust unit test is not colocated in a tests module: {item.rust.key}") - target_key: Final = item.rust.target.key - targets[target_key] = item.rust.target - modules_by_target[target_key].add(f"{module}::tests") - return tuple( - RustTestScope( - target=targets[target_key], - modules=tuple(sorted(modules_by_target[target_key])), - ) - for target_key in sorted(targets) - ) - - -def _traced_rust_scope( - functions: Sequence[RustFunctionIdentity], - targets: Sequence[RustTarget], - repo_root: Path, -) -> tuple[RustTestScope, ...]: - targets_by_name: Final = {target.name: target for target in targets} - modules_by_target: Final[dict[str, set[str]]] = defaultdict(set) - for function in functions: - crate: Final = function.module_path.partition("::")[0] - target: Final = targets_by_name.get(crate) - if target is None: - continue - source_candidates: Final = ( - repo_root / "litellm-rust" / function.file, - repo_root / function.file, - ) - source: Final = next((path for path in source_candidates if path.is_file()), None) - if source is None: - raise ValueError(f"Traced Rust source does not exist: {function.file}") - contents: Final = source.read_text() - if "mod tests" in contents and "#[cfg(test)]" in contents: - modules_by_target[target.key].add(function.test_module) - selected_targets: Final = {target.key: target for target in targets} - scopes: Final = tuple( - RustTestScope(target=selected_targets[key], modules=tuple(sorted(modules))) - for key, modules in sorted(modules_by_target.items()) - if modules - ) - if not scopes: - raise ValueError("Traced Rust functions have no colocated test modules") - return scopes - - -def _merge_rust_scopes(scopes: Sequence[RustTestScope]) -> tuple[RustTestScope, ...]: - targets: Final = {scope.target.key: scope.target for scope in scopes} - modules: Final[dict[str, set[str]]] = defaultdict(set) - features: Final[dict[str, set[str]]] = defaultdict(set) - default_features: Final[dict[str, bool]] = {} - for scope in scopes: - modules[scope.target.key].update(scope.modules) - features[scope.target.key].update(scope.features) - default_features[scope.target.key] = default_features.get(scope.target.key, True) and scope.default_features - return tuple( - RustTestScope( - target=targets[key], - modules=tuple( - sorted( - module - for module in modules[key] - if not any(module.startswith(f"{parent}::") for parent in modules[key]) - ) - ), - features=tuple(sorted(features[key])), - default_features=default_features[key], - ) - for key in sorted(targets) - ) - - -def _owned_rust_tests( - rust: RustTestIdentity | RustTestFamily, - inventory: frozenset[RustTestIdentity], -) -> frozenset[RustTestIdentity]: - if isinstance(rust, RustTestFamily): - return frozenset(identity for identity in inventory if rust.contains(identity)) - return frozenset((rust,)) if rust in inventory else frozenset() - - -class MappingReport(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - python_tests: tuple[str, ...] - rust_tests: tuple[str, ...] - mapped_python_tests: tuple[str, ...] - excluded_python_tests: tuple[str, ...] - unmapped_python_tests: tuple[str, ...] - rust_only_tests: tuple[str, ...] - missing_python_tests: tuple[str, ...] - missing_rust_tests: tuple[str, ...] - duplicate_python_mappings: tuple[str, ...] - duplicate_rust_mappings: tuple[str, ...] - invalid_mapping_exclusions: tuple[str, ...] - mapped_and_excluded_python_tests: tuple[str, ...] - invalid_unit_parity_exclusions: tuple[str, ...] - - @property - def mapped_count(self) -> int: - return len(self.mapped_python_tests) - - @property - def total_count(self) -> int: - return len(self.python_tests) - - @property - def percentage(self) -> float: - return 0.0 if not self.total_count else round(100.0 * self.mapped_count / self.total_count, 1) - - @property - def is_valid(self) -> bool: - return not ( - self.missing_python_tests - or self.missing_rust_tests - or self.duplicate_python_mappings - or self.duplicate_rust_mappings - or self.invalid_mapping_exclusions - or self.mapped_and_excluded_python_tests - or self.invalid_unit_parity_exclusions - ) - - -def audit_mapping( - contract: UnitTestContract, - repo_root: Path, - *, - python_inventory: PythonInventory = collect_python_tests, - rust_inventory: RustInventory = enumerate_rust_tests, -) -> MappingReport: - mapping: Final = contract.mapping - traced_python: tuple[PythonFunctionIdentity, ...] = () - traced_rust: tuple[RustFunctionIdentity, ...] = () - if mapping.python_functions is not None and mapping.python_functions.trace_module is not None: - traced_python, traced_rust = _trace_functions(mapping.python_functions) - python_tests: Final = ( - collect_python_function_inventory(mapping.python_functions, repo_root, traced_python) - if mapping.python_functions is not None - else python_inventory(mapping.python_selectors, repo_root) - ) - unit_parity_tests: Final = python_inventory(contract.unit_parity.python_selectors, repo_root) - traced_scope: Final = _traced_rust_scope(traced_rust, mapping.rust_targets, repo_root) if traced_rust else () - rust_scope: Final = _merge_rust_scopes( - (*mapping.rust_scope, *traced_scope, *_colocated_rust_scope(mapping.mappings)) - ) - rust_tests: Final = rust_inventory(repo_root, rust_scope) - mapped_python: Final = frozenset(item.python for item in mapping.mappings) - excluded_python: Final = frozenset(exclusion.nodeid for exclusion in mapping.exclusions) - rust_ownership: Final = tuple((item.rust, _owned_rust_tests(item.rust, rust_tests)) for item in mapping.mappings) - mapped_rust: Final = frozenset(identity for _, identities in rust_ownership for identity in identities) - duplicate_python: Final = tuple( - sorted(nodeid for nodeid, count in Counter(item.python for item in mapping.mappings).items() if count > 1) - ) - duplicate_exact_rust: Final = frozenset( - identity.key - for identity, count in Counter( - item.rust for item in mapping.mappings if isinstance(item.rust, RustTestIdentity) - ).items() - if count > 1 - ) - duplicate_owned_rust: Final = frozenset( - identity.key - for identity, count in Counter(identity for _, identities in rust_ownership for identity in identities).items() - if count > 1 - ) - duplicate_rust: Final = tuple(sorted(duplicate_exact_rust | duplicate_owned_rust)) - return MappingReport( - python_tests=tuple(sorted(python_tests)), - rust_tests=tuple(sorted(identity.key for identity in rust_tests)), - mapped_python_tests=tuple(sorted(python_tests & mapped_python)), - excluded_python_tests=tuple(sorted((python_tests & excluded_python) - mapped_python)), - unmapped_python_tests=tuple(sorted(python_tests - mapped_python - excluded_python)), - rust_only_tests=tuple(sorted(identity.key for identity in rust_tests - mapped_rust)), - missing_python_tests=tuple(sorted(mapped_python - python_tests)), - missing_rust_tests=tuple(sorted(rust.key for rust, identities in rust_ownership if not identities)), - duplicate_python_mappings=duplicate_python, - duplicate_rust_mappings=duplicate_rust, - invalid_mapping_exclusions=tuple(sorted(excluded_python - python_tests)), - mapped_and_excluded_python_tests=tuple(sorted(mapped_python & excluded_python)), - invalid_unit_parity_exclusions=tuple( - sorted( - exclusion.nodeid - for exclusion in contract.unit_parity.exclusions - if exclusion.nodeid not in unit_parity_tests - ) - ), - ) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py b/tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py deleted file mode 100644 index efb5b2a644a..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final - -from ...shared.reporting.models import SdkFunction -from .cases.ocr import OCR_CONTRACT -from .contracts import UnitTestContract - -UNIT_TEST_CONTRACTS: Final[Mapping[SdkFunction, UnitTestContract]] = MappingProxyType({"ocr": OCR_CONTRACT}) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py b/tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py deleted file mode 100644 index d4bce7bc768..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Final - -from pydantic import ValidationError - -from ...shared.reporting.models import CaseResult -from ...shared.reporting.rendering import ReportSection, render_case_outcome -from .mapping_report import MappingReportArtifact, mapping_report_lines -from .runner import MAPPING_REPORT_ARTIFACT - - -def _render_artifact(body: str) -> str: - try: - artifact: Final = MappingReportArtifact.model_validate_json(body) - except ValidationError as error: - return f"Mapping report artifact is invalid: {error}" - return "\n".join(mapping_report_lines(artifact.report, detailed=artifact.detailed)) - - -def _render_result(result: CaseResult) -> str: - reports: Final = tuple( - _render_artifact(artifact.body) - for artifacts in result.artifacts.values() - for artifact in artifacts - if artifact.kind == MAPPING_REPORT_ARTIFACT - ) - if reports: - return "\n".join((f"Case: {result.case.display_name}", *reports)) - return render_case_outcome(result) - - -def render_mapping_results(results: Sequence[CaseResult]) -> tuple[ReportSection, ...]: - blocks: Final = tuple(_render_result(result) for result in results) - return (ReportSection("Python/Rust unit-test mappings", blocks or ("No mapping cases selected",)),) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/runner.py b/tests/rust-python-harness/strategies/unit_tests_mapping/runner.py deleted file mode 100644 index 540edca9385..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/runner.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from pathlib import Path -from typing import Final - -from ...shared.native_build import ensure_trace_bridge -from ...shared.reporting.models import ResultArtifact -from ...shared.unit_runners.python_runner import collect_python_tests -from ...shared.unit_runners.rust_runner import enumerate_rust_tests -from ...shared.unit_runners.suite_runner import SuiteExecution -from .contracts import UnitTestContract -from .mapping_report import MappingReportArtifact -from .mapping_validator import PythonInventory, RustInventory, audit_mapping - -MAPPING_REPORT_ARTIFACT: Final = "mapping_report" - - -def _audit_problems(artifact: MappingReportArtifact) -> tuple[str, ...]: - report: Final = artifact.report - return ( - *(f"mapped Python test does not exist: {nodeid}" for nodeid in report.missing_python_tests), - *(f"mapped Rust test does not exist: {nodeid}" for nodeid in report.missing_rust_tests), - *(f"Python test has multiple mappings: {nodeid}" for nodeid in report.duplicate_python_mappings), - *(f"Rust test has multiple mappings: {nodeid}" for nodeid in report.duplicate_rust_mappings), - *(f"mapping exclusion does not exist: {nodeid}" for nodeid in report.invalid_mapping_exclusions), - *(f"Python test is both mapped and excluded: {nodeid}" for nodeid in report.mapped_and_excluded_python_tests), - *(f"unit parity exclusion does not exist: {nodeid}" for nodeid in report.invalid_unit_parity_exclusions), - ) - - -def run_suite( - contract: UnitTestContract, - repo_root: Path, - runner_args: Sequence[str] = (), - *, - python_inventory: PythonInventory = collect_python_tests, - rust_inventory: RustInventory = enumerate_rust_tests, -) -> SuiteExecution: - if contract.mapping.python_functions is not None and contract.mapping.python_functions.trace_module is not None: - bridge_error: Final = ensure_trace_bridge(repo_root) - if bridge_error is not None: - return SuiteExecution(problems=(bridge_error,)) - artifact: Final = MappingReportArtifact( - report=audit_mapping( - contract, - repo_root, - python_inventory=python_inventory, - rust_inventory=rust_inventory, - ), - detailed=bool(runner_args), - ) - completeness_problems: Final = ( - tuple(f"Python test has no Rust mapping: {nodeid}" for nodeid in artifact.report.unmapped_python_tests) - if contract.mapping.require_complete - else () - ) - return SuiteExecution( - problems=(*_audit_problems(artifact), *completeness_problems), - artifacts=(ResultArtifact(MAPPING_REPORT_ARTIFACT, artifact.model_dump_json()),), - ) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py deleted file mode 100644 index 6635a0eb522..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py +++ /dev/null @@ -1,314 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Final - -import pytest -from pydantic import ValidationError - -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope -from .contracts import ( - MappingExclusionSpec, - MappingSpec, - RustTestFamily, - RustUnitSpec, - UnitParityExclusionSpec, - UnitParitySpec, - UnitTestContract, -) -from .contracts import TestMapping as MappingPair -from .mapping_validator import audit_mapping - -_TARGET: Final = RustTarget(package="example", name="example", kind="lib") -_SCOPE: Final = RustTestScope(target=_TARGET, modules=("api::tests",)) -_PYTHON_TESTS: Final = frozenset(("test_api.py::test_decode", "test_api.py::test_unmapped")) -_RUST_TEST: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes") -_RUST_ONLY: Final = RustTestIdentity(target=_TARGET, name="api::tests::rust_only") -_RUST_TESTS: Final = frozenset((_RUST_TEST, _RUST_ONLY)) - - -def _python_inventory(*_: object) -> frozenset[str]: - return _PYTHON_TESTS - - -def _rust_inventory(*_: object) -> frozenset[RustTestIdentity]: - return _RUST_TESTS - - -def _contract(*mappings: MappingPair, exclusions: tuple[UnitParityExclusionSpec, ...] = ()) -> UnitTestContract: - return UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(_SCOPE,), - mappings=mappings, - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",), exclusions=exclusions), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - -def _mapping_exclusion(nodeid: str) -> MappingExclusionSpec: - return MappingExclusionSpec(nodeid=nodeid, reason="Python bridge availability is host-only") - - -def test_derives_mapping_status_from_live_inventories(tmp_path: Path) -> None: - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert report.is_valid - assert report.mapped_python_tests == ("test_api.py::test_decode",) - assert report.unmapped_python_tests == ("test_api.py::test_unmapped",) - assert report.rust_only_tests == (_RUST_ONLY.key,) - assert report.percentage == 50.0 - - -def test_reports_stale_and_duplicate_mappings(tmp_path: Path) -> None: - removed: Final = RustTestIdentity(target=_TARGET, name="api::tests::removed") - contract: Final = _contract( - MappingPair(python="test_api.py::removed", rust=removed), - MappingPair(python="test_api.py::removed", rust=_RUST_TEST), - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.missing_python_tests == ("test_api.py::removed",) - assert report.missing_rust_tests == (removed.key,) - assert report.duplicate_python_mappings == ("test_api.py::removed",) - - -def test_reports_duplicate_rust_mapping_and_invalid_exclusion(tmp_path: Path) -> None: - contract: Final = _contract( - MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST), - MappingPair(python="test_api.py::test_unmapped", rust=_RUST_TEST), - exclusions=(UnitParityExclusionSpec(nodeid="test_api.py::removed", reason="Removed test"),), - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.duplicate_rust_mappings == (_RUST_TEST.key,) - assert report.invalid_unit_parity_exclusions == ("test_api.py::removed",) - - -def test_excludes_host_only_python_test_from_unmapped_inventory(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={ - "mapping": partial.mapping.model_copy( - update={"exclusions": (_mapping_exclusion("test_api.py::test_unmapped"),)} - ) - } - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert report.is_valid - assert report.excluded_python_tests == ("test_api.py::test_unmapped",) - assert report.unmapped_python_tests == () - - -def test_reports_missing_and_mapped_mapping_exclusions(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={ - "mapping": partial.mapping.model_copy( - update={ - "exclusions": ( - _mapping_exclusion("test_api.py::test_decode"), - _mapping_exclusion("test_api.py::removed"), - ) - } - ) - } - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.invalid_mapping_exclusions == ("test_api.py::removed",) - assert report.mapped_and_excluded_python_tests == ("test_api.py::test_decode",) - - -def test_resolves_rstest_family_to_generated_cases(tmp_path: Path) -> None: - first_case: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_1_png") - second_case: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_2_pdf") - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=family)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=lambda *_: frozenset((first_case, second_case)), - ) - - assert report.is_valid - assert report.mapped_python_tests == ("test_api.py::test_decode",) - assert report.missing_rust_tests == () - - -def test_reports_missing_rstest_family(tmp_path: Path) -> None: - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=family)) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.missing_rust_tests == (family.key,) - - -def test_reports_concrete_test_owned_by_exact_and_family_mappings(tmp_path: Path) -> None: - generated: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_1_png") - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract( - MappingPair(python="test_api.py::test_decode", rust=family), - MappingPair(python="test_api.py::test_unmapped", rust=generated), - ) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=lambda *_: frozenset((generated,)), - ) - - assert not report.is_valid - assert report.duplicate_rust_mappings == (generated.key,) - - -def test_rstest_family_cases_are_not_rust_only(tmp_path: Path) -> None: - generated: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_1_png") - unrelated: Final = RustTestIdentity(target=_TARGET, name="api::tests::rust_only") - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=family)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=lambda *_: frozenset((generated, unrelated)), - ) - - assert report.rust_only_tests == (unrelated.key,) - - -def test_merges_configured_and_colocated_rust_scopes(tmp_path: Path) -> None: - support_test: Final = RustTestIdentity(target=_TARGET, name="support::tests::rust_only") - configured_scope: Final = RustTestScope( - target=_TARGET, - modules=("support::tests",), - features=("mock",), - default_features=False, - ) - expected_scope: Final = RustTestScope( - target=_TARGET, - modules=("api::tests", "support::tests"), - features=("mock",), - default_features=False, - ) - contract: Final = UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(configured_scope,), - mappings=(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST),), - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - def assert_merged_scope(_: Path, scopes: tuple[RustTestScope, ...]) -> frozenset[RustTestIdentity]: - assert scopes == (expected_scope,) - return frozenset((_RUST_TEST, support_test)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=assert_merged_scope, - ) - - assert report.is_valid - assert report.rust_only_tests == (support_test.key,) - - -def test_merged_rust_scope_removes_modules_contained_by_parent(tmp_path: Path) -> None: - expected_scope: Final = RustTestScope(target=_TARGET, modules=("api",)) - contract: Final = UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(expected_scope,), - mappings=(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST),), - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - def assert_parent_scope(_: Path, scopes: tuple[RustTestScope, ...]) -> frozenset[RustTestIdentity]: - assert scopes == (expected_scope,) - return frozenset((_RUST_TEST,)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=assert_parent_scope, - ) - - assert report.is_valid - - -def test_accepts_descendant_unit_parity_selector() -> None: - contract: Final = UnitTestContract( - mapping=MappingSpec(python_selectors=("tests/api",), rust_scope=(_SCOPE,), mappings=()), - unit_parity=UnitParitySpec(python_selectors=("tests/api/test_ocr.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - assert contract.unit_parity.python_selectors == ("tests/api/test_ocr.py",) - - -@pytest.mark.parametrize( - "mapping_selectors,parity_selectors", - (((), ("tests/api",)), (("tests/api", "tests/api"), ("tests/api",)), (("tests/api",), ("tests/chat",))), -) -def test_rejects_invalid_selector_contracts( - mapping_selectors: tuple[str, ...], parity_selectors: tuple[str, ...] -) -> None: - with pytest.raises(ValidationError): - UnitTestContract( - mapping=MappingSpec(python_selectors=mapping_selectors, rust_scope=(_SCOPE,), mappings=()), - unit_parity=UnitParitySpec(python_selectors=parity_selectors), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - -def test_rejects_duplicate_scopes_and_exclusions() -> None: - exclusion: Final = UnitParityExclusionSpec(nodeid="test_api.py::test_skip", reason="Backend assertion") - with pytest.raises(ValidationError, match="duplicate targets"): - MappingSpec(python_selectors=("test_api.py",), rust_scope=(_SCOPE, _SCOPE), mappings=()) - with pytest.raises(ValidationError, match="duplicate nodeids"): - UnitParitySpec(python_selectors=("test_api.py",), exclusions=(exclusion, exclusion)) - mapping_exclusion: Final = _mapping_exclusion("test_api.py::test_skip") - with pytest.raises(ValidationError, match="mapping exclusions contain duplicate nodeids"): - MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(_SCOPE,), - mappings=(), - exclusions=(mapping_exclusion, mapping_exclusion), - ) - with pytest.raises(ValidationError, match="must be a non-empty string"): - MappingExclusionSpec(nodeid="test_api.py::test_skip", reason=" ") diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py b/tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py deleted file mode 100644 index 36e18a9d109..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py +++ /dev/null @@ -1,99 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from ...shared.reporting.models import CaseResult, Coverage, HarnessCase, ResultArtifact, RunStatus -from ...shared.reporting.strategy import SuiteCaseSpec -from .mapping_report import MappingReportArtifact -from .mapping_validator import MappingReport -from .reporting import render_mapping_results -from .runner import MAPPING_REPORT_ARTIFACT - - -def _report(*, invalid: bool = False, excluded: bool = False) -> MappingReport: - return MappingReport( - python_tests=("test_api.py::test_decode", "test_api.py::test_unmapped"), - rust_tests=("example/lib/example::api::tests::decodes", "example/lib/example::api::tests::rust_only"), - mapped_python_tests=("test_api.py::test_decode",), - excluded_python_tests=(("test_api.py::test_unmapped",) if excluded else ()), - unmapped_python_tests=(() if excluded else ("test_api.py::test_unmapped",)), - rust_only_tests=("example/lib/example::api::tests::rust_only",), - missing_python_tests=("test_api.py::removed",) if invalid else (), - missing_rust_tests=(), - duplicate_python_mappings=(), - duplicate_rust_mappings=(), - invalid_mapping_exclusions=(), - mapped_and_excluded_python_tests=(), - invalid_unit_parity_exclusions=(), - ) - - -def _result(body: str) -> CaseResult: - case: Final = HarnessCase( - strategy_id="unit_tests_mapping", - strategy_label="Unit test mapping", - sdk_function="ocr", - spec=SuiteCaseSpec(coverage=Coverage.COMPLETE, suite="ocr"), - ) - result: Final = CaseResult(case=case) - result.record( - "suite:unit_tests_mapping:ocr:ocr", - RunStatus.PASSED, - artifacts=(ResultArtifact(MAPPING_REPORT_ARTIFACT, body),), - ) - return result - - -def test_renderer_preserves_summary_and_detailed_output() -> None: - summary: Final = MappingReportArtifact(report=_report()).model_dump_json() - detailed: Final = MappingReportArtifact(report=_report(), detailed=True).model_dump_json() - - summary_text: Final = "\n".join(render_mapping_results((_result(summary),))[0].blocks) - detailed_text: Final = "\n".join(render_mapping_results((_result(detailed),))[0].blocks) - - assert "Mapped 1 / 2 (50.0%)" in summary_text - assert "Unmapped Python test details" not in summary_text - assert "Unmapped Python test details\n test_api.py\n test_unmapped" in detailed_text - assert "Rust-only test details" in detailed_text - - -def test_renderer_shows_contract_errors() -> None: - body: Final = MappingReportArtifact(report=_report(invalid=True)).model_dump_json() - rendered: Final = "\n".join(render_mapping_results((_result(body),))[0].blocks) - - assert "Contract: FAIL" in rendered - assert "Missing Python test: test_api.py::removed" in rendered - - -def test_renderer_distinguishes_excluded_python_tests() -> None: - body: Final = MappingReportArtifact(report=_report(excluded=True), detailed=True).model_dump_json() - rendered: Final = "\n".join(render_mapping_results((_result(body),))[0].blocks) - - assert "Excluded 1 / 2 (50.0%)" in rendered - assert "Unmapped 0 / 2 (0.0%)" in rendered - assert "Excluded Python test details\n test_api.py\n test_unmapped" in rendered - - -def test_renderer_handles_empty_inventory_and_malformed_artifact() -> None: - empty: Final = MappingReport( - python_tests=(), - rust_tests=(), - mapped_python_tests=(), - excluded_python_tests=(), - unmapped_python_tests=(), - rust_only_tests=(), - missing_python_tests=(), - missing_rust_tests=(), - duplicate_python_mappings=(), - duplicate_rust_mappings=(), - invalid_mapping_exclusions=(), - mapped_and_excluded_python_tests=(), - invalid_unit_parity_exclusions=(), - ) - empty_text: Final = "\n".join( - render_mapping_results((_result(MappingReportArtifact(report=empty).model_dump_json()),))[0].blocks - ) - invalid_text: Final = "\n".join(render_mapping_results((_result("not-json"),))[0].blocks) - - assert "Mapped 0 / 0 (0.0%)" in empty_text - assert "Mapping report artifact is invalid:" in invalid_text diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py b/tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py deleted file mode 100644 index 2b14c716e1d..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -from functools import partial -from pathlib import Path -from typing import Final - -from ...shared.reporting.models import Coverage, HarnessCase, RunStatus -from ...shared.reporting.strategy import SuiteCaseSpec -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope -from ...shared.unit_runners.suite_runner import run_suites -from .contracts import ( - MappingExclusionSpec, - MappingSpec, - RustUnitSpec, - TestMapping as MappingPair, - UnitParitySpec, - UnitTestContract, -) -from .mapping_report import MappingReportArtifact -from .runner import MAPPING_REPORT_ARTIFACT, run_suite - -_TARGET: Final = RustTarget(package="example", name="example", kind="lib") -_RUST_TEST: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes") -_RUST_ONLY: Final = RustTestIdentity(target=_TARGET, name="api::tests::rust_only") - - -def _python_inventory(*_: object) -> frozenset[str]: - return frozenset(("test_api.py::test_decode", "test_api.py::test_unmapped")) - - -def _rust_inventory(*_: object) -> frozenset[RustTestIdentity]: - return frozenset((_RUST_TEST, _RUST_ONLY)) - - -def _contract(mapping: MappingPair) -> UnitTestContract: - return UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(RustTestScope(target=_TARGET, modules=("api::tests",)),), - mappings=(mapping,), - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - -def _case() -> HarnessCase: - return HarnessCase( - strategy_id="unit_tests_mapping", - strategy_label="Unit test mapping", - sdk_function="ocr", - spec=SuiteCaseSpec(coverage=Coverage.COMPLETE, suite="ocr"), - ) - - -def test_reports_structured_mapping_status_without_running_tests(tmp_path: Path) -> None: - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - case: Final = _case() - - code, report = run_suites( - (case,), - tmp_path, - lambda _: None, - suites={"ocr": contract}, - execute=partial( - run_suite, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ), - ) - - result: Final = report.results[case.key] - artifacts: Final = tuple( - artifact - for values in result.artifacts.values() - for artifact in values - if artifact.kind == MAPPING_REPORT_ARTIFACT - ) - parsed: Final = MappingReportArtifact.model_validate_json(artifacts[0].body) - assert code == 0, report.failures - assert result.status is RunStatus.PASSED - assert parsed.report.mapped_count == 1 - assert parsed.report.total_count == 2 - assert not parsed.detailed - - -def test_fails_when_a_mapping_target_is_missing(tmp_path: Path) -> None: - missing: Final = RustTestIdentity(target=_TARGET, name="api::tests::missing") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=missing)) - case: Final = _case() - - code, report = run_suites( - (case,), - tmp_path, - lambda _: None, - suites={"ocr": contract}, - execute=partial( - run_suite, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ), - ) - - assert code == 1 - assert report.results[case.key].status is RunStatus.FAILED - assert any("mapped Rust test does not exist" in detail for _, detail in report.failures) - - -def test_required_complete_mapping_fails_for_unmapped_python_test(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={"mapping": partial.mapping.model_copy(update={"require_complete": True})} - ) - - execution: Final = run_suite( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ) - - assert execution.problems == ("Python test has no Rust mapping: test_api.py::test_unmapped",) - - -def test_required_complete_mapping_accepts_host_only_exclusion(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={ - "mapping": partial.mapping.model_copy( - update={ - "require_complete": True, - "exclusions": ( - MappingExclusionSpec( - nodeid="test_api.py::test_unmapped", - reason="Python bridge availability is host-only", - ), - ), - } - ) - } - ) - - execution: Final = run_suite( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ) - artifact: Final = MappingReportArtifact.model_validate_json(execution.artifacts[0].body) - - assert execution.problems == () - assert artifact.report.excluded_python_tests == ("test_api.py::test_unmapped",) - - -def test_detail_argument_is_stored_in_artifact(tmp_path: Path) -> None: - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - execution: Final = run_suite( - contract, - tmp_path, - ("full",), - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ) - artifact: Final = MappingReportArtifact.model_validate_json(execution.artifacts[0].body) - - assert artifact.detailed diff --git a/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py b/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py index 0067bf6dfe5..fe3bd2e2f94 100644 --- a/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py +++ b/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py @@ -14,8 +14,8 @@ from ...shared.reporting.strategy import ( StrategyDefinition, SuiteCaseSpec, ) +from ...shared.unit_runners.contracts import UNIT_TEST_CONTRACTS from ...shared.unit_runners.suite_runner import run_suites -from ..unit_tests_mapping.mappings import UNIT_TEST_CONTRACTS from .reporting import render_unit_parity_results from .runner import UnitParityExclusion, UnitParitySuite, run_suite diff --git a/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py b/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py index 8114e12ab96..b9ca5b13e63 100644 --- a/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py +++ b/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py @@ -13,8 +13,8 @@ from ...shared.reporting.strategy import ( StrategyDefinition, SuiteCaseSpec, ) +from ...shared.unit_runners.contracts import UNIT_TEST_CONTRACTS from ...shared.unit_runners.suite_runner import run_suites -from ..unit_tests_mapping.mappings import UNIT_TEST_CONTRACTS from .reporting import render_rust_unit_results from .runner import RustSuite, run_suite diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py b/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py index b379b8bebc9..930c01e524e 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_baggage.py @@ -168,6 +168,46 @@ def test_allowlisted_metadata_subkey_promoted_blob_excluded(): assert all("private_note" not in k for k in span.attributes) +def test_nested_metadata_key_promoted_under_caller_path(): + """A dotted allowlist entry reads the nested caller metadata the proxy stores + under ``requester_metadata`` and lands on the LLM-call span under the caller's + own path (``litellm.metadata.trace_id``, ``litellm.metadata.nested.deep``); + a pre-existing flat dotted key keeps its full name, and unlisted siblings and + the blob stay out.""" + engine, exporter = _engine_and_exporter() + payload = _payload() + payload["metadata"]["a.b"] = "flat" + payload["metadata"]["requester_metadata"] = { + "trace_id": "abc", + "attempt": 0, + "empty": "", + "nested": {"deep": "x", "skipped": "y"}, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + bag = promoted_baggage( + data.identity, + data.request_model, + BAGGAGE_PROMOTED_KEYS, + metadata_keys=( + "requester_metadata.trace_id", + "requester_metadata.attempt", + "requester_metadata.empty", + "requester_metadata.nested.deep", + "a.b", + ), + ) + engine.emit(SpanRole.LLM_CALL, data, ctx_mod.set_request_baggage(bag)) + (span,) = exporter.get_finished_spans() + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}trace_id"] == "abc" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}attempt"] == "0" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}nested.deep"] == "x" + assert span.attributes[f"{LiteLLM.METADATA_PREFIX}a.b"] == "flat" + assert f"{LiteLLM.METADATA_PREFIX}empty" not in span.attributes + assert f"{LiteLLM.METADATA_PREFIX}deep" not in span.attributes + assert f"{LiteLLM.METADATA_PREFIX}nested.skipped" not in span.attributes + assert not any(k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") for k in span.attributes) + + def test_http_attributes_never_promoted(): """Even if http.* is present in baggage, the processor must not stamp it on child spans (it belongs on the SERVER span only).""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 2869c804c07..9b5abae60cc 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -1623,17 +1623,22 @@ def test_provider_model_and_team_metadata_on_real_boundary_flow(): def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans(): """The pre-call hook seeds identity Baggage in the request context so the server span (stamped directly) AND later child spans (service here, via the - Baggage processor) carry identity — not just the LLM-call span.""" + Baggage processor) carry identity — not just the LLM-call span. Only the + caller's ``requester_metadata`` is read from the request dict, so a proxy-owned + sibling such as ``requester_ip_address`` is not stamped from here even though + the default allowlist names it, and an unlisted caller key is not promoted.""" logger, exporter = _logger() server = logger._emitter.start_span( SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME ) + data = { + "model": "gpt-4o", + "metadata": {"requester_ip_address": "127.0.0.1", "requester_metadata": {"trace_id": "abc"}}, + } async def _flow(): # pre-call seeds baggage + stamps the active server span - await logger.async_pre_call_hook( - _Auth(), None, {"model": "gpt-4o"}, "completion" - ) + await logger.async_pre_call_hook(_Auth(), None, data, "completion") # a later service call (same task) must inherit the identity await logger.async_service_success_hook( payload=_ServicePayload("redis", "set"), parent_otel_span=server @@ -1653,6 +1658,46 @@ def test_pre_call_hook_seeds_baggage_onto_server_and_child_spans(): srv.attributes[LiteLLM.TEAM_ID] == "t1" ) # stamped directly on the server span assert srv.attributes[f"{LiteLLM.METADATA_PREFIX}user_api_key_user_id"] == "u1" + assert not any( + k in (f"{LiteLLM.METADATA_PREFIX}requester_ip_address", f"{LiteLLM.METADATA_PREFIX}trace_id") + for s in (redis, srv) + for k in s.attributes + ) + + +def test_pre_call_hook_promotes_nested_request_metadata_key(): + """``baggage_metadata_keys: [requester_metadata.trace_id]`` reads the caller's + ``metadata.trace_id`` (snapshotted by the proxy under ``requester_metadata``) + and stamps ``litellm.metadata.trace_id`` on the server, LLM-call and service + spans of the request; unlisted siblings are not promoted.""" + cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["requester_metadata.trace_id"]) + exporter = InMemorySpanExporter() + logger = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter)) + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + data = {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}} + kwargs = _kwargs() + + async def _flow(): + await logger.async_pre_call_hook(_Auth(), None, data, "completion") + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + await logger.async_log_success_event(kwargs, None, None, None) + await logger.async_service_success_hook(payload=_ServicePayload("redis", "set"), parent_otel_span=server) + + with trace.use_span(server, end_on_exit=False): + asyncio.run(_flow()) + server.end() + + spans = {s.name: s for s in exporter.get_finished_spans()} + key = f"{LiteLLM.METADATA_PREFIX}trace_id" + assert spans[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes[key] == "abc" + assert spans["chat gpt-4o"].attributes[key] == "abc" + assert spans["redis set"].attributes[key] == "abc" + assert data == {"model": "gpt-4o", "metadata": {"requester_metadata": {"trace_id": "abc", "nested": {"deep": "x"}}}} + assert not any( + k.startswith(f"{LiteLLM.METADATA_PREFIX}requester_metadata") or k.endswith("deep") + for s in spans.values() + for k in s.attributes + ) # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 9ec8489f784..7812590b3e7 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -5581,6 +5581,38 @@ class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase): otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) assert "http.route" not in self._attr(span, exp) + def test_nested_metadata_key_promoted_under_caller_path(self): + """``baggage_metadata_keys: [requester_metadata.trace_id]`` stamps the + caller's nested metadata value as ``litellm.metadata.trace_id`` and a deeper + path keeps its dotted name; unlisted siblings stay inside the + ``metadata.requester_metadata`` blob.""" + otel = OpenTelemetry( + config=OpenTelemetryConfig( + baggage_metadata_keys=["requester_metadata.trace_id", "requester_metadata.nested.deep"] + ) + ) + kwargs = self._kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = { + "trace_id": "abc", + "nested": {"deep": "x", "skipped": "y"}, + } + span, exp = self._span() + otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) + attrs = self._attr(span, exp) + assert attrs["litellm.metadata.trace_id"] == "abc" + assert attrs["litellm.metadata.nested.deep"] == "x" + assert "litellm.metadata.deep" not in attrs + assert "litellm.metadata.nested.skipped" not in attrs + assert not any(k.startswith("litellm.metadata.requester_metadata") for k in attrs) + + def test_metadata_keys_default_to_none_promoted(self): + otel = OpenTelemetry() + kwargs = self._kwargs() + kwargs["standard_logging_object"]["metadata"]["requester_metadata"] = {"trace_id": "abc"} + span, exp = self._span() + otel.set_attributes(span, kwargs, {"model": "azure/gpt-4o"}) + assert not any(k.startswith("litellm.metadata.") for k in self._attr(span, exp)) + def test_team_metadata_json_helper(self): keys = ["a", "b"] assert OpenTelemetry._team_metadata_json(None, keys) is None @@ -5631,6 +5663,11 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase): cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"]) assert cfg.baggage_team_metadata_keys == ["from_arg"] + def test_metadata_keys_from_kwargs_and_env(self): + with patch.dict("os.environ", {"LITELLM_OTEL_BAGGAGE_METADATA_KEYS": "requester_metadata.trace_id, a.b"}): + assert OpenTelemetryConfig().baggage_metadata_keys == ["requester_metadata.trace_id", "a.b"] + assert OpenTelemetry(baggage_metadata_keys="x.y").config.baggage_metadata_keys == ["x.y"] + class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): """LIT-3600: include/exclude control over which attributes are stamped on diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 83201aef143..c3400dc40c3 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1,6 +1,7 @@ import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -579,6 +580,20 @@ def test_text_only_streaming_has_index_zero(): ), f"Expected index=0, got {parsed.choices[0].index}" +def test_message_delta_without_usage_returns_chunk_with_no_usage(): + iterator: Final = ModelResponseIterator(None, sync_stream=True) + + model_response: Final = iterator.chunk_parser( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + } + ) + + assert model_response.choices[0].finish_reason == "stop" + assert model_response.usage is None + + def test_streaming_thinking_deltas_count_reasoning_tokens_in_usage(): """Anthropic streaming usage should account for emitted thinking deltas.""" chunks = [ diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py index 1b2ca298694..c8365e7b7c0 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py @@ -38,37 +38,6 @@ def use_local_model_cost_map(): monkeypatch.undo() -@pytest.mark.parametrize( - "model_name,expected_prompt,expected_completion", - [ - ("FW-Kimi-K2.6", 1.045, 4.4), - ("FW-DeepSeek-V4-Pro", 1.925, 3.828), - ("FW-GLM-5.2", 1.54, 4.84), - ("FW-Kimi-K3", 3.3, 16.5), - ("FW-MiniMax-M2.5", 0.33, 1.32), - ("FW-Inkling", 1.0, 4.05), - ("FW-Nemotron-3-Ultra-NVFP4", 0.6, 2.4), - ("FW-Nemotron-Lightning-3.5-30B-A3B", 0.06, 0.22), - ], -) -def test_azure_ai_fw_cost_per_token( - use_local_model_cost_map, model_name, expected_prompt, expected_completion -): - from litellm.llms.azure_ai.cost_calculator import cost_per_token - from litellm.types.utils import Usage - - usage = Usage( - prompt_tokens=1_000_000, - completion_tokens=1_000_000, - total_tokens=2_000_000, - ) - - prompt_cost, completion_cost = cost_per_token(model=model_name, usage=usage) - - assert prompt_cost == pytest.approx(expected_prompt) - assert completion_cost == pytest.approx(expected_completion) - - def test_azure_ai_fw_nemotron_lightning_supports_tool_choice(use_local_model_cost_map): from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig diff --git a/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py b/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py index 7b04efa17dc..ab5a2531461 100644 --- a/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py +++ b/tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py @@ -1,3 +1,4 @@ +from typing import Final from unittest.mock import MagicMock from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig @@ -82,6 +83,7 @@ def test_transform_search_request_uses_only_retrieval_config_from_extra_body(): == "HYBRID" ) assert "unrelatedField" not in body + assert "userContext" not in body def test_transform_search_request_does_not_mutate_extra_body_and_overrides_number_of_results(): @@ -152,3 +154,44 @@ def test_transform_search_request_overrides_filter_without_mutating_extra_body() ]["value"] == "a" ) + + +def _search_body(extra_body: dict[str, object] | None, litellm_params: dict[str, object]) -> dict[str, object]: + config: Final = BedrockVectorStoreConfig() + mock_log: Final = MagicMock() + mock_log.model_call_details = {} + _, body = config.transform_search_vector_store_request( + vector_store_id="kb123", + query="hello", + vector_store_search_optional_params={"max_num_results": 3}, + api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases", + litellm_logging_obj=mock_log, + litellm_params=litellm_params, + extra_body=extra_body, + ) + return body + + +def test_transform_search_request_forwards_user_context_from_extra_body(): + body = _search_body(extra_body={"userContext": {"userId": "alice@example.com"}}, litellm_params={}) + + assert body["userContext"] == {"userId": "alice@example.com"} + assert body["retrievalConfiguration"] == {"vectorSearchConfiguration": {"numberOfResults": 3}} + + +def test_transform_search_request_forwards_top_level_user_context_from_litellm_params(): + body = _search_body( + extra_body=None, + litellm_params={"vector_store_id": "kb123", "user_context": {"userId": "bob@example.com"}}, + ) + + assert body["userContext"] == {"userId": "bob@example.com"} + + +def test_transform_search_request_prefers_extra_body_user_context_over_top_level(): + body = _search_body( + extra_body={"userContext": {"userId": "alice@example.com"}}, + litellm_params={"userContext": {"userId": "bob@example.com"}}, + ) + + assert body["userContext"] == {"userId": "alice@example.com"} diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index f8868cfaf83..33272a1a9e4 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1025,6 +1025,86 @@ def test_handed_out_sync_client_pool_survives_handler_collection(keepalive_serve consumer_client.close() +def _mock_transport() -> httpx.MockTransport: + """Answers anything with a short body, left unread when the caller asked to stream.""" + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, request=request, content=b"ab") + + return httpx.MockTransport(respond) + + +RELEASED_TOO_EARLY = "the handler was released while its response could still read" +NEVER_RELEASED = "the handler outlived the response that was holding it" + +# Every method that can hand back a body the caller has not read yet, which is +# every one that passes stream= down to send(). Parametrized so a method added +# later is covered here rather than being the one that forgets to anchor. +ASYNC_STREAMING_SENDS = ["post", "delete"] +SYNC_STREAMING_SENDS = ["post", "patch", "put", "delete"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ASYNC_STREAMING_SENDS) +async def test_a_streaming_response_holds_its_handler_until_it_is_released(method): + """The finalizer must not run while a body this handler issued can still arrive. + + ``_handler_may_close_client`` cannot see that body: it holds the connection it + reads from and never the client. Anchoring the handler to the response is what + withholds the close, and releasing the anchor is what still delivers one. + """ + handler = AsyncHTTPHandler() + handler.client._transport = _mock_transport() + ref = weakref.ref(handler) + response = await getattr(handler, method)("https://example.invalid/stream", stream=True) + + del handler + gc.collect() + assert ref() is not None, RELEASED_TOO_EARLY + + assert await response.aread() == b"ab" + del response + gc.collect() + assert ref() is None, NEVER_RELEASED + + +@pytest.mark.parametrize("method", SYNC_STREAMING_SENDS) +def test_a_sync_streaming_response_holds_its_handler_until_it_is_released(method): + """The sync finalizer closes inline, so the same anchor has to hold it off.""" + handler = HTTPHandler() + handler.client._transport = _mock_transport() + ref = weakref.ref(handler) + response = getattr(handler, method)("https://example.invalid/stream", stream=True) + + del handler + gc.collect() + assert ref() is not None, RELEASED_TOO_EARLY + + assert response.read() == b"ab" + del response + gc.collect() + assert ref() is None, NEVER_RELEASED + + +@pytest.mark.asyncio +async def test_a_fully_read_response_does_not_hold_its_handler(): + """A non-streaming response is complete when ``post`` returns, so it anchors nothing. + + Otherwise every client close would wait on whatever the caller does next with + a response it has already read. + """ + handler = AsyncHTTPHandler() + handler.client._transport = _mock_transport() + ref = weakref.ref(handler) + response = await handler.post("https://example.invalid/whole") + assert response.content == b"ab" + + del handler + gc.collect() + + assert ref() is None, "a fully-read response pinned its handler" + + def test_sync_close_leaves_caller_supplied_client_open(): supplied = httpx.Client() handler = HTTPHandler(client=supplied) @@ -1675,3 +1755,30 @@ async def test_bounded_get_closes_stream_on_cancellation(respx_mock, monkeypatch finally: await handler.close() assert closed.is_set() + + +@pytest.mark.asyncio +async def test_http2_flag_bypasses_aiohttp_transport(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + monkeypatch.setattr(litellm, "force_ipv4", False) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + + monkeypatch.setattr(litellm, "http2", True) + assert AsyncHTTPHandler._should_use_aiohttp_transport() is False + assert AsyncHTTPHandler._create_async_transport() is None + + monkeypatch.setattr(litellm, "http2", False) + monkeypatch.setenv("LITELLM_HTTP2", "True") + assert AsyncHTTPHandler._should_use_aiohttp_transport() is False + assert AsyncHTTPHandler._create_async_transport() is None + + +@pytest.mark.asyncio +async def test_http2_disabled_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "http2", False) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + + assert AsyncHTTPHandler._should_use_aiohttp_transport() is True diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index fb0311ef39b..7715e7b32ff 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1189,6 +1189,28 @@ def test_reasoning_effort_integer_passthrough(): assert isinstance(result["reasoning_effort"], int) +def test_reasoning_effort_dict_from_anthropic_adapter_flattened_to_effort_string(): + config = FireworksAIConfig() + result = config.map_openai_params( + {"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, + {}, + _REASONING_MODEL, + drop_params=False, + ) + assert result["reasoning_effort"] == "medium" + + +def test_reasoning_effort_dict_without_effort_key_dropped(): + config = FireworksAIConfig() + result = config.map_openai_params( + {"reasoning_effort": {"summary": "detailed"}}, + {}, + _REASONING_MODEL, + drop_params=False, + ) + assert "reasoning_effort" not in result + + def test_reasoning_effort_auto_dropped_to_model_default(): config = FireworksAIConfig() result = config.map_openai_params( diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index d3c21c5bd5a..b54ec10ef17 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -411,3 +411,5 @@ async def test_async_genuine_bad_request_still_raises(provider, stream): ) def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected): assert is_openai_backed_api_base(api_base) is expected + + diff --git a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py index e3feb7d5342..34ad4b9075d 100644 --- a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py @@ -51,23 +51,23 @@ class TestXAIResponsesAPITransformation: assert result["tools"][0]["type"] == "code_interpreter" assert "container" not in result["tools"][0], "Container field should be removed" - def test_instructions_parameter_dropped(self): - """Test that instructions parameter is dropped for XAI""" + def test_instructions_parameter_forwarded(self): + """xAI supports 'instructions' on /v1/responses, so it must survive param mapping""" config = XAIResponsesAPIConfig() params = ResponsesAPIOptionalRequestParams(instructions="You are a helpful assistant.", temperature=0.7) result = config.map_openai_params(response_api_optional_params=params, model="grok-4-fast", drop_params=False) - assert "instructions" not in result, "Instructions should be dropped" + assert result.get("instructions") == "You are a helpful assistant." assert result.get("temperature") == 0.7, "Other params should be preserved" - def test_supported_params_excludes_instructions(self): - """Test that get_supported_openai_params excludes instructions""" + def test_supported_params_includes_instructions(self): + """A system message bridged to 'instructions' must not be rejected for xAI""" config = XAIResponsesAPIConfig() supported = config.get_supported_openai_params("grok-4-fast") - assert "instructions" not in supported, "instructions should not be supported" + assert "instructions" in supported, "instructions should be supported" assert "tools" in supported, "tools should be supported" assert "temperature" in supported, "temperature should be supported" assert "model" in supported, "model should be supported" diff --git a/tests/test_litellm/llms/xai/xai_responses/test_transformation.py b/tests/test_litellm/llms/xai/xai_responses/test_transformation.py index c783918ca06..3ea3fe631bd 100644 --- a/tests/test_litellm/llms/xai/xai_responses/test_transformation.py +++ b/tests/test_litellm/llms/xai/xai_responses/test_transformation.py @@ -53,8 +53,8 @@ class TestXAIResponsesAPITransformation: "container" not in result["tools"][0] ), "Container field should be removed" - def test_instructions_parameter_dropped(self): - """Test that instructions parameter is dropped for XAI""" + def test_instructions_parameter_forwarded(self): + """xAI supports 'instructions' on /v1/responses, so it must survive param mapping""" config = XAIResponsesAPIConfig() params = ResponsesAPIOptionalRequestParams( @@ -65,15 +65,15 @@ class TestXAIResponsesAPITransformation: response_api_optional_params=params, model="grok-4-fast", drop_params=False ) - assert "instructions" not in result, "Instructions should be dropped" + assert result.get("instructions") == "You are a helpful assistant." assert result.get("temperature") == 0.7, "Other params should be preserved" - def test_supported_params_excludes_instructions(self): - """Test that get_supported_openai_params excludes instructions""" + def test_supported_params_includes_instructions(self): + """A system message bridged to 'instructions' must not be rejected for xAI""" config = XAIResponsesAPIConfig() supported = config.get_supported_openai_params("grok-4-fast") - assert "instructions" not in supported, "instructions should not be supported" + assert "instructions" in supported, "instructions should be supported" assert "tools" in supported, "tools should be supported" assert "temperature" in supported, "temperature should be supported" assert "model" in supported, "model should be supported" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 9ea870d3210..aa45b2f6793 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -5,7 +5,7 @@ import json import time from base64 import urlsafe_b64encode from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -15,6 +15,9 @@ from litellm.types.mcp import MCPAuth if TYPE_CHECKING: import httpx + from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey + + from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -6977,6 +6980,11 @@ async def _exchange_persistence_attempted_for_auth_type(auth_type) -> bool: new_callable=AsyncMock, return_value="admin-user", ), + patch( # test-quality-ok: this control tests persistence by auth mode; write-policy behavior is covered separately + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.authorize_oauth_credential_request", + new_callable=AsyncMock, + return_value="admin-user", + ), patch( "litellm.proxy._experimental.mcp_server.discoverable_endpoints._store_per_user_token_server_side", new_callable=AsyncMock, @@ -7124,12 +7132,12 @@ async def test_build_oauth_protected_resource_response_obo_end_to_end(): global_mcp_server_manager.registry.clear() -def _token_request(headers): +def _token_request(headers, path="/token"): """A real Starlette request with case-insensitive headers (matches production).""" from starlette.requests import Request raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()] - return Request({"type": "http", "method": "POST", "path": "/token", "headers": raw, "query_string": b""}) + return Request({"type": "http", "method": "POST", "path": path, "headers": raw, "query_string": b""}) @pytest.fixture @@ -11162,14 +11170,14 @@ async def test_identity_bound_authorization_carries_nonce_and_caller_through_cal ), ) request = Request({"type": "http", "scheme": "https", "server": ("proxy.example.com", 443), - "path": "/authorize", "query_string": b"", "headers": []}) + "path": "/authorize", "query_string": b"", "headers": [(b"authorization", b"Bearer sk-alice")]}) with ( patch( # test-quality-ok: isolate authenticated request resolution from the real encrypted OAuth round trip - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.authorize_oauth_credential_request", new=AsyncMock(return_value="alice")), patch( # test-quality-ok: isolate user access lookup while testing nonce and caller preservation - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._bridge_authorize_access_denial", - new=AsyncMock(return_value=None)), + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._user_can_reach_mcp_server", + new=AsyncMock(return_value=True)), ): authorized = await authorize_with_server( request, server, "client", "http://127.0.0.1:6274/callback", state="client-state", @@ -11374,3 +11382,858 @@ with TestClient(app) as client: assert responses[path]["status"] == 200, responses[path] assert responses[path]["body"]["issuer"] == f"http://testserver/gateway/{path}" assert responses["example/mcp"]["body"]["token_endpoint"] == "http://testserver/gateway/example/token" + + +@pytest.fixture +def jwt_oauth_identity(monkeypatch: pytest.MonkeyPatch) -> tuple["JWTHandler", "RSAPrivateKey"]: + import jwt + from cryptography.hazmat.primitives.asymmetric import rsa + + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_JWTAuth + from litellm.proxy.auth.handle_jwt import JWTHandler + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + signing_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + cache: Final = UserApiKeyCache() + cache.set_cache( + "litellm_jwt_auth_keys_https://idp.example.test/jwks", + [json.loads(jwt.algorithms.RSAAlgorithm.to_jwk(signing_key.public_key()))], + ) + cache.set_cache("jwt-owner", LiteLLM_UserTable(user_id="jwt-owner", user_email="owner@example.test")) + handler: Final = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=cache, + litellm_jwtauth=LiteLLM_JWTAuth(user_id_jwt_field="identity.user_id"), + ) + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://idp.example.test/jwks") + monkeypatch.setenv("JWT_ISSUER", "https://idp.example.test") + monkeypatch.setenv("JWT_AUDIENCE", "litellm-proxy") + monkeypatch.setattr(proxy_server, "jwt_handler", handler) + monkeypatch.setattr(proxy_server, "general_settings", {"enable_jwt_auth": True}) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + return handler, signing_key + + +def _oauth_identity_jwt( + signing_key: "RSAPrivateKey", + *, + expires_in: int = 300, + audience: str = "litellm-proxy", + issuer: str = "https://idp.example.test", + owner: str | None = "jwt-owner", + scope: str = "", + claims: dict[str, object] | None = None, +) -> str: + import jwt + + return jwt.encode( + { + "sub": "not-the-configured-user-id", + "identity": {"user_id": owner}, + "email": "owner@example.test", + "iss": issuer, + "aud": audience, + "exp": int(time.time()) + expires_in, + "scope": scope, + **(claims or {}), + }, + signing_key, + algorithm="RS256", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("header", ["Authorization", "x-litellm-api-key"]) +@pytest.mark.parametrize("policy_allowed", [False, True]) +@pytest.mark.parametrize("server_allowed", [False, True]) +@pytest.mark.parametrize("admin", [False, True]) +@pytest.mark.parametrize("owner_state", ["active", "missing", "inactive", "database_error"]) +async def test_oauth_exchange_stores_token_for_validated_jwt_user( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + header: str, + policy_allowed: bool, + server_allowed: bool, + admin: bool, + owner_state: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import httpx + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + handler, signing_key = jwt_oauth_identity + handler.litellm_jwtauth.custom_validate = lambda claims: policy_allowed + from litellm.proxy._experimental.mcp_server import mcp_server_manager + + manager: Final = MagicMock() + manager.get_allowed_mcp_servers = AsyncMock(return_value=["jwt-oauth-server"] if server_allowed else []) + manager.invalidate_user_oauth_token_cache = AsyncMock() + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + bearer: Final = _oauth_identity_jwt(signing_key, scope="litellm_proxy_admin" if admin else "") + request: Final = _token_request({header: f"Bearer {bearer}"}, path="/jwt-oauth-server/token") + server: Final = MCPServer( + server_id="jwt-oauth-server", + name="jwt-oauth-server", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + oauth2_flow="authorization_code", + authorization_url="https://upstream.example.test/authorize", + token_url="https://upstream.example.test/token", + client_id="registered-client", + ) + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy import proxy_server + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + from litellm.types.llms.custom_http import httpxSpecialProvider + + def upstream_response(outbound: httpx.Request) -> httpx.Response: + assert outbound.url == server.token_url + assert bearer not in str(outbound.headers) + assert bearer.encode() not in outbound.content + return httpx.Response(200, json={"access_token": "upstream-token", "token_type": "Bearer"}) + + database: Final = MagicMock() + users: Final = database.db.litellm_usertable + users.find_unique = AsyncMock(return_value=None) + users.find_first = AsyncMock(return_value=None) + users.create = AsyncMock() + if owner_state in ("missing", "database_error"): + handler.user_api_key_cache.delete_cache("jwt-owner") + if owner_state == "database_error": + users.find_unique.side_effect = RuntimeError("database unavailable") + if owner_state == "inactive": + handler.user_api_key_cache.set_cache( + "jwt-owner", LiteLLM_UserTable(user_id="jwt-owner", metadata={"scim_active": False}) + ) + table: Final = database.db.litellm_mcpusercredentials + table.find_unique = AsyncMock(return_value=None) + table.upsert = AsyncMock() + monkeypatch.setattr(proxy_server, "prisma_client", database) + monkeypatch.setenv("LITELLM_SALT_KEY", "oauth-jwt-test-encryption-key") + clients: Final = LLMClientCache() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", clients) + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream_response)) as transport: + upstream: Final = AsyncHTTPHandler() + await upstream.client.aclose() + upstream.client = transport + clients.set_cache("async_httpx_client" + httpxSpecialProvider.Oauth2Check, upstream) + response: Final = await discoverable_endpoints.exchange_token_with_server( + request=request, + mcp_server=server, + grant_type="authorization_code", + code="upstream-code", + redirect_uri="http://localhost/callback", + client_id="registered-client", + client_secret=None, + code_verifier=None, + ) + assert response.status_code == 200 + assert json.loads(response.body)["access_token"] == "upstream-token" + users.create.assert_not_awaited() + if ( + not server_allowed + or not policy_allowed + or owner_state in ("inactive", "database_error") + or (owner_state == "missing" and not admin) + ): + table.upsert.assert_not_awaited() + return + table.upsert.assert_awaited_once() + stored: Final = table.upsert.call_args.kwargs + assert stored["where"] == {"user_id_server_id": {"user_id": "jwt-owner", "server_id": server.server_id}} + credential: Final = stored["data"]["create"]["credential_b64"] + assert "upstream-token" not in credential + decoded: Final = decrypt_value_helper(credential, key="mcp_user_credential") + assert json.loads(decoded)["access_token"] == "upstream-token" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "rejection", + [ + "expired", + "audience", + "issuer", + "signature", + "missing_user", + "unknown_user", + "disabled", + "not_premium", + "scim_inactive", + "custom_validate", + "missing_database", + ], +) +@pytest.mark.parametrize("credential_write", [False, True]) +async def test_oauth_jwt_identity_rejects_untrusted_or_inactive_owner( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + rejection: str, + credential_write: bool, +) -> None: + from cryptography.hazmat.primitives.asymmetric import rsa + + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + _extract_user_id_from_request, authorize_oauth_credential_request, + ) + + allowed_servers: Final = AsyncMock(return_value=["server-a"]) + monkeypatch.setattr(mcp_server_manager.global_mcp_server_manager, "get_allowed_mcp_servers", allowed_servers) + handler, signing_key = jwt_oauth_identity + key: Final = ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) if rejection == "signature" else signing_key + ) + bearer: Final = _oauth_identity_jwt( + key, + expires_in=-60 if rejection == "expired" else 300, + audience="upstream-only" if rejection == "audience" else "litellm-proxy", + issuer="https://untrusted.example.test" if rejection == "issuer" else "https://idp.example.test", + owner=None if rejection == "missing_user" else "unknown" if rejection == "unknown_user" else "jwt-owner", + ) + if rejection == "disabled": + monkeypatch.setattr(proxy_server, "general_settings", {"enable_jwt_auth": False}) + if rejection == "not_premium": + monkeypatch.setattr(proxy_server, "premium_user", False) + if rejection == "missing_database": + monkeypatch.setattr(proxy_server, "prisma_client", None) + if rejection == "scim_inactive": + handler.user_api_key_cache.set_cache( + "jwt-owner", LiteLLM_UserTable(user_id="jwt-owner", metadata={"scim_active": False}) + ) + if rejection == "custom_validate": + handler.litellm_jwtauth.custom_validate = lambda claims: False + request: Final = _token_request({"Authorization": f"Bearer {bearer}"}) + result: Final = ( + await authorize_oauth_credential_request(request, "server-a") + if credential_write else await _extract_user_id_from_request(request) + ) + assert result is None + allowed_servers.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blocked", [False, True]) +async def test_oauth_jwt_cannot_override_explicit_litellm_key( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + blocked: bool, +) -> None: + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _extract_user_id_from_request + from litellm.proxy._types import UserAPIKeyAuth, hash_token + + handler, signing_key = jwt_oauth_identity + key: Final = "sk-explicit-key" + handler.user_api_key_cache.set_cache(hash_token(key), UserAPIKeyAuth(user_id="key-owner", blocked=blocked)) + request: Final = _token_request( + { + "Authorization": f"Bearer {_oauth_identity_jwt(signing_key)}", + "x-litellm-api-key": key, + } + ) + assert await _extract_user_id_from_request(request) == (None if blocked else "key-owner") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mapping", ["active", "blocked", "inactive_owner", "fallback", "pending", "reject", "custom_reject"] +) +async def test_oauth_jwt_uses_configured_virtual_key_owner( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + mapping: str, +) -> None: + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _extract_user_id_from_request + from litellm.proxy._types import UserAPIKeyAuth, UnregisteredJWTClientBehavior, hash_token + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + + handler, signing_key = jwt_oauth_identity + handler.litellm_jwtauth.virtual_key_claim_field = "sub" + if mapping == "custom_reject": + handler.litellm_jwtauth.custom_validate = lambda claims: False + handler.litellm_jwtauth.unregistered_jwt_client_behavior = ( + UnregisteredJWTClientBehavior.AUTO_REGISTER + if mapping == "pending" + else UnregisteredJWTClientBehavior.REJECT + if mapping == "reject" + else UnregisteredJWTClientBehavior.FALLBACK_TEAM_MAPPING + ) + key_hash: Final = hash_token("sk-mapped-oauth-owner") + handler.user_api_key_cache.set_cache( + jwt_key_mapping_cache_key("sub", "not-the-configured-user-id"), + "__NO_MAPPING__" if mapping in ("fallback", "pending", "reject") else key_hash, + ) + handler.user_api_key_cache.set_cache( + key_hash, UserAPIKeyAuth(token=key_hash, user_id="mapped-owner", blocked=mapping == "blocked") + ) + handler.user_api_key_cache.set_cache( + "mapped-owner", LiteLLM_UserTable(user_id="mapped-owner", metadata={"scim_active": mapping != "inactive_owner"}) + ) + request: Final = _token_request({"Authorization": f"Bearer {_oauth_identity_jwt(signing_key)}"}) + expected: Final = "jwt-owner" if mapping == "fallback" else "mapped-owner" if mapping == "active" else None + assert await _extract_user_id_from_request(request) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed_domain", [None, "allowed.example.test"]) +async def test_oauth_jwt_respects_custom_validation_and_email_policy( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + allowed_domain: str | None, +) -> None: + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _extract_user_id_from_request + + handler, signing_key = jwt_oauth_identity + handler.litellm_jwtauth.custom_validate = lambda claims: True + handler.litellm_jwtauth.user_allowed_email_domain = allowed_domain + request: Final = _token_request({"Authorization": f"Bearer {_oauth_identity_jwt(signing_key)}"}) + assert await _extract_user_id_from_request(request) == (None if allowed_domain else "jwt-owner") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route_allowed", [False, True]) +async def test_oauth_jwt_identity_preserves_separate_mcp_route_authorization( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + route_allowed: bool, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _extract_user_id_from_request + from litellm.proxy._types import LitellmUserRoles, RoleBasedPermissions, RoleMapping + from litellm.proxy.auth.handle_jwt import JWTAuthManager + + handler, signing_key = jwt_oauth_identity + handler.litellm_jwtauth.user_id_jwt_field = "sub" + handler.litellm_jwtauth.roles_jwt_field = "aud" + handler.litellm_jwtauth.object_id_jwt_field = "identity.user_id" + handler.litellm_jwtauth.role_mappings = [ + RoleMapping(role="litellm-proxy", internal_role=LitellmUserRoles.INTERNAL_USER) + ] + handler.litellm_jwtauth.enforce_rbac = True + monkeypatch.setattr( + proxy_server, + "general_settings", + { + "enable_jwt_auth": True, + "role_permissions": [ + RoleBasedPermissions( + role=LitellmUserRoles.INTERNAL_USER, + routes=["mcp_routes"] if route_allowed else ["/models"], + ) + ], + }, + ) + bearer: Final = _oauth_identity_jwt(signing_key) + request: Final = _token_request({"Authorization": f"Bearer {bearer}"}, path="/example/token") + assert await _extract_user_id_from_request(request) == "jwt-owner" + admission: Final = JWTAuthManager.auth_builder( + api_key=bearer, + jwt_handler=handler, + request_data={}, + general_settings=proxy_server.general_settings, + route="/mcp/example", + prisma_client=proxy_server.prisma_client, + user_api_key_cache=handler.user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_server.proxy_logging_obj, + request_method="POST", + ) + if route_allowed: + assert (await admission)["user_id"] == "jwt-owner" + else: + with pytest.raises(HTTPException) as denial: + await admission + assert denial.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("identity", ["sso", "email"]) +@pytest.mark.parametrize("inactive", [False, True]) +@pytest.mark.parametrize("admin", [False, True]) +async def test_oauth_jwt_resolves_canonical_owner_without_cached_identity( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + identity: str, + inactive: bool, + admin: bool, +) -> None: + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _extract_user_id_from_request + from litellm.proxy.auth.handle_jwt import JWTAuthManager + + handler, signing_key = jwt_oauth_identity + external_id: Final = f"external-{identity}-{inactive}-{admin}" + handler.litellm_jwtauth.user_email_jwt_field = "email" + handler.litellm_jwtauth.admin_allowed_routes = ["mcp_routes"] + owner: Final = LiteLLM_UserTable( + user_id="canonical-oauth-owner", + user_email="owner@example.test", + metadata={"scim_active": not inactive}, + organization_memberships=[], + ) + database: Final = MagicMock() + table: Final = database.db.litellm_usertable + table.find_unique = AsyncMock(side_effect=[None, owner if identity == "sso" else None]) + table.find_first = AsyncMock(return_value=owner) + table.update = AsyncMock(return_value=owner) + monkeypatch.setattr(proxy_server, "prisma_client", database) + bearer: Final = _oauth_identity_jwt(signing_key, owner=external_id, scope="litellm_proxy_admin" if admin else "") + request: Final = _token_request({"Authorization": f"Bearer {bearer}"}) + stored_owner: Final = await _extract_user_id_from_request(request) + assert stored_owner == (None if inactive else external_id if admin else "canonical-oauth-owner") + assert table.find_unique.await_count == 2 + if identity == "email": + table.find_first.assert_awaited_once() + if not inactive: + admission: Final = await JWTAuthManager.auth_builder( + api_key=bearer, + jwt_handler=handler, + request_data={}, + general_settings=proxy_server.general_settings, + route="/mcp/example", + prisma_client=database, + user_api_key_cache=handler.user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + assert stored_owner == admission["user_id"] + + +@pytest.mark.asyncio +async def test_oauth_jwt_identity_does_not_provision_or_synchronize_teams( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], +) -> None: + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _extract_user_id_from_request + + handler, signing_key = jwt_oauth_identity + handler.litellm_jwtauth.enforce_team_based_model_access = True + handler.litellm_jwtauth.team_id_default = "new-team" + handler.litellm_jwtauth.team_id_upsert = True + handler.litellm_jwtauth.sync_user_role_and_teams = True + owner: Final = LiteLLM_UserTable(user_id="jwt-owner", teams=["existing-team"]) + handler.user_api_key_cache.set_cache("jwt-owner", owner) + request: Final = _token_request( + {"Authorization": f"Bearer {_oauth_identity_jwt(signing_key)}"}, path="/example/token" + ) + assert await _extract_user_id_from_request(request) == "jwt-owner" + assert owner.teams == ["existing-team"] + proxy_server.prisma_client.db.litellm_teamtable.find_unique.assert_not_called() + proxy_server.prisma_client.db.litellm_teamtable.upsert.assert_not_called() + proxy_server.prisma_client.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("state", ["active", "inactive", "missing_database"]) +async def test_oauth_refresh_revalidates_the_same_active_user_rule( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + state: str, +) -> None: + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _reload_active_user_by_id + + handler, _ = jwt_oauth_identity + handler.user_api_key_cache.set_cache( + "jwt-owner", LiteLLM_UserTable(user_id="jwt-owner", metadata={"scim_active": state != "inactive"}) + ) + if state == "missing_database": + monkeypatch.setattr(proxy_server, "prisma_client", None) + expected: Final = None if state == "active" else "no_active_key" if state == "inactive" else "unresolvable" + assert await _reload_active_user_by_id("jwt-owner") == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mapped", [False, True]) +@pytest.mark.parametrize("state", ["allowed", "route_denied", "server_denied", "blocked", "expired", "lookup_error", "cancelled"]) +async def test_oauth_credential_write_keeps_virtual_key_permissions( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + mapped: bool, + state: str, +) -> None: + import asyncio + + from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy._experimental.mcp_server.bridge_token_flow import authorize_oauth_credential_request + from litellm.proxy._types import UserAPIKeyAuth, hash_token + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + + handler, signing_key = jwt_oauth_identity + key: Final = "sk-oauth-permission-test" + hashed: Final = hash_token(key) + credential: Final = UserAPIKeyAuth( + token=hashed, + user_id="jwt-owner", + blocked=state == "blocked", + expires=datetime.now(timezone.utc) - timedelta(seconds=60) if state == "expired" else None, + allowed_routes=["openai_routes"] if state == "route_denied" else ["mcp_routes"], + agent_id="agent-scope", + org_id="org-scope", + end_user_id="end-user-scope", + ) + handler.user_api_key_cache.set_cache(hashed, credential) + if mapped: + handler.litellm_jwtauth.virtual_key_claim_field = "sub" + handler.user_api_key_cache.set_cache(jwt_key_mapping_cache_key("sub", "not-the-configured-user-id"), hashed) + manager: Final = MagicMock() + manager.get_allowed_mcp_servers = AsyncMock( + return_value=[] if state == "server_denied" else ["server-a"], + side_effect=(asyncio.CancelledError() if state == "cancelled" else RuntimeError("permission lookup unavailable") if state == "lookup_error" else None), + ) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + bearer: Final = _oauth_identity_jwt(signing_key) if mapped else key + request: Final = _token_request({"Authorization": f"Bearer {bearer}"}, path="/server-a/token") + if state == "cancelled": + with pytest.raises(asyncio.CancelledError): + await authorize_oauth_credential_request(request, "server-a") + manager.get_allowed_mcp_servers.assert_awaited_once() + return + assert await authorize_oauth_credential_request(request, "server-a") == ("jwt-owner" if state == "allowed" else None) + if state in ("allowed", "server_denied", "lookup_error"): + manager.get_allowed_mcp_servers.assert_awaited_once() + writer: Final = manager.get_allowed_mcp_servers.call_args.args[0] + assert (writer.user_id, writer.token, writer.org_id, writer.agent_id, writer.end_user_id) == ( + "jwt-owner", + hashed, + "org-scope", + "agent-scope", + "end-user-scope", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("server_id", ["team-a-server", "team-b-server"]) +async def test_oauth_writer_preserves_claimed_team_instead_of_expanding_user_roster( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + server_id: str, +) -> None: + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy._experimental.mcp_server.bridge_token_flow import authorize_oauth_credential_request + from litellm.proxy._types import LiteLLM_TeamTable, Member + + handler, signing_key = jwt_oauth_identity + handler.litellm_jwtauth.team_id_jwt_field = "team" + handler.litellm_jwtauth.team_id_upsert = True + handler.litellm_jwtauth.user_id_upsert = True + handler.litellm_jwtauth.sync_user_role_and_teams = True + handler.user_api_key_cache.set_cache("jwt-owner", LiteLLM_UserTable(user_id="jwt-owner", teams=["a", "b"])) + handler.user_api_key_cache.set_cache( + "team_id:a", + LiteLLM_TeamTable(team_id="a", models=[], members_with_roles=[Member(user_id="jwt-owner", role="user")]), + ) + manager: Final = MagicMock() + manager.get_allowed_mcp_servers = AsyncMock(return_value=["team-a-server"]) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + bearer: Final = _oauth_identity_jwt(signing_key, claims={"team": "a"}) + request: Final = _token_request({"Authorization": f"Bearer {bearer}"}, path=f"/{server_id}/token") + assert await authorize_oauth_credential_request(request, server_id) == ( + "jwt-owner" if server_id == "team-a-server" else None + ) + manager.get_allowed_mcp_servers.assert_awaited_once() + writer: Final = manager.get_allowed_mcp_servers.call_args.args[0] + assert writer.team_id == "a" + assert not writer.mcp_admitted_user_subject + proxy_server.prisma_client.db.litellm_teamtable.create.assert_not_called() + proxy_server.prisma_client.db.litellm_usertable.create.assert_not_called() + proxy_server.prisma_client.db.litellm_usertable.update.assert_not_called() + assert handler.litellm_jwtauth.user_id_upsert and handler.litellm_jwtauth.team_id_upsert + assert handler.litellm_jwtauth.sync_user_role_and_teams + + +@pytest.mark.asyncio +async def test_oauth_write_denial_does_not_erase_identity_binding( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + _, signing_key = jwt_oauth_identity + monkeypatch.setenv("LITELLM_SALT_KEY", "oauth-identity-binding-test-salt") + manager: Final = MagicMock() + manager.get_allowed_mcp_servers = AsyncMock(return_value=[]) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + server: Final = MCPServer( + server_id="bound-server", name="bound-server", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="client", + token_url="https://upstream.example.test/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://upstream.example.test", audiences=["client"], + ), + ) + request: Final = _token_request({"Authorization": f"Bearer {_oauth_identity_jwt(signing_key)}"}) + code: Final = discoverable_endpoints.seal_bridge_authorization_code( + "upstream-code", "another-owner", server.server_id, "bound-nonce", + ) + with pytest.raises(HTTPException) as denied: + await discoverable_endpoints.exchange_token_with_server( + request=request, mcp_server=server, grant_type="authorization_code", code=code, + redirect_uri="http://localhost/callback", client_id="client", client_secret=None, code_verifier="verifier", + ) + assert denied.value.status_code == 403 + assert denied.value.detail == {"error": "oauth_principal_mismatch"} + manager.get_allowed_mcp_servers.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("admin_only", [False, True]) +async def test_signed_oauth_callback_honors_credential_write_policy( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + admin_only: bool, +) -> None: + import httpx + import litellm + + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager + from litellm.proxy._types import MCPTransport + from litellm.types.llms.custom_http import httpxSpecialProvider + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server: Final = MCPServer( + server_id="signed-server", name="signed-server", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="client", + token_url="https://upstream.example.test/token", + ) + monkeypatch.setattr(proxy_server, "general_settings", { + "enable_jwt_auth": True, + "admin_only_routes": [f"/v1/mcp/server/{server.server_id}/oauth-user-credential"] if admin_only else [], + }) + monkeypatch.setenv("LITELLM_SALT_KEY", "signed-oauth-test-salt") + manager: Final = MagicMock() + manager.get_allowed_mcp_servers = AsyncMock(return_value=[server.server_id]) + manager.invalidate_user_oauth_token_cache = AsyncMock() + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + table: Final = proxy_server.prisma_client.db.litellm_mcpusercredentials + table.find_unique = AsyncMock(return_value=None) + table.upsert = AsyncMock() + clients: Final = LLMClientCache() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", clients) + + def upstream_response(outbound: httpx.Request) -> httpx.Response: + assert outbound.url == server.token_url + assert b"code=upstream-code" in outbound.content + return httpx.Response(200, json={"access_token": "upstream-token", "token_type": "Bearer"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream_response)) as transport: + upstream: Final = AsyncHTTPHandler() + await upstream.client.aclose() + upstream.client = transport + clients.set_cache("async_httpx_client" + httpxSpecialProvider.Oauth2Check, upstream) + response: Final = await discoverable_endpoints.exchange_token_with_server( + request=_token_request({}, path="/signed-server/token"), mcp_server=server, + grant_type="authorization_code", + code=discoverable_endpoints.seal_bridge_authorization_code("upstream-code", "jwt-owner", server.server_id), + redirect_uri="http://localhost/callback", client_id="client", client_secret=None, code_verifier=None, + ) + assert response.status_code == 200 + assert json.loads(response.body)["access_token"] == "upstream-token" + if admin_only: + table.upsert.assert_not_awaited() + else: + table.upsert.assert_awaited_once() + assert table.upsert.call_args.kwargs["where"]["user_id_server_id"] == { + "user_id": "jwt-owner", "server_id": server.server_id, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed", [False, True]) +@pytest.mark.parametrize("credential", [ + "jwt", "key", "expired_jwt", "wrong_audience", "bad_signature", "malformed_jwt", "missing_issuer", + "foreign_explicit", "blank_explicit", "unknown_key", "blocked_key", "expired_key", "opaque_record", + "opaque_outage", "opaque_oidc", "opaque_custom", "foreign_unscoped", "foreign_configured", "encrypted", "invalid_encrypted", "envelope", "master", +]) +async def test_identity_bound_authorize_preserves_presented_jwt_permissions( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + allowed: bool, + credential: str, +) -> None: + import jwt + from datetime import datetime, timedelta, timezone + from urllib.parse import parse_qs, urlparse + + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._types import JWTIssuerConfig, UserAPIKeyAuth, hash_token + from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + handler, signing_key = jwt_oauth_identity + master: Final = "browser-session-test-signing-key-123456789" + monkeypatch.setattr(proxy_server, "master_key", master) + monkeypatch.setattr(proxy_server, "user_custom_auth", (lambda: None) if credential == "opaque_custom" else None) + handler.litellm_jwtauth.oidc_userinfo_enabled = credential == "opaque_oidc" + if credential == "foreign_unscoped": + monkeypatch.delenv("JWT_ISSUER") + if credential == "foreign_configured": + handler.litellm_jwtauth.issuers = [JWTIssuerConfig( + issuer="https://unrelated.example.test", jwks_url="https://idp.example.test/jwks", + audience="litellm-proxy", user_id_jwt_field="identity.user_id", + )] + proxy_server.prisma_client.get_data = AsyncMock( + return_value=None, side_effect=RuntimeError("database unavailable") if credential == "opaque_outage" else None, + ) + handler.user_api_key_cache.set_cache("cookie-owner", LiteLLM_UserTable(user_id="cookie-owner")) + key: Final = "opaque-record" if credential == "opaque_record" else "sk-browser-gateway-key" + if credential in ("key", "blocked_key", "expired_key", "opaque_record"): + handler.user_api_key_cache.set_cache(hash_token(key), UserAPIKeyAuth( + token=hash_token(key), user_id="jwt-owner", blocked=credential in ("blocked_key", "opaque_record"), + expires=datetime.now(timezone.utc) - timedelta(seconds=60) if credential == "expired_key" else None, + )) + monkeypatch.setenv("LITELLM_SALT_KEY", "authorize-policy-test-salt") + server: Final = MCPServer( + server_id="bound-server", name="bound-server", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="client", + authorization_url="https://upstream.example.test/authorize", token_url="https://upstream.example.test/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://upstream.example.test", audiences=["client"], + ), + ) + manager: Final = MagicMock() + # The full user roster permits the server; the presented JWT may have narrower access. + manager.get_allowed_mcp_servers = AsyncMock( + side_effect=lambda auth: [server.server_id] if allowed or auth.mcp_admitted_user_subject else [], + ) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + bearer: Final = ( + key if credential in ("key", "blocked_key", "expired_key", "opaque_record", "unknown_key") + else "opaque-bearer" if credential in ("opaque_outage", "opaque_oidc", "opaque_custom") + else "not.a.jwt" if credential == "malformed_jwt" + else "llm_env_invalid" if credential == "envelope" + else "v2:gcm:invalid" if credential == "invalid_encrypted" + else master if credential == "master" + else ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( + LiteLLM_UserTable(user_id="jwt-owner", user_role="internal_user"), + ) if credential == "encrypted" + else jwt.encode({"iss": "https://idp.example.test"}, "wrong-signing-key-at-least-32-bytes", algorithm="HS256") + if credential == "bad_signature" + else jwt.encode({"sub": "jwt-owner"}, signing_key, algorithm="RS256") if credential == "missing_issuer" + else _oauth_identity_jwt( + signing_key, + expires_in=-60 if credential == "expired_jwt" else 300, + audience="another-service" if credential == "wrong_audience" else "litellm-proxy", + issuer="https://unrelated.example.test" if credential.startswith("foreign_") or credential == "blank_explicit" else "https://idp.example.test", + ) + ) + cookie: Final = jwt.encode( + {"user_id": "cookie-owner", "login_method": "sso", "exp": int(time.time()) + 300}, master, algorithm="HS256", + ) + response: Final = await discoverable_endpoints.authorize_with_server( + request=_token_request({ + "Authorization": f"Bearer {bearer}", "Cookie": f"token={cookie}", + **({"x-litellm-api-key": bearer} if credential == "foreign_explicit" else {}), + **({"x-litellm-api-key": ""} if credential == "blank_explicit" else {}), + }), + mcp_server=server, client_id="client", redirect_uri="http://127.0.0.1:6274/callback", + state="client-state", code_challenge="pkce-challenge", code_challenge_method="S256", + ) + redirect: Final = urlparse(response.headers["location"]) + query: Final = parse_qs(redirect.query) + if allowed and credential in ("jwt", "key", "foreign_unscoped", "foreign_configured"): + assert redirect.hostname == "upstream.example.test" + assert query["nonce"] and response.headers.get("set-cookie") + assert all(call.args[0].user_id == "jwt-owner" for call in manager.get_allowed_mcp_servers.await_args_list) + else: + assert redirect.hostname == "127.0.0.1" + assert query["error"] == ["access_denied"] + assert query["state"] == ["client-state"] + assert "set-cookie" not in response.headers + + proxy_server.prisma_client.db.litellm_mcpusercredentials.upsert.assert_not_called() + proxy_server.prisma_client.db.litellm_usertable.create.assert_not_called() + proxy_server.prisma_client.db.litellm_teamtable.create.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("credential", ["none", "opaque", "foreign_jwt"]) +@pytest.mark.parametrize("cookie_state", ["allowed", "server_denied", "expired", "missing"]) +async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + credential: str, + cookie_state: str, +) -> None: + import jwt + from urllib.parse import parse_qs, urlparse + + from litellm.models.user import LiteLLM_UserTable + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPOAuthIdentityBinding, MCPServer + + handler, signing_key = jwt_oauth_identity + master: Final = "browser-session-test-signing-key-123456789" + monkeypatch.setattr(proxy_server, "master_key", master) + monkeypatch.setattr(proxy_server, "user_custom_auth", None) + monkeypatch.setenv("LITELLM_SALT_KEY", "authorize-policy-test-salt") + handler.user_api_key_cache.set_cache("cookie-owner", LiteLLM_UserTable(user_id="cookie-owner")) + proxy_server.prisma_client.get_data = AsyncMock(return_value=None) + server: Final = MCPServer( + server_id="bound-server", name="bound-server", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="client", + authorization_url="https://upstream.example.test/authorize", token_url="https://upstream.example.test/token", + oauth_identity_binding=MCPOAuthIdentityBinding( + mode="enforce", issuer="https://upstream.example.test", audiences=["client"], + ), + ) + manager: Final = MagicMock() + manager.get_allowed_mcp_servers = AsyncMock(return_value=[] if cookie_state == "server_denied" else [server.server_id]) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + bearer: Final = ( + _oauth_identity_jwt(signing_key, issuer="https://unrelated.example.test") + if credential == "foreign_jwt" else "unrelated-upstream-bearer" + ) + cookie: Final = jwt.encode( + {"user_id": "cookie-owner", "login_method": "sso", "exp": int(time.time()) + (-60 if cookie_state == "expired" else 300)}, + master, algorithm="HS256", + ) + response: Final = await discoverable_endpoints.authorize_with_server( + request=_token_request({ + **({"Authorization": f"Bearer {bearer}"} if credential != "none" else {}), + **({"Cookie": f"token={cookie}"} if cookie_state != "missing" else {}), + }), + mcp_server=server, client_id="client", redirect_uri="http://127.0.0.1:6274/callback", + state="client-state", code_challenge="pkce-challenge", code_challenge_method="S256", + ) + redirect: Final = urlparse(response.headers["location"]) + query: Final = parse_qs(redirect.query) + if cookie_state == "allowed": + assert redirect.hostname == "upstream.example.test" + assert query["nonce"] and response.headers.get("set-cookie") + manager.get_allowed_mcp_servers.assert_awaited_once() + assert manager.get_allowed_mcp_servers.call_args.args[0].user_id == "cookie-owner" + elif cookie_state == "server_denied": + assert query["error"] == ["access_denied"] + assert query["state"] == ["client-state"] + else: + assert redirect.path == "/sso/key/generate" + manager.get_allowed_mcp_servers.assert_not_awaited() + proxy_server.prisma_client.db.litellm_mcpusercredentials.upsert.assert_not_called() + proxy_server.prisma_client.db.litellm_usertable.create.assert_not_called() + proxy_server.prisma_client.db.litellm_teamtable.create.assert_not_called() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py index f141cb2e316..7c5320ed4f4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py @@ -137,6 +137,22 @@ class TestCheckModelAccess: assert result.code == -1 assert "claude-3-opus-20240229" in result.message + @pytest.mark.asyncio + async def test_should_log_internal_denial_reason_and_hide_allowlist_from_client(self, caplog): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_access_denied import model_access_denied_client_message + + auth = UserAPIKeyAuth(api_key="sk-test-key", models=["gpt-3.5-turbo"]) + + with caplog.at_level("WARNING", logger="LiteLLM"): + result = await _check_model_access("gpt-4o\r\nforged", user_api_key_auth=auth) + + assert result is not None + assert result.message == model_access_denied_client_message(model="gpt-4o\r\nforged") + denial_records = [r for r in caplog.records if "gpt-3.5-turbo" in r.getMessage()] + assert len(denial_records) == 1 + assert "Tried to access gpt-4oforged" in denial_records[0].getMessage() + @pytest.mark.asyncio async def test_should_deny_empty_oauth_passthrough_placeholder(self): """Regression: process_mcp_request() returns an empty UserAPIKeyAuth() diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 8c8b755195f..26ae28a57d2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -25,6 +25,7 @@ from litellm.proxy._types import ( LiteLLM_TeamTable, LiteLLM_UserTable, LitellmUserRoles, + ModelAccessDeniedProxyException, ProxyErrorTypes, ProxyException, SSOUserDefinedValues, @@ -530,12 +531,14 @@ async def test_can_team_access_model_error_lists_direct_and_access_group_models( assert await can_team_access_model("direct-model", team_object, None) is True assert await can_team_access_model("group-model", team_object, None) is True - with pytest.raises(ProxyException) as exc_info: + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: await can_team_access_model("blocked-model", team_object, None) assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied - assert "direct-model" in exc_info.value.message - assert "group-model" in exc_info.value.message + assert "direct-model" in exc_info.value.internal_message + assert "group-model" in exc_info.value.internal_message + assert "direct-model" not in exc_info.value.message + assert "group-model" not in exc_info.value.message @pytest.mark.asyncio @@ -1675,10 +1678,128 @@ def test_can_object_call_model_no_access_to_alias_or_underlying(): # Should raise ProxyException with appropriate error type assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied - assert "key not allowed to access model" in str(exc_info.value.message) + assert "is not available for this API key" in str(exc_info.value.message) assert "my-fake-gpt" in str(exc_info.value.message) +_DENIED_MESSAGE_TEMPLATE: Final = ( + "The requested model '{model}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) + + +def test_can_object_call_model_denial_hides_allowlist_and_keeps_detail_on_exception(caplog): + with caplog.at_level("DEBUG", logger="LiteLLM Proxy"): + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: + _can_object_call_model( + model="anthropic-sonnet-4-5", + llm_router=None, + models=["internal-models"], + object_type="key", + ) + + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="anthropic-sonnet-4-5") + assert "internal-models" not in exc_info.value.message + assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied + assert exc_info.value.param == "model" + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + assert exc_info.value.internal_message == ( + "key not allowed to access model. This key can only access models=['internal-models']. " + "Tried to access anthropic-sonnet-4-5" + ) + assert "internal-models" not in caplog.text + + +@pytest.mark.asyncio +async def test_access_group_fallback_grant_does_not_log_a_denial(caplog): + from litellm.proxy.auth.auth_checks import can_team_access_model + + team_object = LiteLLM_TeamTable(team_id="team-123", models=["direct-model"], access_group_ids=["ag-1"]) + + with ( + patch( # test-quality-ok: access-group lookup has no dependency-injection seam + "litellm.proxy.auth.auth_checks._get_models_from_access_groups", + new=AsyncMock(return_value=["group-model"]), + ), + caplog.at_level("DEBUG", logger="LiteLLM Proxy"), + ): + assert await can_team_access_model("group-model", team_object, None) is True + + assert "not allowed to access model" not in caplog.text + + +@pytest.mark.parametrize( + "object_type, expected_type", + [ + ("team", ProxyErrorTypes.team_model_access_denied), + ("user", ProxyErrorTypes.user_model_access_denied), + ("org", ProxyErrorTypes.org_model_access_denied), + ], +) +def test_can_object_call_model_denial_same_client_message_for_every_object_type(object_type, expected_type): + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: + _can_object_call_model( + model="anthropic-sonnet-4-5", + llm_router=None, + models=["internal-models"], + object_type=object_type, + ) + + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="anthropic-sonnet-4-5") + assert exc_info.value.type == expected_type + assert f"{object_type} not allowed to access model" in exc_info.value.internal_message + + +@pytest.mark.asyncio +async def test_can_user_call_model_no_default_models_hides_policy_detail(): + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import can_user_call_model + + user_object = LiteLLM_UserTable(user_id="test-user", models=[SpecialModelNames.no_default_models.value]) + + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: + await can_user_call_model(model="restricted-model", llm_router=None, user_object=user_object) + + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="restricted-model") + assert "only team models allowed" in exc_info.value.internal_message + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + +@pytest.mark.asyncio +async def test_check_team_member_model_access_denied_hides_member_allowlist(): + from litellm.proxy._types import LiteLLM_TeamMembership + from litellm.proxy.auth.auth_checks import _check_team_member_model_access + from litellm.proxy.common_utils.user_api_key_cache import team_membership_reservation_cache_key + + membership = LiteLLM_TeamMembership( + user_id="alice", + team_id="team-a", + litellm_budget_table=LiteLLM_BudgetTable(allowed_models=["fast-models"]), + ) + cache = UserApiKeyCache() + await cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="alice", team_id="team-a"), + value=membership, + model_type=LiteLLM_TeamMembership, + ) + + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: + await _check_team_member_model_access( + model="mock-vision", + team_object=LiteLLM_TeamTable(team_id="team-a"), + valid_token=UserAPIKeyAuth(token="sk-test", user_id="alice", team_id="team-a"), + llm_router=_make_team_scoped_router(), + prisma_client=None, + user_api_key_cache=cache, + proxy_logging_obj=MagicMock(), + ) + + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="mock-vision") + assert "fast-models" not in exc_info.value.message + assert "Allowed member models = ['fast-models']" in exc_info.value.internal_message + assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + + # -- Team-member access-group resolution with team-scoped DB models ----------- diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 6e9770bced8..125b8862dfc 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -29,8 +29,14 @@ from prisma.errors import ( from litellm._logging import verbose_proxy_logger from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError -from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth -from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler +from litellm.proxy._types import ( + ModelAccessDeniedProxyException, + ProxyErrorTypes, + ProxyException, + UserAPIKeyAuth, +) +from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler, _as_proxy_exception +from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException class _EngineHttp500: @@ -982,3 +988,80 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( assert records[0].levelname == expect_level expected_logger_name = "LiteLLM Proxy.stdout" if expect_level == "WARNING" else "LiteLLM Proxy" assert records[0].name == expected_logger_name + + +_DENIED_CLIENT_MESSAGE = ( + "The requested model 'gpt-5.6' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) + + +def _denied_proxy_exception() -> ModelAccessDeniedProxyException: + return ModelAccessDeniedProxyException( + message=_DENIED_CLIENT_MESSAGE, + internal_message="key not allowed to access model. This key can only access models=['internal-models']. " + "Tried to access gpt-5.6\r\nWARNING forged log line", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=status.HTTP_403_FORBIDDEN, + ) + + +def _denied_jwt_exception() -> ModelAccessDeniedHTTPException: + return ModelAccessDeniedHTTPException( + internal_message="Role=engineer not allowed to call model=gpt-5.6\r\nWARNING forged log line. " + "Allowed models=['internal-models']", + status_code=status.HTTP_403_FORBIDDEN, + detail=_DENIED_CLIENT_MESSAGE, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "make_denial", + [ + pytest.param(_denied_proxy_exception, id="proxy_exception"), + pytest.param(_denied_jwt_exception, id="jwt_http_exception"), + ], +) +async def test_handle_authentication_error_keeps_internal_message_on_model_access_denial(make_denial, caplog): + handler = UserAPIKeyAuthExceptionHandler() + denial = make_denial() + + with ( + patch( # test-quality-ok: handler reads proxy_server globals at call time + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( # test-quality-ok: handler reads proxy_server globals at call time + "litellm.proxy.auth.auth_exception_handler.seed_request_identity", + ), + patch( # test-quality-ok: handler reads proxy_server globals at call time + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + caplog.at_level("WARNING", logger="LiteLLM Proxy"), + pytest.raises(ModelAccessDeniedProxyException) as exc_info, + ): + await handler._handle_authentication_error(denial, MagicMock(), {}, "/v1/chat/completions", None, "sk-bad-key") + + assert exc_info.value.code == str(status.HTTP_403_FORBIDDEN) + assert "internal-models" not in str(exc_info.value.message) + assert exc_info.value.internal_message == denial.internal_message + assert [r for r in caplog.records if r.levelname == "WARNING" and "internal-models" in r.getMessage()] == [] + + +def test_as_proxy_exception_keeps_jwt_scope_denial_message_shape(): + detail = {"error": _DENIED_CLIENT_MESSAGE} + denial = ModelAccessDeniedHTTPException( + internal_message="model=gpt-5.6 not allowed. Allowed_models=['internal-models']", + status_code=status.HTTP_403_FORBIDDEN, + detail=detail, + ) + plain = _as_proxy_exception(HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=detail)) + + converted = _as_proxy_exception(denial) + + assert converted.to_dict() == plain.to_dict() + assert converted.internal_message == denial.internal_message diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index df36e220d7e..965acd57bf3 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1159,7 +1159,7 @@ async def test_managed_batch_routes_pass_team_model_access_check(route, request_ is True ) - with pytest.raises(Exception, match="team not allowed to access model"): + with pytest.raises(Exception, match="is not available for this API key"): await can_team_access_model( model=model, team_object=LiteLLM_TeamTable(team_id="team-other", models=["some-other-model"]), diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 814e31535e0..15defb196af 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -2,13 +2,15 @@ import asyncio import re import time from collections.abc import Mapping, Sequence -from typing import Optional +from typing import Final, Optional from unittest.mock import AsyncMock, MagicMock, patch from fastapi import HTTPException import httpx import pytest +import litellm + from litellm.proxy._types import ( DEFAULT_JWKS_STALE_TTL, JWTLiteLLMRoleMap, @@ -21,6 +23,8 @@ from litellm.proxy._types import ( Member, ProxyErrorTypes, ProxyException, + RoleBasedPermissions, + ScopeMapping, ) from litellm.caching.dual_cache import DualCache from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry @@ -33,6 +37,7 @@ from litellm.proxy.auth.handle_jwt import ( JWTHandler, NoMatchingJWTPublicKeyError, ) +from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException from litellm.types.agents import AgentResponse @@ -6790,6 +6795,88 @@ async def test_sync_user_role_and_teams_singular_claim_only_recognized_under_fla assert user.teams == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ["identity", "authorize", "admit"]) +@pytest.mark.parametrize("existing_user", [False, True]) +@pytest.mark.parametrize("model_allowed", [False, True]) +async def test_jwt_identity_and_authorization_keep_provisioning_in_admission( + monkeypatch: pytest.MonkeyPatch, operation: str, existing_user: bool, model_allowed: bool +) -> None: + from litellm.proxy._types import ScopeMapping + from litellm.proxy.auth.auth_checks import UserNotFoundError + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + private_key, jwk = _get_rsa_key_and_jwk("identity-mode") + cache: Final = UserApiKeyCache() + cache.set_cache("litellm_jwt_auth_keys_https://identity.example/jwks", [jwk]) + user_id: Final = f"identity-mode-{operation}-{existing_user}-{model_allowed}" + user: Final = LiteLLM_UserTable(user_id=user_id, organization_memberships=[]) + if existing_user: + cache.set_cache(user_id, user) + database: Final = MagicMock() + users: Final = database.db.litellm_usertable + users.find_unique = AsyncMock(return_value=None) + users.find_first = AsyncMock(return_value=None) + users.create = AsyncMock(return_value=user) + handler: Final = JWTHandler() + handler.update_environment( + prisma_client=database, + user_api_key_cache=cache, + litellm_jwtauth=LiteLLM_JWTAuth( + user_id_jwt_field="sub", + user_id_upsert=True, + enforce_scope_based_access=True, + scope_mappings=[ScopeMapping(scope="allowed", models=["allowed-model"])], + ), + ) + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", "https://identity.example/jwks") + monkeypatch.setenv("JWT_ISSUER", "https://identity.example") + monkeypatch.setenv("JWT_AUDIENCE", "gateway") + token: Final = _encode_rsa_jwt( + private_key, "https://identity.example", "gateway", "identity-mode", {"sub": user_id, "scope": "allowed"} + ) + common: Final = { + "api_key": token, + "jwt_handler": handler, + "prisma_client": database, + "user_api_key_cache": cache, + "parent_otel_span": None, + "proxy_logging_obj": MagicMock(), + } + if operation == "identity": + if not existing_user: + with pytest.raises(UserNotFoundError): + await JWTAuthManager.resolve_identity(**common) + else: + identity: Final = await JWTAuthManager.resolve_identity(**common) + assert identity.user_id == user_id + assert identity.user_object is not None and identity.user_object.user_id == user_id + users.create.assert_not_awaited() + return + authorize: Final = JWTAuthManager.auth_builder if operation == "admit" else JWTAuthManager.authorize_jwt + pending: Final = authorize( + **common, + request_data={"model": "allowed-model" if model_allowed else "forbidden-model"}, + general_settings={}, + route="/mcp/example", + ) + if not model_allowed: + with pytest.raises(HTTPException) as denial: + await pending + assert denial.value.status_code == 403 + users.create.assert_not_awaited() + return + if operation == "authorize" and not existing_user: + with pytest.raises(UserNotFoundError): + await pending + else: + result: Final = await pending + assert result["user_id"] == user_id + assert result["user_object"] is not None + assert result["user_object"].user_id == user_id + assert users.create.await_count == (0 if operation == "authorize" or existing_user else 1) + + def _entra_agent_registry() -> AgentRegistry: registry = AgentRegistry() registry.register_agent( @@ -6916,7 +7003,8 @@ def _entra_signed_app_token(monkeypatch, azp: str, scope: str) -> tuple[JWTHandl @pytest.mark.asyncio @pytest.mark.parametrize("is_admin_token", [False, True], ids=["standard_jwt", "proxy_admin_jwt"]) -async def test_auth_builder_propagates_agent_id_from_jwt_claim(monkeypatch, is_admin_token: bool): +@pytest.mark.parametrize("identity_only", [False, True]) +async def test_auth_builder_propagates_agent_id_from_jwt_claim(monkeypatch, is_admin_token: bool, identity_only: bool): """auth_builder carries the resolved agent id into JWTAuthBuilderResult on both the admin and standard paths.""" jwt_handler, token = _entra_signed_app_token( monkeypatch, @@ -6925,6 +7013,14 @@ async def test_auth_builder_propagates_agent_id_from_jwt_claim(monkeypatch, is_a ) jwt_handler.bind_agent_lookup(_entra_agent_registry()) + if identity_only: + identity = await JWTAuthManager.resolve_identity( + api_key=token, jwt_handler=jwt_handler, prisma_client=None, + user_api_key_cache=None, parent_otel_span=None, proxy_logging_obj=None, + ) + assert identity.agent_id == "canonical-agent-id" + return + result = await JWTAuthManager.auth_builder( api_key=token, jwt_handler=jwt_handler, @@ -6942,7 +7038,8 @@ async def test_auth_builder_propagates_agent_id_from_jwt_claim(monkeypatch, is_a @pytest.mark.asyncio -async def test_auth_builder_denies_jwt_naming_unregistered_agent_before_admin_check(monkeypatch): +@pytest.mark.parametrize("identity_only", [False, True]) +async def test_auth_builder_denies_jwt_naming_unregistered_agent_before_admin_check(monkeypatch, identity_only: bool): """An unknown agent claim is rejected even when the token would otherwise be a proxy admin.""" jwt_handler, token = _entra_signed_app_token( monkeypatch, @@ -6951,6 +7048,14 @@ async def test_auth_builder_denies_jwt_naming_unregistered_agent_before_admin_ch ) jwt_handler.bind_agent_lookup(_entra_agent_registry()) + if identity_only: + with pytest.raises(HTTPException) as denial: + await JWTAuthManager.resolve_identity( + api_key=token, jwt_handler=jwt_handler, prisma_client=None, + user_api_key_cache=None, parent_otel_span=None, proxy_logging_obj=None, + ) + assert denial.value.status_code == 403 + return with pytest.raises(HTTPException) as exc_info: await JWTAuthManager.auth_builder( api_key=token, @@ -6965,3 +7070,77 @@ async def test_auth_builder_denies_jwt_naming_unregistered_agent_before_admin_ch ) assert exc_info.value.status_code == 403 + + +_JWT_DENIED_CLIENT_MESSAGE = ( + "The requested model 'gpt-5.6' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) + + +def test_can_rbac_role_call_model_denial_hides_role_allowlist_from_client(): + general_settings = { + "role_permissions": [ + RoleBasedPermissions(role=LitellmUserRoles.INTERNAL_USER, models=["gpt-5.6-mini"]), + ] + } + + with pytest.raises(ModelAccessDeniedHTTPException) as exc_info: + JWTAuthManager.can_rbac_role_call_model( + rbac_role=LitellmUserRoles.INTERNAL_USER, + general_settings=general_settings, + model="gpt-5.6", + ) + + assert exc_info.value.status_code == 403 + assert exc_info.value.detail == _JWT_DENIED_CLIENT_MESSAGE + assert exc_info.value.internal_message == ( + "Role=internal_user not allowed to call model=gpt-5.6. Allowed models=['gpt-5.6-mini']" + ) + + +def test_check_scope_based_access_denial_hides_scope_allowlist_from_client(): + with pytest.raises(ModelAccessDeniedHTTPException) as exc_info: + JWTAuthManager.check_scope_based_access( + scope_mappings=[ScopeMapping(scope="litellm.api.consumer", models=["gpt-5.6-mini"])], + scopes=["litellm.api.consumer"], + request_data={"model": "gpt-5.6"}, + general_settings={}, + ) + + assert exc_info.value.status_code == 403 + assert exc_info.value.detail == {"error": _JWT_DENIED_CLIENT_MESSAGE} + assert exc_info.value.internal_message == "model=gpt-5.6 not allowed. Allowed_models=['gpt-5.6-mini']" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("admission", [False, True]) +async def test_admin_jwt_team_header_only_provisions_during_admission(monkeypatch, admission: bool): + from litellm.proxy.management_endpoints import team_endpoints + + handler, token = _entra_signed_app_token( + monkeypatch, azp="canonical-agent-id", scope=LiteLLM_JWTAuth().admin_jwt_scope, + ) + handler.bind_agent_lookup(_entra_agent_registry()) + handler.litellm_jwtauth.team_id_upsert = True + handler.litellm_jwtauth.admin_allowed_routes = ["openai_routes"] + database = MagicMock() + database.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + create_team = AsyncMock(return_value=LiteLLM_TeamTable(team_id="new-team").model_dump()) + monkeypatch.setattr(team_endpoints, "new_team", create_team) + resolve = JWTAuthManager.auth_builder if admission else JWTAuthManager.authorize_jwt + + result = await resolve( + api_key=token, jwt_handler=handler, request_data={}, general_settings={}, + route="/chat/completions", prisma_client=database, + user_api_key_cache=handler.user_api_key_cache, parent_otel_span=None, + proxy_logging_obj=MagicMock(), request_headers={"x-litellm-team-id": "new-team"}, + ) + + assert result["is_proxy_admin"] is True + if admission: + create_team.assert_awaited_once() + assert result["team_id"] == "new-team" + else: + create_team.assert_not_awaited() + assert result["team_id"] is None diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py index 130b0da000b..d0ad068aeb4 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py @@ -4,6 +4,7 @@ Tests for the Content Filter Guardrail import json import os +from typing import Final from unittest.mock import MagicMock import pytest @@ -11,6 +12,10 @@ import pytest from fastapi import HTTPException +from litellm.constants import ( + CONTENT_FILTER_STREAMING_HOLDBACK_CHARS, + CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS, +) from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, ) @@ -22,7 +27,9 @@ from litellm.types.guardrails import ( ) from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter import ( ContentFilterCategoryConfig, + ContentFilterDetection, ) +from litellm.types.utils import StandardLoggingGuardrailInformation class TestContentFilterGuardrail: @@ -900,6 +907,341 @@ class TestContentFilterGuardrail: # masked_entity_count for email is the real count, not N×. assert entry["masked_entity_count"].get("email") == 1 + @staticmethod + async def _collect_streamed_text( + guardrail: ContentFilterGuardrail, + chunks: list[str], + metadata: dict[str, list[StandardLoggingGuardrailInformation]], + ) -> str: + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + async def mock_stream(): + for i, content in enumerate(chunks): + yield ModelResponseStream( + id=f"c{i}", + choices=[StreamingChoices(delta=Delta(content=content), index=0)], + model="gpt-4", + ) + yield ModelResponseStream( + id="final", + choices=[ + StreamingChoices( + delta=Delta(content=""), index=0, finish_reason="stop" + ) + ], + model="gpt-4", + ) + + yielded: Final[list[str]] = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=MagicMock(), + response=mock_stream(), + request_data={"messages": [], "model": "gpt-4o", "metadata": metadata}, + ): + yielded.append(chunk.choices[0].delta.content or "") + return "".join(yielded) + + @pytest.mark.asyncio + async def test_streaming_hook_scans_bounded_window_per_chunk(self): + """ + Regression: the streaming hook used to re-scan the whole accumulated + buffer on every chunk, so scan work grew quadratically with the length + of the response. Each scan must now cover only the new chunk plus a + bounded tail of what came before, without dropping any output. + """ + scanned_lengths: Final[list[int]] = [] + + class RecordingGuardrail(ContentFilterGuardrail): + def _filter_single_text( + self, + text: str, + detections: list[ContentFilterDetection] | None = None, + ) -> str: + scanned_lengths.append(len(text)) + return super()._filter_single_text(text, detections=detections) + + guardrail: Final = RecordingGuardrail( + guardrail_name="test-streaming-bounded-scan", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + action=ContentFilterAction.MASK, + ) + ], + event_hook=GuardrailEventHooks.post_call, + ) + chunk: Final = "Item: a plain household object description. " + chunks: Final = [chunk] * 200 + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, {}) + + assert streamed == chunk * 200 + assert len(chunk) * 200 > 4 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS + window_bound: Final = 2 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS + len(chunk) + 1 + assert max(scanned_lengths) <= window_bound, ( + f"scan input grew to {max(scanned_lengths)} chars for a " + f"{len(chunk)}-char chunk; expected at most {window_bound}" + ) + + @pytest.mark.asyncio + async def test_streaming_hook_retries_refused_cut_once_per_context_length(self): + """ + A single URL that keeps growing crosses every proposed cut, so no cut is + ever safe. The trim check must then back off instead of adding two extra + scans on every chunk, and the whole URL must still come out masked. + """ + scanned_lengths: Final[list[int]] = [] + + class RecordingGuardrail(ContentFilterGuardrail): + def _filter_single_text( + self, + text: str, + detections: list[ContentFilterDetection] | None = None, + ) -> str: + scanned_lengths.append(len(text)) + return super()._filter_single_text(text, detections=detections) + + guardrail: Final = RecordingGuardrail( + guardrail_name="test-streaming-refused-cut-backoff", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="url", + action=ContentFilterAction.MASK, + ) + ], + event_hook=GuardrailEventHooks.post_call, + ) + text: Final = "See https://example.com/" + "a" * (8 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS) + " now." + chunks: Final = [text[i : i + 16] for i in range(0, len(text), 16)] + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, {}) + streamed_scans: Final = len(scanned_lengths) + + full_scan: Final = await guardrail.apply_guardrail( + inputs={"texts": [text]}, request_data={}, input_type="response" + ) + assert streamed == full_scan["texts"][0] == "See [URL_REDACTED] now." + extra_scans: Final = streamed_scans - len(chunks) + assert extra_scans <= 2 * (len(text) // CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS), ( + f"{extra_scans} scans beyond one per chunk for {len(chunks)} chunks; the refused cut must back off" + ) + + @pytest.mark.asyncio + async def test_streaming_hook_blocks_match_longer_than_holdback_across_chunks( + self, + ): + """ + A blocked phrase longer than the holdback window arrives in small chunks, + so its start has already been yielded before its end shows up. The scan + still has to see the whole phrase and block. + """ + phrase: Final = "alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima" + assert len(phrase) > CONTENT_FILTER_STREAMING_HOLDBACK_CHARS + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-long-block", + blocked_words=[BlockedWord(keyword=phrase, action=ContentFilterAction.BLOCK)], + event_hook=GuardrailEventHooks.post_call, + ) + text: Final = "Here is the codeword list: " + phrase + " and that is all." + chunks: Final = [text[i : i + 4] for i in range(0, len(text), 4)] + metadata: Final[dict[str, list[StandardLoggingGuardrailInformation]]] = {} + + with pytest.raises(HTTPException) as exc_info: + await self._collect_streamed_text(guardrail, chunks, metadata) + + assert exc_info.value.detail["keyword"] == phrase + entry: Final = metadata["standard_logging_guardrail_information"][0] + assert entry["guardrail_status"] == "guardrail_intervened" + assert [d["keyword"] for d in entry["guardrail_response"]] == [phrase] + + @pytest.mark.asyncio + async def test_streaming_hook_blocks_keyword_longer_than_scan_context(self): + """ + A blocked keyword longer than the default retained context arrives after + enough text that the buffer has already been trimmed at least once. The + retained tail must be wide enough that the keyword's start is still in the + buffer when its end arrives, so the stream is blocked. + """ + phrase: Final = " ".join(f"token{i:03d}" for i in range(80)) + assert len(phrase) > CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-keyword-wider-than-context", + blocked_words=[BlockedWord(keyword=phrase, action=ContentFilterAction.BLOCK)], + event_hook=GuardrailEventHooks.post_call, + ) + filler: Final = "plain filler sentence. " * (3 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS // 23) + text: Final = filler + phrase + " and that is all." + chunks: Final = [text[i : i + 16] for i in range(0, len(text), 16)] + metadata: Final[dict[str, list[StandardLoggingGuardrailInformation]]] = {} + + with pytest.raises(HTTPException) as exc_info: + await self._collect_streamed_text(guardrail, chunks, metadata) + + assert exc_info.value.detail["keyword"] == phrase + entry: Final = metadata["standard_logging_guardrail_information"][0] + assert entry["guardrail_status"] == "guardrail_intervened" + + @pytest.mark.asyncio + async def test_streaming_hook_keeps_early_exception_phrase_suppressing_later_keyword(self): + """ + Category exception phrases suppress category matches anywhere in the + scanned text. An exception phrase at the start of a long response must keep + suppressing a category keyword that arrives long after the buffer would + otherwise have been trimmed, exactly as one scan of the full text does. + """ + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-exception-context", + categories=[{"category": "harmful_self_harm", "enabled": True, "action": "BLOCK"}], + event_hook=GuardrailEventHooks.post_call, + ) + exception_phrase: Final = guardrail.loaded_categories["harmful_self_harm"].exceptions[0] + keyword: Final = next(iter(guardrail.category_keywords)) + filler: Final = "plain filler sentence. " * (3 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS // 23) + text: Final = f"Resources on {exception_phrase} matter. {filler}Someone said {keyword} in a novel." + chunks: Final = [text[i : i + 16] for i in range(0, len(text), 16)] + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, {}) + + full_scan: Final = await guardrail.apply_guardrail( + inputs={"texts": [text]}, request_data={}, input_type="response" + ) + assert streamed == full_scan["texts"][0] == text + + @pytest.mark.asyncio + async def test_streaming_hook_blocks_conditional_pair_split_by_long_sentence(self): + """ + Conditional categories block an identifier word and a block word that + share one sentence. When the sentence runs longer than the retained + context, the identifier at its start must still be in the buffer when the + block word arrives, so the stream is blocked like a scan of the full text. + """ + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-conditional-context", + categories=[{"category": "harmful_child_safety", "enabled": True, "action": "BLOCK"}], + event_hook=GuardrailEventHooks.post_call, + ) + conditional: Final = guardrail.conditional_categories["harmful_child_safety"] + identifier, block_word = conditional["identifier_words"][0], conditional["block_words"][-1] + filler: Final = "and then more plain words " * (3 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS // 26) + text: Final = f"In this chapter the {identifier} {filler}shared an {block_word} moment. The end." + chunks: Final = [text[i : i + 16] for i in range(0, len(text), 16)] + metadata: Final[dict[str, list[StandardLoggingGuardrailInformation]]] = {} + + with pytest.raises(HTTPException): + await guardrail.apply_guardrail(inputs={"texts": [text]}, request_data={}, input_type="response") + with pytest.raises(HTTPException) as exc_info: + await self._collect_streamed_text(guardrail, chunks, metadata) + + assert "harmful_child_safety" in str(exc_info.value.detail) + entry: Final = metadata["standard_logging_guardrail_information"][0] + assert entry["guardrail_status"] == "guardrail_intervened" + + @pytest.mark.asyncio + async def test_streaming_hook_blocks_conditional_identifier_straddling_cut(self): + """ + The buffer is cut at a character offset, so a conditional identifier word + can sit half in the dropped head and half in the retained tail. That cut + must be refused: otherwise the block word arriving later in the same + sentence finds no identifier and the stream passes where a scan of the + full text blocks. + """ + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-conditional-straddle", + categories=[{"category": "harmful_child_safety", "enabled": True, "action": "BLOCK"}], + event_hook=GuardrailEventHooks.post_call, + ) + conditional: Final = guardrail.conditional_categories["harmful_child_safety"] + identifier, block_word = conditional["identifier_words"][0], conditional["block_words"][-1] + chunk_size: Final = 16 + first_cut: Final = ( + 2 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS // chunk_size + 1 + ) * chunk_size - CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS + prefix: Final = ("plain words " * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS)[: first_cut - 2] + filler: Final = "and then more plain words " * (3 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS // 26) + text: Final = f"{prefix}{identifier} {filler}shared an {block_word} moment. The end." + assert text[first_cut - 2 : first_cut - 2 + len(identifier)] == identifier + chunks: Final = [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)] + metadata: Final[dict[str, list[StandardLoggingGuardrailInformation]]] = {} + + with pytest.raises(HTTPException): + await guardrail.apply_guardrail(inputs={"texts": [text]}, request_data={}, input_type="response") + with pytest.raises(HTTPException) as exc_info: + await self._collect_streamed_text(guardrail, chunks, metadata) + + assert "harmful_child_safety" in str(exc_info.value.detail) + entry: Final = metadata["standard_logging_guardrail_information"][0] + assert entry["guardrail_status"] == "guardrail_intervened" + + @pytest.mark.asyncio + async def test_streaming_hook_masks_every_email_in_long_stream_and_logs_once( + self, + ): + """ + A response made of nothing but emails, several times longer than the + rescanned buffer, must come out as nothing but redaction tags, and the log + must carry one email detection, matching what a single scan of the full + text reports. Wherever the buffer is cut, an email sits on the cut, so + dropping text without checking that the cut leaves the masked output + unchanged corrupts the stream. + """ + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-many-emails", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + action=ContentFilterAction.MASK, + ) + ], + event_hook=GuardrailEventHooks.post_call, + ) + emails: Final = [f"user{i:03d}@example.com" for i in range(200)] + text: Final = " ".join(emails) + assert len(text) > 4 * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS + chunks: Final = [text[i : i + 3] for i in range(0, len(text), 3)] + metadata: Final[dict[str, list[StandardLoggingGuardrailInformation]]] = {} + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, metadata) + + assert streamed == " ".join(["[EMAIL_REDACTED]"] * len(emails)) + entry: Final = metadata["standard_logging_guardrail_information"][0] + assert entry["guardrail_status"] == "success" + assert [d["pattern_name"] for d in entry["guardrail_response"]] == ["email"] + assert entry["masked_entity_count"] == {"email": 1} + + @pytest.mark.asyncio + async def test_streaming_hook_logs_detection_masked_long_before_stream_end(self): + """ + An email at the start of a long response is masked and then falls out of + the rescanned buffer well before the stream ends. The final log entry must + still report it, as a scan of the full text would. + """ + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-early-detection", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name="email", + action=ContentFilterAction.MASK, + ) + ], + event_hook=GuardrailEventHooks.post_call, + ) + filler: Final = "filler text " * CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS + text: Final = f"Contact one@example.com for details. {filler}" + chunks: Final = [text[i : i + 40] for i in range(0, len(text), 40)] + metadata: Final[dict[str, list[StandardLoggingGuardrailInformation]]] = {} + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, metadata) + + assert streamed == text.replace("one@example.com", "[EMAIL_REDACTED]") + entry: Final = metadata["standard_logging_guardrail_information"][0] + assert entry["guardrail_status"] == "success" + assert [d["pattern_name"] for d in entry["guardrail_response"]] == ["email"] + assert entry["masked_entity_count"] == {"email": 1} + def test_init_with_plain_dicts(self): """ Test initialization with plain dicts (DB format). diff --git a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py index 8377db57b6e..fc2fb949143 100644 --- a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py @@ -202,6 +202,63 @@ def test_initialize_presidio_forwards_analyze_chunk_size_bytes(): assert initialized[-1].presidio_analyze_chunk_size_bytes == 250_000 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mode, filter_scope, expect_output_scanned", + [ + ("pre_mcp_call", None, False), + (["pre_mcp_call", "post_mcp_call"], None, False), + ({"tags": {"team:mcp": "pre_mcp_call"}, "default": ["pre_mcp_call", "post_mcp_call"]}, None, False), + ({"tags": {"team:mcp": ["pre_mcp_call"]}, "default": "pre_call"}, None, True), + ({"tags": {}}, None, True), + ("pre_mcp_call", "both", True), + ("pre_mcp_call", "output", True), + ("pre_call", None, True), + ], +) +async def test_initialize_presidio_mcp_only_mode_skips_post_call_output_scan(mode, filter_scope, expect_output_scanned): + """Regression: an MCP-only Presidio guardrail used to also scan the LLM + response on post_call, so a blocked MCP tool call that the model repeated in + its answer turned the whole request into an HTTP 400 instead of a 200.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.guardrails import GuardrailEventHooks + from litellm.types.utils import Choices, Message, ModelResponse + + llm_answer = "Call me at 415-555-2671" + litellm_params = { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": mode, + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "mock_redacted_text": {"text": "Call me at ", "items": []}, + "default_on": True, + } + if filter_scope is not None: + litellm_params["presidio_filter_scope"] = filter_scope + + guardrail_handler = InMemoryGuardrailHandler() + result = guardrail_handler.initialize_guardrail( + guardrail={"guardrail_name": "test_presidio_mcp_scope", "litellm_params": litellm_params} + ) + guardrail_id = result["guardrail_id"] + callbacks = [ + guardrail_handler.guardrail_id_to_custom_guardrail[guardrail_id], + *guardrail_handler.guardrail_id_to_sibling_callbacks[guardrail_id], + ] + + request_data = {"metadata": {}} + response = ModelResponse( + choices=[Choices(message=Message(role="assistant", content=llm_answer), index=0, finish_reason="stop")] + ) + for callback in callbacks: + if callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call): + await callback.async_post_call_success_hook( + data=request_data, user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + assert (response.choices[0].message.content != llm_answer) is expect_output_scanned + + @pytest.mark.parametrize( "config_value, expected", [(True, True), (False, False), (None, False)], diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index 1aa9382f3fe..76027d6b7e2 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -416,6 +416,45 @@ class TestRotateVirtualKeyInSecretManager: assert call_kwargs["new_secret_name"] == "test-key-alias-new" assert call_kwargs["new_secret_value"] == "sk-new-key" + @pytest.mark.parametrize("key_alias", ["test-key-alias", None]) + @pytest.mark.asyncio + async def test_rotated_hook_without_request_body_syncs_secret_manager( + self, monkeypatch: pytest.MonkeyPatch, key_alias: str | None + ): + import litellm + from litellm.proxy._types import GenerateKeyResponse, LiteLLM_VerificationToken + from litellm.secret_managers.base_secret_manager import BaseSecretManager + from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem + + mock_secret_manager: Final = MagicMock(spec=BaseSecretManager) + mock_secret_manager.async_rotate_secret = AsyncMock(return_value={"status": "success"}) + monkeypatch.setattr(litellm, "secret_manager_client", mock_secret_manager) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.AWS_SECRET_MANAGER) + monkeypatch.setattr( + litellm, + "_key_management_settings", + KeyManagementSettings(store_virtual_keys=True, prefix_for_stored_virtual_keys="litellm/"), + ) + monkeypatch.setattr(litellm, "store_audit_logs", False) + + existing_key_row: Final = LiteLLM_VerificationToken(token="hashed-old-token", key_alias=key_alias) + response: Final = GenerateKeyResponse(token_id="hashed-new-token", key="sk-new-key", key_alias=key_alias) + + await KeyManagementEventHooks.async_key_rotated_hook( + data=None, + existing_key_row=existing_key_row, + response=response, + user_api_key_dict=MagicMock(), + ) + + expected_secret_name: Final = f"litellm/{key_alias or 'virtual-key-hashed-old-token'}" + mock_secret_manager.async_rotate_secret.assert_awaited_once_with( + current_secret_name=expected_secret_name, + new_secret_name=expected_secret_name, + new_secret_value="sk-new-key", + optional_params=None, + ) + @pytest.mark.asyncio async def test_rotate_virtual_key_when_store_virtual_keys_disabled(self): """Test that rotation is skipped when store_virtual_keys is False.""" @@ -474,6 +513,112 @@ class TestRotateVirtualKeyInSecretManager: mock_secret_manager.async_rotate_secret.assert_not_called() +class TestKeyUpdatedSecretManagerSync: + + @staticmethod + def _configure_secret_manager( + monkeypatch: pytest.MonkeyPatch, stored_value: str | None, store_virtual_keys: bool = True + ) -> MagicMock: + import litellm + from litellm.secret_managers.base_secret_manager import BaseSecretManager + from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem + + mock_secret_manager: Final = MagicMock(spec=BaseSecretManager) + mock_secret_manager.async_read_secret = AsyncMock(return_value=stored_value) + mock_secret_manager.async_rotate_secret = AsyncMock(return_value={"status": "success"}) + monkeypatch.setattr(litellm, "secret_manager_client", mock_secret_manager) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.AWS_SECRET_MANAGER) + monkeypatch.setattr( + litellm, + "_key_management_settings", + KeyManagementSettings(store_virtual_keys=store_virtual_keys, prefix_for_stored_virtual_keys="litellm/"), + ) + monkeypatch.setattr(litellm, "store_audit_logs", False) + return mock_secret_manager + + @pytest.mark.parametrize("existing_alias", ["old-alias", None]) + @pytest.mark.asyncio + async def test_updated_hook_renames_secret_when_alias_changes( + self, monkeypatch: pytest.MonkeyPatch, existing_alias: str | None + ): + from litellm.proxy._types import LiteLLM_VerificationToken, UpdateKeyRequest + + mock_secret_manager: Final = self._configure_secret_manager(monkeypatch, stored_value="sk-stored-key") + existing_key_row: Final = LiteLLM_VerificationToken(token="hashed-token", key_alias=existing_alias) + + await KeyManagementEventHooks.async_key_updated_hook( + data=UpdateKeyRequest(key="hashed-token", key_alias="new-alias"), + existing_key_row=existing_key_row, + response=MagicMock(), + user_api_key_dict=MagicMock(), + ) + + current_secret_name: Final = f"litellm/{existing_alias or 'virtual-key-hashed-token'}" + mock_secret_manager.async_read_secret.assert_awaited_once_with( + secret_name=current_secret_name, optional_params=None + ) + mock_secret_manager.async_rotate_secret.assert_awaited_once_with( + current_secret_name=current_secret_name, + new_secret_name="litellm/new-alias", + new_secret_value="sk-stored-key", + optional_params=None, + ) + + @pytest.mark.parametrize("requested_alias", ["same-alias", None]) + @pytest.mark.asyncio + async def test_updated_hook_leaves_secret_alone_when_alias_unchanged( + self, monkeypatch: pytest.MonkeyPatch, requested_alias: str | None + ): + from litellm.proxy._types import LiteLLM_VerificationToken, UpdateKeyRequest + + mock_secret_manager: Final = self._configure_secret_manager(monkeypatch, stored_value="sk-stored-key") + + await KeyManagementEventHooks.async_key_updated_hook( + data=UpdateKeyRequest(key="hashed-token", key_alias=requested_alias, max_budget=10.0), + existing_key_row=LiteLLM_VerificationToken(token="hashed-token", key_alias="same-alias"), + response=MagicMock(), + user_api_key_dict=MagicMock(), + ) + + mock_secret_manager.async_read_secret.assert_not_awaited() + mock_secret_manager.async_rotate_secret.assert_not_awaited() + + @pytest.mark.asyncio + async def test_updated_hook_skips_rename_when_secret_missing(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy._types import LiteLLM_VerificationToken, UpdateKeyRequest + + mock_secret_manager: Final = self._configure_secret_manager(monkeypatch, stored_value=None) + + await KeyManagementEventHooks.async_key_updated_hook( + data=UpdateKeyRequest(key="hashed-token", key_alias="new-alias"), + existing_key_row=LiteLLM_VerificationToken(token="hashed-token", key_alias="old-alias"), + response=MagicMock(), + user_api_key_dict=MagicMock(), + ) + + mock_secret_manager.async_rotate_secret.assert_not_awaited() + + @pytest.mark.asyncio + async def test_updated_hook_ignores_alias_change_when_store_virtual_keys_disabled( + self, monkeypatch: pytest.MonkeyPatch + ): + from litellm.proxy._types import LiteLLM_VerificationToken, UpdateKeyRequest + + mock_secret_manager: Final = self._configure_secret_manager( + monkeypatch, stored_value="sk-stored-key", store_virtual_keys=False + ) + + await KeyManagementEventHooks.async_key_updated_hook( + data=UpdateKeyRequest(key="hashed-token", key_alias="new-alias"), + existing_key_row=LiteLLM_VerificationToken(token="hashed-token", key_alias="old-alias"), + response=MagicMock(), + user_api_key_dict=MagicMock(), + ) + + mock_secret_manager.async_read_secret.assert_not_awaited() + mock_secret_manager.async_rotate_secret.assert_not_awaited() + + class TestKeyUpdatedAuditLogObjectId: """Tests that /key/update audit logs never store the raw virtual key (issue #31620).""" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 73e6ceabdb6..e0785b002b2 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -28,6 +28,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, _join_url_paths, + anthropic_proxy_route, azure_proxy_route, bedrock_llm_proxy_route, bedrock_proxy_route, @@ -585,6 +586,7 @@ class TestVertexAIPassThroughHandler: "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router", pass_through_router, ) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") endpoint = f"/v1/projects/{test_project}/locations/{test_location}/publishers/google/models/gemini-1.5-flash:generateContent" @@ -4286,6 +4288,329 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: assert "sk-master-1234" not in " ".join(f"{name}:{value}" for name, value in forwarded.items()) +class TestAnthropicPassthroughVirtualKeyLeak: + VKEY = "sk-litellm-victim-key" + PROXY_KEY = "sk-ant-api03-proxy-configured-key" + ENDPOINT = "v1/messages" + + async def _run( + self, + monkeypatch, + headers: list[tuple[bytes, bytes]], + authenticated: UserAPIKeyAuth | None = None, + master_key: str | None = "sk-master-1234", + proxy_api_key: str | None = None, + ) -> tuple[HTTPException | None, dict | None]: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers + from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import ( + PassthroughEndpointRouter, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", master_key) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + if proxy_api_key is None: + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + else: + monkeypatch.setenv("ANTHROPIC_API_KEY", proxy_api_key) + caller: Final = authenticated if authenticated is not None else UserAPIKeyAuth(api_key=self.VKEY) + + async def receive(): + return {"type": "http.request", "body": b"{}", "more_body": False} + + request = Request( + { + "type": "http", + "method": "POST", + "path": f"/anthropic/{self.ENDPOINT}", + "headers": headers, + "query_string": b"", + }, + receive=receive, + ) + + captured: dict = {} + + def fake_create_pass_through_route(**kwargs): + captured.update(kwargs) + return AsyncMock(return_value={"status": "success"}) + + module = "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints" + monkeypatch.setattr(f"{module}.passthrough_endpoint_router", PassthroughEndpointRouter(lambda: None)) + raised: HTTPException | None = None + with ( + mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route), + mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=caller)), + ): + try: + await anthropic_proxy_route( + endpoint=self.ENDPOINT, + request=request, + fastapi_response=Response(), + user_api_key_dict=caller, + ) + except HTTPException as exc: + raised = exc + + if not captured: + return raised, None + upstream: Final = HttpPassThroughEndpointHelpers.forward_headers_from_request( + request_headers=dict(request.headers), + headers=dict(captured["custom_headers"] or {}), + forward_headers=captured.get("_forward_headers", False), + ) + return raised, upstream + + @staticmethod + def _blob(forwarded: dict) -> str: + return " ".join(f"{name}:{value}" for name, value in forwarded.items()) + + @pytest.mark.asyncio + async def test_authorization_bearer_virtual_key_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", f"Bearer {self.VKEY}".encode()), (b"content-type", b"application/json")], + ) + assert forwarded is None, "credential-less request must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + assert "ANTHROPIC_API_KEY" in str(raised.detail) and "use_in_pass_through" in str(raised.detail) + + @pytest.mark.asyncio + async def test_x_api_key_virtual_key_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"x-api-key", self.VKEY.encode()), (b"content-type", b"application/json")], + ) + assert forwarded is None, "a virtual key that authenticated via x-api-key must be stripped, not forwarded" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + async def test_x_litellm_api_key_virtual_key_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"x-litellm-api-key", self.VKEY.encode()), (b"content-type", b"application/json")], + ) + assert forwarded is None, "credential-less request must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + async def test_master_key_in_authorization_is_rejected_not_forwarded(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-master-1234"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key="sk-master-1234", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert forwarded is None, "the master key must never reach Anthropic" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("header", "value"), + [ + pytest.param(b"x-api-key", b"sk-ant-api03-callers-own-key", id="x-api-key"), + pytest.param(b"authorization", b"Bearer sk-ant-api03-callers-own-key", id="authorization"), + ], + ) + async def test_without_a_master_key_the_callers_own_anthropic_key_still_forwards( + self, monkeypatch, header: bytes, value: bytes + ): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", None) + raised, forwarded = await self._run( + monkeypatch, + [(header, value), (b"anthropic-version", b"2023-06-01"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key="sk-ant-api03-callers-own-key", user_role=LitellmUserRoles.INTERNAL_USER), + master_key=None, + ) + assert raised is None, "with no master key the proxy authenticated nothing, so nothing of the caller's is a LiteLLM secret" + assert forwarded is not None + assert forwarded.get(header.decode()) == value.decode() + + @pytest.mark.asyncio + async def test_without_a_master_key_a_custom_auth_credential_is_still_stripped(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", AsyncMock()) + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-custom-auth-token"), (b"anthropic-version", b"2023-06-01")], + authenticated=UserAPIKeyAuth(api_key="sk-custom-auth-token", user_role=LitellmUserRoles.INTERNAL_USER), + master_key=None, + ) + assert raised is not None and raised.status_code == 401 + assert forwarded is None + + @pytest.mark.asyncio + async def test_without_a_master_key_an_oauth2_token_is_still_stripped(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"enable_oauth2_auth": True}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_auth", None) + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer oauth2-access-token"), (b"anthropic-version", b"2023-06-01")], + authenticated=UserAPIKeyAuth(api_key="oauth2-access-token", user_role=LitellmUserRoles.INTERNAL_USER), + master_key=None, + ) + assert raised is not None and raised.status_code == 401 + assert forwarded is None + + @pytest.mark.asyncio + async def test_byo_anthropic_oauth_token_still_forwards_without_virtual_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.VKEY.encode()), + (b"authorization", b"Bearer sk-ant-oat01-caller-oauth-token"), + (b"anthropic-version", b"2023-06-01"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("authorization") == "Bearer sk-ant-oat01-caller-oauth-token" + assert forwarded.get("anthropic-version") == "2023-06-01" + assert "x-litellm-api-key" not in forwarded + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_byo_x_api_key_still_forwards_without_virtual_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.VKEY}".encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key" + assert "authorization" not in forwarded + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_custom_auth_caller_keeps_own_authorization_token(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-ant-oat01-caller-oauth-token"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key=None), + master_key=None, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("authorization") == "Bearer sk-ant-oat01-caller-oauth-token" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "credential_header", + sorted(SpecialHeaders.litellm_credential_header_names() - {"authorization", "x-api-key", "x-litellm-api-key"}), + ) + async def test_every_non_anthropic_credential_header_is_dropped_by_name(self, monkeypatch, credential_header): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.VKEY.encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (credential_header.encode(), b"some-distinct-caller-secret-value"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key" + assert credential_header not in forwarded + assert "x-litellm-api-key" not in forwarded + assert self.VKEY not in self._blob(forwarded) + assert "some-distinct-caller-secret-value" not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_virtual_key_in_operator_configured_header_is_stripped(self, monkeypatch): + with mock.patch.dict( # test-quality-ok: general_settings is the real proxy config surface for litellm_key_header_name; no injection seam exists on this route + "litellm.proxy.proxy_server.general_settings", + {"litellm_key_header_name": "x-company-key"}, + ): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-company-key", f"Bearer {self.VKEY}".encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (b"content-type", b"application/json"), + ], + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == "sk-ant-api03-caller-own-key" + assert "x-company-key" not in forwarded + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_proxy_credential_replaces_virtual_key_sent_as_bearer(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.VKEY}".encode()), + (b"anthropic-version", b"2023-06-01"), + (b"content-type", b"application/json"), + ], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == self.PROXY_KEY + assert "authorization" not in forwarded + assert forwarded.get("anthropic-version") == "2023-06-01" + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_proxy_credential_replaces_virtual_key_sent_as_x_api_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [(b"x-api-key", self.VKEY.encode()), (b"content-type", b"application/json")], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == self.PROXY_KEY + assert self.VKEY not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_proxy_credential_wins_over_callers_own_x_api_key(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"x-litellm-api-key", self.VKEY.encode()), + (b"x-api-key", b"sk-ant-api03-caller-own-key"), + (b"content-type", b"application/json"), + ], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-api-key") == self.PROXY_KEY + assert "sk-ant-api03-caller-own-key" not in self._blob(forwarded) + + @pytest.mark.asyncio + async def test_x_pass_and_hop_by_hop_handling_is_unchanged(self, monkeypatch): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.VKEY}".encode()), + (b"x-pass-anthropic-beta", b"interleaved-thinking-2025-05-14"), + (b"x-pass-authorization", b"Bearer smuggled"), + (b"content-length", b"2"), + (b"host", b"proxy.internal"), + (b"accept-encoding", b"br"), + (b"user-agent", b"curl/8.7.1"), + ], + proxy_api_key=self.PROXY_KEY, + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("anthropic-beta") == "interleaved-thinking-2025-05-14" + assert forwarded.get("user-agent") == "curl/8.7.1" + assert "authorization" not in forwarded + assert "content-length" not in forwarded + assert "host" not in forwarded + assert "accept-encoding" not in forwarded + + class TestVertexPassthroughDefaultLocationOnShortRoutes: PROJECT = "test-project" SHORT_ROUTE = "publishers/google/models/gemini-2.5-flash:generateContent" diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 832435711c6..1cceaf95b09 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +import litellm from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import app @@ -282,6 +283,41 @@ def test_rag_query_returns_response_cost_header(client_internal_user): assert response.headers.get("x-litellm-response-cost") == "3.45e-06" +@pytest.mark.parametrize( + ("upstream_error", "expected_status"), + [ + (litellm.BadRequestError(message="filter andAll needs two clauses", model="kb", llm_provider="bedrock"), 400), + (litellm.NotFoundError(message="Knowledge Base does not exist", model="kb", llm_provider="bedrock"), 404), + (RuntimeError("pipeline blew up"), 500), + ], +) +def test_rag_query_surfaces_upstream_status_code(client_internal_user, upstream_error, expected_status): + """A vector store rejection must reach the caller with its own status code, never a blanket 500.""" + with ( + patch( # test-quality-ok: the handler calls the module-level litellm.aquery directly; no injection seam + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new=AsyncMock(side_effect=upstream_error), + ), + patch("litellm.vector_store_registry", None), # test-quality-ok: proxy module global, no injection seam + patch("litellm.proxy.proxy_server.prisma_client", None), # test-quality-ok: proxy module global, no injection seam + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "bedrock/us.anthropic.claude-sonnet-5", + "messages": [{"role": "user", "content": "How was this document ingested?"}], + "retrieval_config": { + "vector_store_id": "L7INRFMVQT", + "custom_llm_provider": "bedrock", + "retrieval_filter": {"andAll": [{"equals": {"key": "department", "value": "billing"}}]}, + }, + }, + ) + + assert response.status_code == expected_status, response.text + assert str(upstream_error) in response.json()["detail"]["error"] + + def test_rag_query_stream_returns_event_stream(client_internal_user): """ A stream=true /v1/rag/query must return an SSE response. Returning the raw diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 82f2ef097aa..f5c97142dde 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -287,7 +287,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( ) assert response.status_code == 403 - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -611,7 +611,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( ) assert response.status_code == 403 - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -658,7 +658,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( assert response.status_code == 403 assert "team" in response.text.lower() - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -703,7 +703,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( assert response.status_code == 403 assert "project" in response.text.lower() - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -757,7 +757,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope ) assert response.status_code == 403 - assert "Team member not allowed to access model" in response.text + assert "is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -783,7 +783,7 @@ async def test_realtime_transcription_websocket_default_model_checks_key_scope() websocket.close.assert_awaited_once() _, close_kwargs = websocket.close.call_args assert close_kwargs["code"] == 1008 - assert "not allowed to access model" in close_kwargs["reason"] + assert "is not available for this API key" in close_kwargs["reason"] @pytest.mark.asyncio @@ -825,7 +825,7 @@ async def test_realtime_transcription_websocket_default_model_checks_team_scope( websocket.close.assert_awaited_once() _, close_kwargs = websocket.close.call_args assert close_kwargs["code"] == 1008 - assert "not allowed to access model" in close_kwargs["reason"] + assert "is not available for this API key" in close_kwargs["reason"] @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index f90d5daf768..615938f2e33 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -356,29 +356,6 @@ def test_openai_style_cache_write_tokens_are_netted_out(): ) -def test_sub_input_cache_write_price_is_an_extra_saving(): - """A few models price writes below input; there the premium is a real credit. - - Clamping the premium at zero would silently undercount these, so the subtraction - stays signed. ``azure/eu/gpt-4o-2024-11-20`` ships a write price at ~0.5x input. - """ - model = "azure/eu/gpt-4o-2024-11-20" - info = litellm.get_model_info(model=model) - input_cost = info["input_cost_per_token"] - cheap_write = info["cache_creation_input_token_cost"] - assert 0 < cheap_write < input_cost, "fixture drifted: this test needs a model pricing cache writes below input" - - result = compute_savings_spend( - model=model, - custom_llm_provider=None, - compression_saved_tokens=0, - gateway_injected_cache=True, - usage_object=_caching_usage(read=1000, written=4000), - ) - assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write)) - assert result.prompt_caching > 0 - - def test_negative_cache_write_count_clamps_to_zero(): """A malformed negative write count must not be read as a saving.""" input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 099afa57eec..72668dd3528 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from botocore.credentials import Credentials from fastapi import Request +from opentelemetry.trace import INVALID_SPAN, NonRecordingSpan, SpanContext from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers @@ -559,6 +560,7 @@ def _batches_request_mock() -> MagicMock: request_mock.headers = {"Content-Type": "application/json"} request_mock.client = MagicMock() request_mock.client.host = "127.0.0.1" + request_mock.state.parent_otel_span = None return request_mock @@ -2813,7 +2815,7 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data(): litellm.model_group_settings = original_model_group_settings -from typing import Optional +from typing import Final, Optional from fastapi.responses import Response @@ -3536,6 +3538,163 @@ def test_add_litellm_metadata_from_request_headers_explicit_trace_id_beats_trace assert data["litellm_session_id"] == "explicit-trace-id-value" +def _otel_span_with_trace_id(trace_id: int) -> NonRecordingSpan: + return NonRecordingSpan(SpanContext(trace_id=trace_id, span_id=0x00F067AA0BA902B7, is_remote=False)) + + +def _request_mock_without_trace_headers() -> MagicMock: + request_mock: Final = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.path = "/v1/chat/completions" + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + return request_mock + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_defaults_trace_id_to_otel_server_span(): + """With OTel on and a client that sends no trace headers, the request's + litellm_trace_id (and so the spend log session_id) must be the W3C trace-id + of the proxy's server span, so a trace in the OTel backend can be looked up + in the Logs UI and vice versa.""" + otel_trace_id: Final = 0x4BF92F3577B34DA6A3CE929D0E0E4736 + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="hashed-key", parent_otel_span=_otel_span_with_trace_id(otel_trace_id) + ) + + data: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6", "messages": [{"role": "user", "content": "hi"}]}, + request=_request_mock_without_trace_headers(), + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + ) + + assert data["litellm_trace_id"] == format(otel_trace_id, "032x") + assert data["metadata"]["trace_id"] == format(otel_trace_id, "032x") + assert "litellm_session_id" not in data + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_falls_back_to_request_state_otel_span(): + """Custom auth hooks return a UserAPIKeyAuth without parent_otel_span even + though user_api_key_auth already opened the server span on request.state, + so the fallback must read the span from there or custom-auth requests would + keep getting an unrelated session id.""" + otel_trace_id: Final = 0x4BF92F3577B34DA6A3CE929D0E0E4736 + request_mock: Final = _request_mock_without_trace_headers() + request_mock.state.parent_otel_span = _otel_span_with_trace_id(otel_trace_id) + + data: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6"}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", parent_otel_span=None), + proxy_config=MagicMock(), + general_settings={}, + ) + + assert data["litellm_trace_id"] == format(otel_trace_id, "032x") + assert data["metadata"]["trace_id"] == format(otel_trace_id, "032x") + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_otel_span_does_not_override_caller_trace_id(): + """A caller's own trace identity (x-litellm-trace-id header or body + metadata.trace_id) keeps priority over the OTel server span's trace-id.""" + span: Final = _otel_span_with_trace_id(0x4BF92F3577B34DA6A3CE929D0E0E4736) + + header_request: Final = _request_mock_without_trace_headers() + header_request.headers = {"Content-Type": "application/json", "x-litellm-trace-id": "caller-trace"} + from_header: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6"}, + request=header_request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", parent_otel_span=span), + proxy_config=MagicMock(), + general_settings={}, + ) + assert from_header["litellm_trace_id"] == "caller-trace" + assert from_header["metadata"]["trace_id"] == "caller-trace" + + from_body: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6", "metadata": {"trace_id": "body-trace"}}, + request=_request_mock_without_trace_headers(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", parent_otel_span=span), + proxy_config=MagicMock(), + general_settings={}, + ) + assert "litellm_trace_id" not in from_body + assert from_body["metadata"]["trace_id"] == "body-trace" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +async def test_add_litellm_data_to_request_otel_span_does_not_override_body_trace_id_on_litellm_metadata_routes(path): + """On routes that keep LiteLLM state in litellm_metadata, the caller's body + metadata.trace_id is only promoted into litellm_metadata later in the + pipeline, so the OTel fallback must look at the requester metadata too or + it would claim the slot first and the caller's id would be lost.""" + request_mock: Final = _request_mock_without_trace_headers() + request_mock.url.path = path + request_mock.url.__str__.return_value = f"http://localhost{path}" + data: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6", "metadata": {"trace_id": "body-trace"}}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-key", parent_otel_span=_otel_span_with_trace_id(0x4BF92F3577B34DA6A3CE929D0E0E4736) + ), + proxy_config=MagicMock(), + general_settings={}, + ) + assert "litellm_trace_id" not in data + assert data["litellm_metadata"]["trace_id"] == "body-trace" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("empty_trace_id", [None, ""]) +async def test_add_litellm_data_to_request_otel_span_fills_empty_body_trace_id(empty_trace_id): + """A serialized-but-empty litellm_trace_id in the body (null or "") carries + no identity, so it must not block the OTel server span fallback.""" + otel_trace_id: Final = 0x4BF92F3577B34DA6A3CE929D0E0E4736 + data: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6", "litellm_trace_id": empty_trace_id}, + request=_request_mock_without_trace_headers(), + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-key", parent_otel_span=_otel_span_with_trace_id(otel_trace_id) + ), + proxy_config=MagicMock(), + general_settings={}, + ) + assert data["litellm_trace_id"] == format(otel_trace_id, "032x") + assert data["metadata"]["trace_id"] == format(otel_trace_id, "032x") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("parent_otel_span", [None, "invalid_span", "not_a_span", "plain_string"]) +async def test_add_litellm_data_to_request_no_trace_id_without_valid_otel_span(parent_otel_span): + """No OTel span (OTel off), a span with an invalid context, an object that + only quacks like a span, or a value that is not a span at all (custom auth + is typed loosely and can hand back anything) must leave litellm_trace_id + unset, and never fail the request, so downstream keeps generating its own id.""" + span: Final = { + "invalid_span": INVALID_SPAN, + "not_a_span": MagicMock(), + "plain_string": "not-a-span", + }.get(parent_otel_span) + data: Final = await add_litellm_data_to_request( + data={"model": "gpt-5.6"}, + request=_request_mock_without_trace_headers(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", parent_otel_span=span), + proxy_config=MagicMock(), + general_settings={}, + ) + assert "litellm_trace_id" not in data + assert "trace_id" not in data["metadata"] + + def test_add_litellm_metadata_from_request_headers_anthropic_metadata_beats_baggage(): """The existing Anthropic metadata.user_id session_id path must win over a baggage session.id fallback.""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 6f55449abab..d1928b9cd52 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -19,7 +19,7 @@ import fastapi.routing import httpx import pytest import yaml -from fastapi import FastAPI +from fastapi import FastAPI, Request from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient @@ -31,10 +31,17 @@ from litellm.caching.caching import RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.caching.dual_cache import DualCache -from litellm.proxy._types import LitellmUserRoles, TokenCountRequest, UserAPIKeyAuth +from litellm.proxy._types import ( + LitellmUserRoles, + ModelAccessDeniedProxyException, + ProxyErrorTypes, + ProxyException, + TokenCountRequest, + UserAPIKeyAuth, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash -from litellm.proxy.proxy_server import app, initialize +from litellm.proxy.proxy_server import app, initialize, openai_exception_handler from litellm.utils import _invalidate_model_cost_lowercase_map example_embedding_result = { @@ -10085,6 +10092,7 @@ async def _lit6973_drive_realtime_session( backend_logged_failure: bool = False, phase_one_exit: str | None = None, websocket: MagicMock | None = None, + model_access_exception: ProxyException | None = None, ) -> MagicMock: """Drive realtime_websocket_endpoint through one of its reservation-settling exits. @@ -10117,10 +10125,10 @@ async def _lit6973_drive_realtime_session( if backend_logged_failure: logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True - from litellm.proxy._types import ProxyException - model_access_error: Final = ( - ProxyException(message="key cannot access model", type="auth_error", param="model", code=401) + model_access_exception + if model_access_exception is not None + else ProxyException(message="key cannot access model", type="auth_error", param="model", code=401) if phase_one_exit == "model_access" else None ) @@ -10947,6 +10955,74 @@ def test_validate_max_ui_session_budget_empty_restores_default(empty_value): assert _validate_general_settings_ui_litellm_value("max_ui_session_budget", empty_value) == 1.0 +def _model_access_denied_proxy_exception(): + return ModelAccessDeniedProxyException( + message="The requested model 'gpt-5.6\r\nWARNING forged log line' is not available for this API key, " + "or the model name is invalid. Check the models available to you and try again.", + internal_message="key not allowed to access model. This key can only access models=['internal-models']. " + "Tried to access gpt-5.6\r\nWARNING forged log line", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=403, + ) + + +def _http_request_scope(): + return Request({"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": []}) + + +@pytest.mark.asyncio +async def test_openai_exception_handler_logs_sanitized_model_access_denial(caplog): + with caplog.at_level("WARNING", logger="LiteLLM Proxy"): + response = await openai_exception_handler(_http_request_scope(), _model_access_denied_proxy_exception()) + + assert response.status_code == 403 + body = json.loads(response.body) + assert "internal-models" not in body["error"]["message"] + denial_records = [r for r in caplog.records if "internal-models" in r.getMessage()] + assert len(denial_records) == 1 + assert denial_records[0].levelname == "WARNING" + assert "\n" not in denial_records[0].getMessage() + assert "\r" not in denial_records[0].getMessage() + assert "gpt-5.6WARNING forged log line" in denial_records[0].getMessage() + + +@pytest.mark.asyncio +async def test_openai_exception_handler_no_denial_log_for_plain_proxy_exception(caplog): + denial = ProxyException( + message="Authentication Error, Invalid proxy server token passed", + type=ProxyErrorTypes.auth_error, + param="None", + code=401, + ) + + with caplog.at_level("WARNING", logger="LiteLLM Proxy"): + response = await openai_exception_handler(_http_request_scope(), denial) + + assert response.status_code == 401 + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + +@pytest.mark.asyncio +async def test_realtime_model_access_denial_logs_sanitized_internal_message(caplog): + reservation = {"reserved_cost": 0.0, "input_cost": 0.0, "finalized": False, "entries": []} + + with caplog.at_level("WARNING", logger="LiteLLM Proxy"): + ws = await _lit6973_drive_realtime_session( + reservation, + backend_logged_success=False, + phase_one_exit="model_access", + model_access_exception=_model_access_denied_proxy_exception(), + ) + + ws.close.assert_awaited_once() + assert "internal-models" not in ws.close.await_args.kwargs["reason"] + denial_records = [r for r in caplog.records if "internal-models" in r.getMessage()] + assert len(denial_records) == 1 + assert "\n" not in denial_records[0].getMessage() + assert "gpt-5.6WARNING forged log line" in denial_records[0].getMessage() + + def test_general_settings_ui_defaults_unchanged_for_existing_fields(): """The spec-default mechanism added for max_ui_session_budget must not change what clearing the pre-existing fields restores (None for Float/Select, False for Boolean).""" diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 264bcd6fb75..54748efb480 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -11,9 +11,13 @@ aquery carries the completion response with real usage and cost. """ import asyncio +import json +from typing import Final from unittest.mock import patch +import httpx import pytest +import respx import litellm from litellm._internal_context import is_internal_call @@ -259,6 +263,86 @@ async def test_aquery_streaming_bills_sub_call_costs_into_final_event(): assert standard_logging_object["response_cost"] >= 0.003 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("retrieval_config_json", "top_level_filter_json", "expected_filter_json"), + ( + ( + '{"vector_store_id":"vs_test_123","custom_llm_provider":"openai","top_k":50,' + '"retrieval_filter":{"equals":{"key":"tenant","value":"retrieval"}}}', + None, + '{"equals":{"key":"tenant","value":"retrieval"}}', + ), + ( + '{"vector_store_id":"vs_test_123","custom_llm_provider":"openai","top_k":50,' + '"filters":{"equals":{"key":"tenant","value":"alias"}}}', + None, + '{"equals":{"key":"tenant","value":"alias"}}', + ), + ( + '{"vector_store_id":"vs_test_123","custom_llm_provider":"openai","top_k":50}', + '{"equals":{"key":"tenant","value":"top-level"}}', + '{"equals":{"key":"tenant","value":"top-level"}}', + ), + ( + '{"vector_store_id":"vs_test_123","custom_llm_provider":"openai","top_k":50,' + '"retrieval_filter":{"equals":{"key":"tenant","value":"retrieval"}},' + '"filters":{"equals":{"key":"tenant","value":"alias"}}}', + '{"equals":{"key":"tenant","value":"top-level"}}', + '{"equals":{"key":"tenant","value":"retrieval"}}', + ), + ( + '{"vector_store_id":"vs_test_123","custom_llm_provider":"openai","top_k":50}', + None, + None, + ), + ), +) +async def test_aquery_forwards_filters_to_vector_store_search( + retrieval_config_json: str, + top_level_filter_json: str | None, + expected_filter_json: str | None, + monkeypatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + retrieval_config: Final = json.loads(retrieval_config_json) + top_level_filter: Final = json.loads(top_level_filter_json) if top_level_filter_json is not None else None + expected_filter: Final = json.loads(expected_filter_json) if expected_filter_json is not None else None + + with respx.mock(assert_all_called=True) as respx_mock: + search_route: Final = respx_mock.post("https://example.com/v1/vector_stores/vs_test_123/search").mock( + return_value=httpx.Response( + 200, + content='{"object":"vector_store.search_results.page","search_query":"q","data":[]}', + ) + ) + respx_mock.post("https://example.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + content=( + '{"id":"chatcmpl-test","object":"chat.completion","created":1,"model":"gpt-4o-mini",' + '"choices":[{"index":0,"message":{"role":"assistant","content":"answer"},"finish_reason":"stop"}],' + '"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}' + ), + ) + ) + response: Final = await litellm.aquery( + model="openai/gpt-4o-mini", + messages=json.loads('[{"role":"user","content":"most frequent causes of low nicotine"}]'), + retrieval_config=retrieval_config, + filters=top_level_filter, + api_key="sk-test", + api_base="https://example.com/v1", + ) + request_body: Final = json.loads(search_route.calls.last.request.content) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "answer" + assert request_body["query"] == "most frequent causes of low nicotine" + assert request_body.get("filters") == expected_filter + assert request_body["max_num_results"] == 50 + + @pytest.mark.asyncio async def test_aquery_forwards_provider_retrieval_config_and_router_to_search(): """ diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 5d97b0531d6..343fc873fa4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -752,6 +752,49 @@ def test_completed_event_restores_usage_hidden_by_stream_options_none(): assert completed.response.usage.output_tokens == 5 +def _empty_choices_chunk(usage: Usage | None = None) -> ModelResponseStream: + return ModelResponseStream(id=CHAT_COMPLETION_ID, model="claude-haiku-4-5", choices=[], usage=usage) + + +@pytest.mark.asyncio +async def test_leading_empty_choices_chunk_does_not_kill_the_stream(): + """ + Azure leads some streams with a `prompt_filter_results` chunk whose `choices` is empty. + The bridge used to index `choices[0]` on it and die before the first token. + """ + iterator = _build_iterator([_empty_choices_chunk(), _chunk("Hello"), _chunk("!", finish_reason="stop")]) + + events = [event async for event in iterator] + + event_types = [getattr(event, "type", None) for event in events] + assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1 + assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + + +@pytest.mark.asyncio +async def test_trailing_empty_choices_usage_chunk_reaches_response_completed(): + """ + With `stream_options.include_usage` (which the bridge always sets) the last upstream chunk + carries only usage and an empty `choices`. It must not crash the stream, and its usage must + still land on `response.completed`. + """ + usage: Final = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + iterator = _build_iterator([_chunk("Hello"), _chunk("", finish_reason="stop"), _empty_choices_chunk(usage)]) + + events = [event async for event in iterator] + + completed = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert completed.response.usage.input_tokens == 10 + assert completed.response.usage.output_tokens == 5 + + +def test_is_reasoning_end_ignores_empty_choices_chunk(): + assert _build_iterator([])._is_reasoning_end(_empty_choices_chunk()) is False + + def test_object_tool_call_arguments_stream_as_valid_json(): """A provider that sends decoded object arguments must still stream valid JSON. diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 5e0e794d93e..dbf54ec3b9b 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -5,17 +5,20 @@ completion_start_time = end_time.""" import json from datetime import datetime -from typing import Optional +from typing import Final, Optional from unittest.mock import Mock, patch import httpx import pytest +from pydantic_core import PydanticSerializationError +import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.streaming_iterator import ( ResponsesAPIStreamingIterator, SyncResponsesAPIStreamingIterator, + _estimate_usage_from_text, ) from litellm.types.llms.openai import ( ResponseAPIUsage, @@ -31,16 +34,23 @@ def _sse_event(payload: dict) -> bytes: def _mock_config() -> Mock: mock_config = Mock(spec=BaseResponsesAPIConfig) - mock_responses_api_response = Mock(spec=ResponsesAPIResponse) - mock_responses_api_response.id = "resp_ttft" + mock_responses_api_response = ResponsesAPIResponse( + id="resp_ttft", + created_at=0, + status="completed", + model="gpt-4o-mini", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2), + ) def _transform(model, parsed_chunk, logging_obj): evt_type = parsed_chunk.get("type") if evt_type == "response.completed": - completed = Mock(spec=ResponseCompletedEvent) - completed.type = ResponsesAPIStreamEvents.RESPONSE_COMPLETED - completed.response = mock_responses_api_response - return completed + return ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=mock_responses_api_response, + ) stub = Mock() stub.type = evt_type return stub @@ -54,6 +64,8 @@ def _make_iterator( sse_events: list[bytes], logging_obj: LiteLLMLoggingObj, trailing_error: Optional[Exception] = None, + config: Mock | None = None, + request_data: dict | None = None, ) -> ResponsesAPIStreamingIterator: async def aiter_bytes(): for evt in sse_events: @@ -68,10 +80,11 @@ def _make_iterator( return ResponsesAPIStreamingIterator( response=mock_response, model="gpt-4o-mini", - responses_api_provider_config=_mock_config(), + responses_api_provider_config=config or _mock_config(), logging_obj=logging_obj, litellm_metadata={}, custom_llm_provider="openai", + request_data=request_data, ) @@ -329,6 +342,88 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead(): assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params +def _mock_config_with_completed_response(response: ResponsesAPIResponse) -> Mock: + mock_config = Mock(spec=BaseResponsesAPIConfig) + + def _transform(model, parsed_chunk, logging_obj): + evt_type = parsed_chunk.get("type") + if evt_type == "response.completed": + return ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=response, + ) + stub = Mock() + stub.type = evt_type + if "delta" in parsed_chunk: + stub.delta = parsed_chunk.get("delta") + if "item" in parsed_chunk: + stub.item = parsed_chunk.get("item") + return stub + + mock_config.transform_streaming_response.side_effect = _transform + return mock_config + + +def _responses_api_response_without_usage() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_no_usage", + created_at=int(datetime(2025, 1, 1).timestamp()), + status="completed", + model="gpt-4o-mini", + object="response", + output=[], + usage=None, + ) + + +@pytest.mark.asyncio +async def test_completed_event_without_usage_gets_text_estimate(): + """A response.completed event carrying usage: null still bills: the + iterator estimates usage from the request input and generated text.""" + response = _responses_api_response_without_usage() + iterator = _make_iterator( + sse_events=[ + _sse_event({"type": "response.output_text.delta", "delta": "hello world"}), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=_logging_obj_stub(), + config=_mock_config_with_completed_response(response), + request_data={"input": "count these input tokens please"}, + ) + + async for _ in iterator: + pass + + usage = iterator.completed_response.response.usage + assert usage is not None + assert usage.input_tokens > 0 + assert usage.output_tokens > 0 + assert usage.total_tokens == usage.input_tokens + usage.output_tokens + + +@pytest.mark.asyncio +async def test_completed_event_with_usage_is_left_untouched(): + """Provider-reported usage on response.completed wins over the estimate.""" + response = _responses_api_response_with_usage() + iterator = _make_iterator( + sse_events=[ + _sse_event({"type": "response.output_text.delta", "delta": "hello world"}), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=_logging_obj_stub(), + config=_mock_config_with_completed_response(response), + request_data={"input": "count these input tokens please"}, + ) + + async for _ in iterator: + pass + + usage = iterator.completed_response.response.usage + assert usage.input_tokens == 20 + assert usage.output_tokens == 60 + assert usage.total_tokens == 80 + + def _responses_api_response_with_usage() -> ResponsesAPIResponse: return ResponsesAPIResponse( id="resp_lit6427", @@ -628,3 +723,222 @@ async def test_streaming_logging_copy_keeps_client_usage_when_response_fails_val assert isinstance(client_usage, ResponseAPIUsage) assert client_usage.input_tokens == 29 assert client_usage.cost == pytest.approx(0.0001) + + +@pytest.mark.asyncio +async def test_completed_event_without_usage_counts_tool_call_arguments(): + """A function-call-only stream still bills output tokens: streamed + function_call_arguments deltas feed the text estimate.""" + response = _responses_api_response_without_usage() + iterator = _make_iterator( + sse_events=[ + _sse_event( + { + "type": "response.output_item.added", + "item": {"type": "function_call", "name": "get_weather", "call_id": "call_1"}, + } + ), + _sse_event( + { + "type": "response.function_call_arguments.delta", + "delta": '{"location": "San Francisco", "unit": "celsius"}', + } + ), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=_logging_obj_stub(), + config=_mock_config_with_completed_response(response), + request_data={"input": "what is the weather in san francisco"}, + ) + + async for _ in iterator: + pass + + usage = iterator.completed_response.response.usage + assert usage is not None + assert usage.output_tokens > 0 + assert usage.total_tokens == usage.input_tokens + usage.output_tokens + + +@pytest.mark.asyncio +async def test_completed_event_without_usage_counts_multimodal_input_as_messages(): + """Multimodal request input is counted as chat messages, not as a JSON blob: + a huge base64 image must not inflate the estimated input tokens.""" + image_input: Final = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "what is in this image"}, + { + "type": "input_image", + "image_url": "data:image/png;base64," + "A" * 4000, + }, + ], + } + ] + json_count: Final = litellm.token_counter(model="gpt-4o-mini", text=json.dumps(image_input)) + response = _responses_api_response_without_usage() + iterator = _make_iterator( + sse_events=[ + _sse_event({"type": "response.output_text.delta", "delta": "it is a cat"}), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=_logging_obj_stub(), + config=_mock_config_with_completed_response(response), + request_data={"input": image_input}, + ) + + async for _ in iterator: + pass + + usage = iterator.completed_response.response.usage + assert usage is not None + assert usage.input_tokens < json_count / 2 + + +@pytest.mark.asyncio +async def test_completed_event_survives_a_failing_usage_estimate(): + """A malformed request input that makes the message transformer raise must not + break a stream that previously completed: the estimate is best-effort and + falls back to usage None.""" + malformed_input: Final = [{"type": "message", "role": "user", "content": 42}] + with pytest.raises(ValueError, match="Invalid content type"): + _estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world") + + response = _responses_api_response_without_usage() + iterator = _make_iterator( + sse_events=[ + _sse_event({"type": "response.output_text.delta", "delta": "hello world"}), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=_logging_obj_stub(), + config=_mock_config_with_completed_response(response), + request_data={"input": malformed_input}, + ) + + yielded: list = [] + async for chunk in iterator: + yielded.append(chunk) + + assert yielded + assert iterator.completed_response.response.usage is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tool_delta_event_type", + ["response.custom_tool_call_input.delta", "response.mcp_call_arguments.delta"], +) +async def test_completed_event_without_usage_counts_tool_input_deltas(tool_delta_event_type): + """Custom-tool and MCP argument deltas feed the streamed usage fallback the + same way function_call_arguments deltas do.""" + response = _responses_api_response_without_usage() + iterator = _make_iterator( + sse_events=[ + _sse_event({"type": tool_delta_event_type, "delta": '{"query": "weather in sf"}'}), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=_logging_obj_stub(), + config=_mock_config_with_completed_response(response), + request_data={"input": "what is the weather in san francisco"}, + ) + + async for _ in iterator: + pass + + usage = iterator.completed_response.response.usage + assert usage is not None + assert usage.output_tokens > 0 + assert usage.total_tokens == usage.input_tokens + usage.output_tokens + + +@pytest.mark.asyncio +async def test_completed_event_with_a_dict_response_is_typed_and_billed(): + """transform_streaming_response can model_construct a terminal event whose + response stays a plain dict; the iterator must type it so the estimated + usage reaches the cost stamping path.""" + dict_response: Final = { + "id": "resp_dict", + "model": "gpt-4o-mini", + "object": "response", + "output": [], + "usage": None, + } + + def _transform(model, parsed_chunk, logging_obj): + if parsed_chunk.get("type") == "response.completed": + return ResponseCompletedEvent.model_construct(type="response.completed", response=dict_response) + stub: Final = Mock() + stub.type = parsed_chunk.get("type") + if "delta" in parsed_chunk: + stub.delta = parsed_chunk.get("delta") + return stub + + config: Final = Mock(spec=BaseResponsesAPIConfig) + config.transform_streaming_response.side_effect = _transform + logging_obj: Final = _logging_obj_stub() + logging_obj._response_cost_calculator.return_value = 0.000704 + iterator: Final = _make_iterator( + sse_events=[ + _sse_event({"type": "response.output_text.delta", "delta": "hello world"}), + _sse_event({"type": "response.completed", "response": {}}), + ], + logging_obj=logging_obj, + config=config, + request_data={"input": "count these input tokens please"}, + ) + + yielded: Final = [chunk async for chunk in iterator] + + terminal_event: Final = iterator.completed_response + assert yielded[-1] is terminal_event + completed_response: Final = terminal_event.response + assert isinstance(completed_response, ResponsesAPIResponse) + usage: Final = completed_response.usage + assert usage is not None + assert usage.input_tokens > 0 + assert usage.output_tokens > 0 + assert usage.cost == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_any_call(result=completed_response) + + +def test_billed_terminal_response_keeps_a_response_that_already_has_usage(): + from litellm.responses.streaming_iterator import _billed_terminal_response + + response: Final = _responses_api_response_with_usage() + + assert _billed_terminal_response(response, None) is response + + +def test_billed_terminal_response_copies_when_estimating_and_leaves_the_original_untouched(): + from litellm.responses.streaming_iterator import _billed_terminal_response + + response: Final = _responses_api_response_without_usage() + estimated: Final = ResponseAPIUsage(input_tokens=3, output_tokens=4, total_tokens=7) + + billed: Final = _billed_terminal_response(response, lambda: estimated) + + assert billed is not response + assert billed.usage is estimated + assert response.usage is None + + +def test_persist_completed_response_to_cache_survives_an_unserializable_response(monkeypatch): + bad_response: Final = ResponsesAPIResponse.model_construct(id="r", output=[object()], usage=None) + with pytest.raises(PydanticSerializationError): + bad_response.model_dump_json() + + logging_obj: Final = _logging_obj_stub() + caching_handler: Final = Mock() + caching_handler.request_kwargs = {"stream": True} + logging_obj._llm_caching_handler = caching_handler + iterator: Final = _make_iterator(sse_events=[], logging_obj=logging_obj) + iterator.completed_response = ResponseCompletedEvent.model_construct( + type="response.completed", response=bad_response + ) + cache: Final = Mock() + monkeypatch.setattr(litellm, "cache", cache) + + iterator._persist_completed_response_to_cache(is_async=False) + + cache.add_cache.assert_not_called() diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index 8d83f4ca8a6..6f963cec6cc 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -283,8 +283,6 @@ async def exercise_async_concurrency(native: object, api_base: str) -> None: def exercise_routes(native_path: Path, api_base: str) -> object: native: Final = load_native(native_path) - if hasattr(native, "_trace"): - raise AssertionError("release wheel exposed trace-parity diagnostics") exercise_sync(native, api_base) asyncio.run(exercise_async(native, api_base)) asyncio.run(exercise_async_concurrency(native, api_base)) diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index b427a1a3bd8..af0e932400f 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -49,6 +49,20 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] @pytest.mark.parametrize( "category,changed,expected", [ + ("provider-harness", ["tests/e2e/provider_cache.py"], "run"), + ("provider-harness", ["tests/e2e/conftest.py"], "run"), + ("provider-harness", ["tests/e2e/e2e_http.py"], "run"), + ("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"), + ("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"), + ("provider-harness", [".circleci/config.yml"], "run"), + ("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"), + ("provider-harness", ["pyproject.toml"], "run"), + ("provider-harness", ["uv.lock"], "run"), + ("provider-harness", ["tests/e2e/PROVIDER_CACHE.md"], "skip"), + ("provider-harness", ["tests/e2e/ui/test_example.py"], "skip"), + ("provider-harness", ["tests/e2e/quota_management/test_quota.py"], "skip"), + ("provider-harness", ["litellm/main.py"], "skip"), + ("provider-harness", ["ui/litellm-dashboard/src/App.tsx"], "skip"), # docs-only: skip everything ("backend", DOCS, "skip"), ("client", DOCS, "skip"), diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7b53d3a58df..a5ed7175649 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1616,73 +1616,6 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map): ) -AZURE_GPT_5_6_MAP_KEYS = ( - "azure/gpt-5.6", - "azure/gpt-5.6-sol", - "azure/gpt-5.6-terra", - "azure/gpt-5.6-luna", - "azure/us/gpt-5.6", - "azure/us/gpt-5.6-sol", - "azure/us/gpt-5.6-terra", - "azure/us/gpt-5.6-luna", - "azure/eu/gpt-5.6", - "azure/eu/gpt-5.6-sol", - "azure/eu/gpt-5.6-terra", - "azure/eu/gpt-5.6-luna", -) - - -def test_azure_gpt_5_6_cache_write_tokens_are_billed(_local_model_cost_map): - """ - Azure bills gpt-5.6 prompt cache writes at 1.25x the input rate on every - tier, but the azure entries carried no ``cache_creation_input_token_cost``, - so cache-write tokens were billed at the plain input rate instead. - """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - usage = Usage( - completion_tokens=100, - prompt_tokens=2000, - total_tokens=2100, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, text_tokens=687), - cache_creation_input_tokens=1313, - ) - - input_cost, output_cost = generic_cost_per_token( - model="azure/gpt-5.6-luna", usage=usage, custom_llm_provider="azure" - ) - - assert input_cost == pytest.approx(687 * 2e-07 + 1313 * 2.5e-07) - assert output_cost == pytest.approx(100 * 1.2e-06) - - -@pytest.mark.parametrize("model", AZURE_GPT_5_6_MAP_KEYS) -def test_azure_gpt_5_6_rates_match_azure_price_page(_local_model_cost_map, model): - """ - Per the Azure OpenAI price page (rendered 2026-08-26): cache writes cost - 1.25x input on every gpt-5.6 tier, and Data Zone costs 1.1x Global for - standard and priority alike (us/eu priority rates previously sat at 1.25x). - """ - entry = litellm.model_cost[model] - input_keys = [key for key in entry if key.startswith("input_cost_per_token")] - assert input_keys - for key in input_keys: - suffix = key[len("input_cost_per_token") :] - assert entry["cache_creation_input_token_cost" + suffix] == pytest.approx(entry[key] * 1.25) - - zone = model.split("/")[1] - if zone in ("us", "eu"): - global_entry = litellm.model_cost["azure/" + model.split("/", 2)[2]] - prefixes = ("input_cost_per_token", "output_cost_per_token", "cache_read", "cache_creation") - token_cost_keys = [key for key in entry if key.startswith(prefixes)] - global_token_cost_keys = [key for key in global_entry if key.startswith(prefixes)] - assert len(token_cost_keys) >= 9 - assert sorted(token_cost_keys) == sorted(global_token_cost_keys) - for key in token_cost_keys: - assert entry[key] == pytest.approx(global_entry[key] * 1.1), key - - def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch): """ Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex diff --git a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py index 10d1d6fecd1..250b587aaf1 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py +++ b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py @@ -3,14 +3,7 @@ from pathlib import Path import pytest -import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, -) REPO_ROOT = Path(__file__).parents[2] MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -21,94 +14,12 @@ GEMINI = "gemini/gemini-3.1-flash-lite-image" VERTEX = "vertex_ai/gemini-3.1-flash-lite-image" ALL_KEYS = (UNPREFIXED, GEMINI, VERTEX) -INPUT_COST = 2.5e-07 -INPUT_COST_BATCHES = 1.25e-07 -OUTPUT_TEXT_COST = 1.5e-06 -OUTPUT_TEXT_COST_BATCHES = 7.5e-07 -OUTPUT_IMAGE_TOKEN_COST = 3e-05 -OUTPUT_COST_PER_1K_IMAGE = 0.0336 -INPUT_COST_PER_IMAGE = 0.00028 -CACHE_READ_COST = 2.5e-08 -MAX_INPUT_TOKENS = 65536 -MAX_OUTPUT_TOKENS = 4096 -TOKENS_PER_1K_IMAGE = 1120 - -SHARED_FIELDS = { - "mode": "image_generation", - "input_cost_per_token": INPUT_COST, - "input_cost_per_token_batches": INPUT_COST_BATCHES, - "input_cost_per_image": INPUT_COST_PER_IMAGE, - "output_cost_per_token": OUTPUT_TEXT_COST, - "output_cost_per_token_batches": OUTPUT_TEXT_COST_BATCHES, - "output_cost_per_image": OUTPUT_COST_PER_1K_IMAGE, - "output_cost_per_image_token": OUTPUT_IMAGE_TOKEN_COST, - "max_input_tokens": MAX_INPUT_TOKENS, - "max_output_tokens": MAX_OUTPUT_TOKENS, - "max_tokens": MAX_OUTPUT_TOKENS, - "supported_endpoints": ["/v1/chat/completions", "/v1/completions", "/v1/batch"], - "supported_output_modalities": ["text", "image"], - "supports_reasoning": False, - "supports_response_schema": False, - "supports_system_messages": True, - "supports_vision": True, -} - -VERTEX_ROUTE_FIELDS = { - "litellm_provider": "vertex_ai-language-models", - "cache_read_input_token_cost": CACHE_READ_COST, - "supported_modalities": ["text", "image", "video"], - "supports_function_calling": False, - "supports_pdf_input": True, - "supports_prompt_caching": True, - "supports_video_input": True, -} - -PER_ROUTE_FIELDS = { - UNPREFIXED: VERTEX_ROUTE_FIELDS, - VERTEX: VERTEX_ROUTE_FIELDS, - GEMINI: { - "litellm_provider": "gemini", - "supported_modalities": ["text", "image"], - "supports_function_calling": True, - "supports_prompt_caching": False, - "rpm": 1000, - "tpm": 4000000, - }, -} - -GROUNDING_FIELDS = ( - "supports_web_search", - "search_context_cost_per_query", - "web_search_billing_unit", -) - def _load(path: Path) -> dict: with open(path, encoding="utf-8") as f: return json.load(f) -@pytest.fixture -def local_model_cost_map(monkeypatch): - original_model_cost = litellm.model_cost - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.get_model_info.cache_clear() - try: - yield - finally: - litellm.model_cost = original_model_cost - litellm.get_model_info.cache_clear() - - -@pytest.mark.parametrize("model", ALL_KEYS) -@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) -def test_per_route_capabilities_match_model_cards(model: str, path: Path): - info = _load(path)[model] - for field, value in PER_ROUTE_FIELDS[model].items(): - assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" - - @pytest.mark.parametrize("model", ALL_KEYS) def test_backup_matches_main(model: str): assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model) @@ -124,18 +35,3 @@ def test_vertex_prefix_routes_to_vertex(): routed_model, provider, _, _ = get_llm_provider(model=VERTEX) assert routed_model == UNPREFIXED assert provider == "vertex_ai" - - -def _one_k_image_response() -> ImageResponse: - return ImageResponse( - data=[ImageObject(b64_json="img1")], - usage=ImageUsage( - input_tokens=50 + TOKENS_PER_1K_IMAGE, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=50, - image_tokens=TOKENS_PER_1K_IMAGE, - ), - output_tokens=TOKENS_PER_1K_IMAGE, - total_tokens=50 + TOKENS_PER_1K_IMAGE + TOKENS_PER_1K_IMAGE, - ), - ) diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 3dccb2b35bf..7fcdc8473d7 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -4,6 +4,7 @@ from datetime import datetime import contextlib import copy import json +import logging import os from collections.abc import Mapping from dataclasses import dataclass @@ -3850,3 +3851,27 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_ assert "extra_headers" not in body assert body["model"] == "gpt-5.4" assert {k: request.headers[k] for k in FORWARDED_CLIENT_HEADERS} == FORWARDED_CLIENT_HEADERS + + +@pytest.mark.parametrize("http2_on", [True, False]) +def test_aiohttp_openai_warns_only_when_http2_enabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool +): + from litellm.main import base_llm_aiohttp_handler + + monkeypatch.setattr(litellm, "http2", http2_on) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + + handler_completion: Final = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(base_llm_aiohttp_handler, "completion", handler_completion) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + litellm.completion( + model="aiohttp_openai/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-test", + ) + + assert handler_completion.called + warned: Final = "aiohttp_openai/ always uses aiohttp" in caplog.text + assert warned is http2_on diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index fb42ab6c893..1e6636ec3d6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -16424,7 +16424,7 @@ class TestMemberAutoRouterInference: project_id="router-project", team_id="router-team", models=["restricted-model"], ), model_type=LiteLLM_ProjectTableCachedObj, ) - with pytest.raises(ProxyException, match="not allowed to access model"): + with pytest.raises(ProxyException, match="is not available for this API key"): await self._route(self._router(), self._request(actor=self.actor.model_copy(update={ "models": ["member-router"] if ceiling == "key" else self.actor.models, "project_id": "router-project" if ceiling == "project" else None, @@ -16453,7 +16453,7 @@ class TestMemberAutoRouterInference: assert self.database.db.litellm_accessgrouptable.find_unique.await_count == 1 self.database.db.litellm_accessgrouptable.find_unique.return_value = group.model_copy(update={"access_model_names": []}) await evict_and_broadcast(cache_keys=("access_group_id:router-group",), user_api_key_cache=self.cache) - with pytest.raises(ProxyException, match="not allowed to access model"): + with pytest.raises(ProxyException, match="is not available for this API key"): await self._route(router, request) assert self.database.db.litellm_accessgrouptable.find_unique.await_count == 2 @@ -16471,7 +16471,7 @@ class TestMemberAutoRouterInference: key="team_id:router-team", model_type=LiteLLM_TeamTable, value=self.team.model_copy(update={"models": ["member-router"]}), ) - with pytest.raises(ProxyException, match="not allowed to access model"): + with pytest.raises(ProxyException, match="is not available for this API key"): await self._route(router, self._request()) self.database.db.litellm_teamtable.find_unique.reset_mock() admin: Final = self._request(tag="admin") diff --git a/tests/test_litellm/test_router_silent_experiment.py b/tests/test_litellm/test_router_silent_experiment.py index bfdf39bad71..d62962da275 100644 --- a/tests/test_litellm/test_router_silent_experiment.py +++ b/tests/test_litellm/test_router_silent_experiment.py @@ -1,11 +1,73 @@ import asyncio import time +from collections.abc import Callable, Mapping +from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.router import Router +from litellm.router import _silent_experiment_kwargs_snapshot +from litellm.router import _silent_experiment_targets + + +class _RecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.success_kwargs: list[dict[str, object]] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.success_kwargs.append(kwargs) + + def shadow_successes(self) -> list[dict[str, object]]: + return [ + call + for call in self.success_kwargs + if call.get("litellm_params", {}).get("metadata", {}).get("is_silent_experiment") is True + ] + + +@pytest.fixture +def recording_logger(): + original_callbacks: Final = litellm.callbacks + logger: Final = _RecordingLogger() + litellm.callbacks = [logger] + try: + yield logger + finally: + litellm.callbacks = original_callbacks + + +async def _wait_for_shadow_successes(logger: _RecordingLogger, expected: int, timeout: float = 5.0) -> None: + deadline: Final = time.monotonic() + timeout + while len(logger.shadow_successes()) < expected and time.monotonic() < deadline: + await asyncio.sleep(0.05) + + +def _wait_for_shadow_successes_sync(logger: _RecordingLogger, expected: int, timeout: float = 5.0) -> None: + deadline: Final = time.monotonic() + timeout + while len(logger.shadow_successes()) < expected and time.monotonic() < deadline: + time.sleep(0.05) + + +def _streaming_model_list(silent_model: object) -> list[dict[str, object]]: + return [ + { + "model_name": "primary-model", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake-key", "silent_model": silent_model}, + }, + { + "model_name": "shadow-a", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-key", "silent_model": "shadow-b"}, + }, + { + "model_name": "shadow-b", + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "fake-key"}, + }, + ] class _NonCopyableSpan: @@ -65,8 +127,7 @@ def test_get_silent_experiment_kwargs(): assert result["metadata"]["is_silent_experiment"] is True assert result["metadata"]["foo"] == "bar" assert "litellm_call_id" not in result - # stream must be forced to False so callbacks fire in background - assert result["stream"] is False + assert result["stream"] is True # proxy_server_request must be preserved for spend log metadata assert "proxy_server_request" in result # CRITICAL: metadata must be a DIFFERENT dict object than the original, @@ -86,6 +147,247 @@ def test_get_silent_experiment_kwargs(): assert result["metadata"]["user_api_key_auth"] is mock_auth +def test_get_silent_experiment_kwargs_without_stream_stays_non_streaming(): + router = Router(model_list=[{"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "k"}}]) + result = router._get_silent_experiment_kwargs(metadata={"foo": "bar"}, stream=False) + assert result["stream"] is False + assert "stream" not in router._get_silent_experiment_kwargs(metadata={"foo": "bar"}) + + +@pytest.mark.parametrize( + "silent_model, expected", + [ + ("shadow-a", ("shadow-a",)), + (["shadow-a", "shadow-b"], ("shadow-a", "shadow-b")), + ([], ()), + (None, ()), + (42, ()), + (["shadow-a", 42], ()), + ], +) +def test_silent_experiment_targets(silent_model, expected): + assert _silent_experiment_targets(silent_model) == expected + + +@pytest.mark.asyncio +async def test_streaming_shadow_is_streamed_and_drained_async(recording_logger): + router = Router(model_list=_streaming_model_list("shadow-a")) + response = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + stream_options={"include_usage": True}, + mock_response="pong", + metadata={"foo": "bar"}, + ) + chunks = [chunk async for chunk in response] + assert chunks + await _wait_for_shadow_successes(recording_logger, expected=1) + + shadow_successes = recording_logger.shadow_successes() + assert len(shadow_successes) == 1 + shadow = shadow_successes[0] + assert shadow["stream"] is True + assert shadow["stream_options"] == {"include_usage": True} + assert shadow["litellm_params"]["metadata"]["model_group"] == "shadow-a" + assert shadow["async_complete_streaming_response"] is not None + + +def test_streaming_shadow_is_streamed_and_drained_sync(recording_logger): + router = Router(model_list=_streaming_model_list("shadow-a")) + response = router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + mock_response="pong", + metadata={"foo": "bar"}, + ) + chunks = list(response) + assert chunks + _wait_for_shadow_successes_sync(recording_logger, expected=1) + + shadow_successes = recording_logger.shadow_successes() + assert len(shadow_successes) == 1 + assert shadow_successes[0]["stream"] is True + assert shadow_successes[0]["litellm_params"]["metadata"]["model_group"] == "shadow-a" + assert shadow_successes[0]["async_complete_streaming_response"] is not None + + +@pytest.mark.asyncio +async def test_multiple_shadow_targets_fan_out_async(recording_logger): + router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + metadata = {"foo": "bar"} + response = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + mock_response="pong", + metadata=metadata, + ) + assert [chunk async for chunk in response] + await _wait_for_shadow_successes(recording_logger, expected=2) + + shadow_successes = recording_logger.shadow_successes() + model_groups = sorted(call["litellm_params"]["metadata"]["model_group"] for call in shadow_successes) + assert model_groups == ["shadow-a", "shadow-b"] + shadow_metadatas = [call["litellm_params"]["metadata"] for call in shadow_successes] + assert shadow_metadatas[0] is not shadow_metadatas[1] + assert all(call["stream"] is True for call in shadow_successes) + assert "is_silent_experiment" not in metadata + assert metadata.get("model_group") != "shadow-a" + primary_successes = [call for call in recording_logger.success_kwargs if call not in shadow_successes] + assert len(primary_successes) == 1 + assert primary_successes[0]["litellm_params"]["metadata"]["model_group"] == "primary-model" + + +def test_multiple_shadow_targets_fan_out_sync(recording_logger): + router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + response = router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"foo": "bar"}, + ) + assert response.choices[0].message.content == "pong" + _wait_for_shadow_successes_sync(recording_logger, expected=2) + + shadow_successes = recording_logger.shadow_successes() + model_groups = sorted(call["litellm_params"]["metadata"]["model_group"] for call in shadow_successes) + assert model_groups == ["shadow-a", "shadow-b"] + assert all(call["stream"] is False for call in shadow_successes) + + +def _tagged_primary_model_list() -> list[dict[str, object]]: + return [ + { + "model_name": "primary-model", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "fake-key", + "silent_model": "shadow-b", + "tags": ["primary-only"], + }, + }, + { + "model_name": "shadow-b", + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "fake-key"}, + }, + ] + + +def test_silent_experiment_kwargs_snapshot_is_isolated_from_later_primary_mutations(): + metadata = {"foo": "bar"} + kwargs: dict[str, object] = {"metadata": metadata, "stream": True} + snapshot = _silent_experiment_kwargs_snapshot(kwargs) + kwargs["messages"] = [{"role": "user", "content": "added by the primary"}] + metadata["tags"] = ["primary-only"] + + assert dict(snapshot) == {"metadata": {"foo": "bar"}, "stream": True} + assert dict(_silent_experiment_kwargs_snapshot({"stream": False, "metadata": None})) == { + "stream": False, + "metadata": None, + } + + +def test_sync_shadow_gets_kwargs_snapshot_taken_before_primary_mutates_them(recording_logger): + deferred: list[Callable[[], None]] = [] + + class _DeferredThread: + def __init__(self, target, args, kwargs, daemon) -> None: + deferred.append(lambda: target(*args, **kwargs)) + + def start(self) -> None: + return None + + router = Router(model_list=_tagged_primary_model_list()) + with patch( # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race + "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread) + ): + response = router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"foo": "bar"}, + ) + assert response.choices[0].message.content == "pong" + assert len(deferred) == 1 + deferred[0]() + _wait_for_shadow_successes_sync(recording_logger, expected=1) + + shadow_successes = recording_logger.shadow_successes() + assert len(shadow_successes) == 1 + shadow_metadata = shadow_successes[0]["litellm_params"]["metadata"] + assert shadow_metadata["model_group"] == "shadow-b" + assert "primary-only" not in shadow_metadata.get("tags", []) + + +def test_sync_shadow_workers_do_not_share_metadata_with_each_other(recording_logger): + workers: list[tuple[Mapping[str, object], Callable[[], None]]] = [] + + class _DeferredThread: + def __init__(self, target, args, kwargs, daemon) -> None: + workers.append((kwargs, lambda: target(*args, **kwargs))) + + def start(self) -> None: + return None + + router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) + with patch( # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race + "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread) + ): + router.completion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"foo": "bar"}, + ) + assert len(workers) == 2 + (first_kwargs, run_first), (_, run_second) = workers + first_kwargs["metadata"].pop("foo") + run_second() + run_first() + _wait_for_shadow_successes_sync(recording_logger, expected=2) + + metadata_by_group = { + call["litellm_params"]["metadata"]["model_group"]: call["litellm_params"]["metadata"] + for call in recording_logger.shadow_successes() + } + assert metadata_by_group["shadow-b"]["foo"] == "bar" + assert "foo" not in metadata_by_group["shadow-a"] + + +@pytest.mark.asyncio +async def test_async_shadow_does_not_inherit_primary_deployment_tags(recording_logger): + router = Router(model_list=_tagged_primary_model_list()) + response = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata={"foo": "bar"}, + ) + assert response.choices[0].message.content == "pong" + await _wait_for_shadow_successes(recording_logger, expected=1) + + shadow_successes = recording_logger.shadow_successes() + assert len(shadow_successes) == 1 + assert "primary-only" not in shadow_successes[0]["litellm_params"]["metadata"].get("tags", []) + + +@pytest.mark.asyncio +async def test_shadow_of_a_shadow_is_not_launched(recording_logger): + router = Router(model_list=_streaming_model_list(["shadow-a"])) + response = await router.acompletion( + model="primary-model", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + ) + assert response.choices[0].message.content == "pong" + await _wait_for_shadow_successes(recording_logger, expected=2, timeout=1.0) + + model_groups = [call["litellm_params"]["metadata"]["model_group"] for call in recording_logger.shadow_successes()] + assert model_groups == ["shadow-a"] + + def test_silent_experiment_completion_direct(): """ Test _silent_experiment_completion directly (for router code coverage). @@ -127,6 +429,25 @@ async def test_silent_experiment_acompletion_direct(): ) +@pytest.mark.asyncio +async def test_run_silent_experiment_drains_stream_so_callbacks_fire(recording_logger): + router = Router(model_list=_streaming_model_list(None)) + silent_kwargs: Final = { + "stream": True, + "stream_options": {"include_usage": True}, + "mock_response": "pong", + "metadata": {"is_silent_experiment": True, "model_group": "shadow-b"}, + } + await router._run_silent_experiment("shadow-b", [{"role": "user", "content": "hi"}], silent_kwargs) + await _wait_for_shadow_successes(recording_logger, expected=1) + + shadow_successes = recording_logger.shadow_successes() + assert len(shadow_successes) == 1 + assert shadow_successes[0]["stream"] is True + assert shadow_successes[0]["async_complete_streaming_response"] is not None + assert silent_kwargs["stream"] is True + + @pytest.mark.asyncio async def test_router_silent_experiment_acompletion(): """ diff --git a/tests/test_litellm/test_xai_responses_auto_routing.py b/tests/test_litellm/test_xai_responses_auto_routing.py index fbf2453d7fb..d405ea1e6c6 100644 --- a/tests/test_litellm/test_xai_responses_auto_routing.py +++ b/tests/test_litellm/test_xai_responses_auto_routing.py @@ -2,14 +2,30 @@ Test automatic routing to xAI Responses API when tools are present """ +import json +from collections.abc import Mapping +from typing import Final from unittest.mock import MagicMock, patch - +import httpx import pytest import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.main import responses_api_bridge_check +class _RecordingResponsesHandler: + """MockTransport handler that serves a canned /responses reply and keeps the body xAI would have received""" + + def __init__(self, reply: Mapping[str, object]) -> None: + self.reply: Final = reply + self.request_body: Mapping[str, object] | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.request_body = json.loads(request.content) + return httpx.Response(200, json=dict(self.reply), request=request) + + class TestXAIResponsesAutoRouting: """Test that xAI requests with tools automatically route to Responses API""" @@ -254,6 +270,44 @@ class TestXAIResponsesAutoRouting: # Note: This test may need adjustment based on actual mock_response behavior # The key is that the responses_api_bridge_check logic routes correctly + def test_system_message_survives_web_search_bridge(self): + """A system message becomes 'instructions' on the bridged /responses call, and xAI accepts it""" + handler: Final = _RecordingResponsesHandler( + reply={ + "id": "resp_test", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "grok-4.6", + "output": [ + { + "type": "message", + "id": "msg_test", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "1.0.0", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + ) + + response: Final = litellm.completion( + model="xai/grok-4.6", + messages=[ + {"role": "system", "content": "Answer briefly."}, + {"role": "user", "content": "newest litellm version?"}, + ], + web_search_options={"search_context_size": "medium"}, + api_key="fake-key", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))), + ) + + assert response.choices[0].message.content == "1.0.0" + assert handler.request_body is not None + assert handler.request_body["instructions"] == "Answer briefly." + assert handler.request_body["tools"] == [{"type": "web_search"}] + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py index e3575c33b17..1c968126c42 100644 --- a/tests/test_litellm/vector_stores/test_main.py +++ b/tests/test_litellm/vector_stores/test_main.py @@ -7,6 +7,7 @@ executor, and it must never leak into litellm_params/kwargs where logging would model_dump() it (the #19550 serialization trap). """ +import json from unittest.mock import MagicMock, patch import pytest @@ -15,6 +16,7 @@ import litellm.vector_stores.main as vector_stores_main from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.vector_stores.main import search MOCK_SEARCH_RESPONSE = { @@ -89,3 +91,26 @@ def test_search_router_not_in_litellm_params(): litellm_params = mock_handler.call_args.kwargs["litellm_params"] assert "router" not in litellm_params.model_dump(exclude_none=True) assert getattr(litellm_params, "router", None) is None + + +def test_search_forwards_top_level_user_context_to_bedrock_retrieve(): + """Regression (LIT-4415): a top-level userContext, the shape the OpenAI SDK's extra_body + produces on the proxy path, reaches the Bedrock Retrieve request body.""" + client = MagicMock(spec=HTTPHandler) + client.post.return_value = MagicMock(status_code=200, json=MagicMock(return_value={"retrievalResults": []})) + + search( + vector_store_id="kb123", + query="q", + custom_llm_provider="bedrock", + aws_region_name="us-west-2", + aws_access_key_id="test-key-id", + aws_secret_access_key="test-secret-key", + userContext={"userId": "alice@example.com"}, + client=client, + litellm_logging_obj=MagicMock(), + ) + + posted = json.loads(client.post.call_args.kwargs["data"]) + assert posted["userContext"] == {"userId": "alice@example.com"} + assert posted["retrievalQuery"] == {"text": "q"} diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index ab43d1acb00..e8a7732e4cb 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -307,7 +307,7 @@ async def test_chat_completion(): model="gpt-4", messages=[{"role": "user", "content": "Hello!"}], ) - assert "key not allowed to access model." in str(e) + assert "is not available for this API key" in str(e) @pytest.mark.asyncio diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 85b45c07bc2..a1bb370a074 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -1,8 +1,6 @@ from __future__ import annotations import importlib -from pathlib import Path -from types import SimpleNamespace from typing import Final import pytest @@ -10,16 +8,10 @@ import pytest models = importlib.import_module("tests.rust-python-harness.shared.reporting.models") strategy_module = importlib.import_module("tests.rust-python-harness.shared.reporting.strategy") ui = importlib.import_module("tests.rust-python-harness.shared.reporting.ui") -mapping_validator = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.mapping_validator") -mappings = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.mappings") -ocr_mapping = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.cases.ocr") +contracts = importlib.import_module("tests.rust-python-harness.shared.unit_runners.contracts") cli = importlib.import_module("tests.rust-python-harness.cli") -native_build = importlib.import_module("tests.rust-python-harness.shared.native_build") -audit_mapping = mapping_validator.audit_mapping -UNIT_TEST_CONTRACTS = mappings.UNIT_TEST_CONTRACTS -OCR_CONTRACT = ocr_mapping.OCR_CONTRACT -REPO_ROOT = Path(__file__).resolve().parents[1] +UNIT_TEST_CONTRACTS = contracts.UNIT_TEST_CONTRACTS CaseResult = models.CaseResult Coverage = models.Coverage HarnessCase = models.HarnessCase @@ -49,7 +41,6 @@ def _case(module: str = "tests.example") -> HarnessCase: "tests.rust-python-harness.strategies.trace_parity.sdk.messages.case", "tests.rust-python-harness.strategies.trace_parity.sdk.chat_completions.case", "tests.rust-python-harness.strategies.trace_parity.sdk.transcription.case", - "tests.rust-python-harness.strategies.trace_parity.gateway.messages.case", ], ) def test_implemented_namespace_case_modules_remain_importable(module: str) -> None: @@ -117,70 +108,14 @@ def test_should_format_developer_facing_run_context() -> None: assert _format_duration(1.25) == "1.2s" -def test_should_leave_functions_without_mapping_contracts_unimplemented() -> None: +def test_should_leave_functions_without_unit_test_contracts_unimplemented() -> None: assert "messages" not in UNIT_TEST_CONTRACTS -def test_should_report_a_bridge_that_cannot_be_imported() -> None: - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "get_native_bridge", lambda: None) - message: Final = native_build.trace_bridge_error() - - assert message is not None - assert "not importable" in message - - -def test_should_report_a_bridge_built_without_the_trace_feature() -> None: - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=None)) - message: Final = native_build.trace_bridge_error() - - assert message is not None - assert native_build.BRIDGE_FEATURE in message - - -def test_should_accept_a_bridge_built_with_the_trace_feature() -> None: - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=object())) - - assert native_build.trace_bridge_error() is None - - -def test_should_not_rebuild_the_bridge_while_reporting_its_state() -> None: - def forbidden_rebuild(repo_root: object) -> tuple[bool, str]: - raise AssertionError("trace_bridge_error must not rebuild the native bridge") - - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "_rebuild", forbidden_rebuild) - patch.setattr(native_build, "get_native_bridge", lambda: None) - - assert native_build.trace_bridge_error() is not None - - -def test_should_derive_ocr_mapping_status_from_live_tests() -> None: - bridge_error: Final = native_build.trace_bridge_error() - if bridge_error is not None: - pytest.skip(bridge_error) - - report = audit_mapping(OCR_CONTRACT, repo_root=REPO_ROOT) - - assert report.is_valid, ( - f"Missing Python tests: {list(report.missing_python_tests)}\n" - f"Missing Rust tests: {list(report.missing_rust_tests)}\n" - f"Duplicate Python mappings: {list(report.duplicate_python_mappings)}\n" - f"Invalid mapping exclusions: {list(report.invalid_mapping_exclusions)}\n" - f"Invalid parity exclusions: {list(report.invalid_unit_parity_exclusions)}" - ) - assert report.mapped_count == len(OCR_CONTRACT.mapping.mappings) - assert report.total_count == ( - report.mapped_count + len(report.excluded_python_tests) + len(report.unmapped_python_tests) - ) - - def test_strategy_subcommand_accepts_function_filter(capsys: pytest.CaptureFixture[str]) -> None: - exit_code: Final = cli.main(["run", "unit_tests_mapping", "--function", "messages"]) + exit_code: Final = cli.main(["run", "unit_tests_rust", "--function", "messages"]) captured: Final = capsys.readouterr() assert exit_code == 0 assert "- messages: not_implemented" in captured.out - assert "unit_tests_mapping:messages: not_implemented" not in captured.out + assert "unit_tests_rust:messages: not_implemented" not in captured.out diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index f11b74c939d..1625e0cbfb8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -104,8 +104,8 @@ export function AutoRoutersPanel({ Add Auto Router - Routes each request to a model by classifying its complexity. Called like any other model, so clients keep - using a single model name. + Choose a classifier to route each request to a model. Called like any other model, so clients keep using a + single model name. Array.from(new Set(models)); const COMPLEXITY_TYPE_LABELS: Record = { llm: "LLM Classifier", + capability: "Capability", + llm_v2: "Fuse v2", heuristic_first: "Heuristic first", hybrid: "Hybrid", custom: "Custom classifier", diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx new file mode 100644 index 00000000000..ac6851349ea --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -0,0 +1,75 @@ +import React, { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const initial: ComplexityRouterConfigValue = { + classifier_type: "llm", + tiers: { SIMPLE: ["efficient"], MEDIUM: [], COMPLEX: [], REASONING: ["capable"] }, +}; + +function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) { + const [value, setValue] = useState(initialValue); + return ( + + {value.classifier_type} + + ); +} + +describe("AutoRouterClassifierTabs", () => { + it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid"] as const)( + "groups %s under Complexity without resetting its configuration", + (classifier_type) => { + const onChange = vi.fn(); + renderWithProviders( + + Existing classifier settings + , + ); + expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Existing classifier settings"); + fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + expect(onChange).not.toHaveBeenCalled(); + }, + ); + + it.each([ + ["capability", "Capability"], + ["llm_v2", "Fuse v2"], + ] as const)("opens saved %s settings and switches back to local Complexity", (classifier_type, label) => { + renderWithProviders(
); + expect(screen.getByRole("tab", { name: label })).toHaveAttribute("aria-selected", "true"); + fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("heuristic"); + }); + + it("keeps custom tiers editable under Complexity and explains why forecast tabs are disabled", () => { + const onChange = vi.fn(); + renderWithProviders( + + Custom tiers + , + ); + expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Custom tiers"); + for (const name of ["Capability", "Fuse v2"]) { + const tab = screen.getByRole("tab", { name }); + expect(tab).toHaveAttribute("aria-disabled", "true"); + expect(tab).toHaveAccessibleDescription("Restore standard tiers to use Capability or Fuse v2."); + fireEvent.click(tab); + } + expect(onChange).not.toHaveBeenCalled(); + expect(screen.getByText("Restore standard tiers to use Capability or Fuse v2.")).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx new file mode 100644 index 00000000000..98c0d4aab2f --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -0,0 +1,58 @@ +import React, { useId } from "react"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { effectiveClassifierType, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { transitionClassifierType } from "./classifier_type_transition"; +import { isForecastClassifier } from "./forecast_classifier_config"; + +interface AutoRouterClassifierTabsProps { + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + children: React.ReactNode; +} + +const AutoRouterClassifierTabs: React.FC = ({ value, onChange, children }) => { + const restrictionId = useId(); + const classifierType = effectiveClassifierType(value); + const selected = isForecastClassifier(classifierType) ? classifierType : "complexity"; + const hasCustomTiers = Boolean(value.custom_tier_set); + + const handleChange = (tab: unknown) => { + if (tab === selected) return; + if (tab === "complexity") { + onChange(transitionClassifierType(value, isForecastClassifier(classifierType) ? "heuristic" : classifierType)); + } else if (!hasCustomTiers && (tab === "capability" || tab === "llm_v2")) { + onChange(transitionClassifierType(value, tab)); + } + }; + + return ( + +

Classifier type

+ + Complexity + + Capability + + + Fuse v2 + + + {hasCustomTiers && ( +

+ Restore standard tiers to use Capability or Fuse v2. +

+ )} + {children} + + ); +}; + +export default AutoRouterClassifierTabs; diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index a6f2e65793a..64b08fc9ed1 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -1,3 +1,4 @@ +import { transitionClassifierType } from "./classifier_type_transition"; import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; @@ -17,7 +18,6 @@ import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import ClassifierVisionConfig from "./ClassifierVisionConfig"; import type { ReasoningEffort } from "./complexity_router_tiers"; -import { nonReasoningTierFields } from "./nonReasoningTierFields"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { ClassificationFrequency, @@ -33,12 +33,10 @@ import { DEFAULT_CLASSIFIER_FALLBACK, DEFAULT_CLASSIFIER_TIMEOUT_MS, DEFAULT_CLASSIFICATION_RUBRIC, - NEW_CLASSIFIER_CLASSIFICATION_RUBRIC, ClassificationRubric, effectiveTierLabel, heuristicScoringRole, usesLlmClassifier, - DEFAULT_HEURISTIC_FIRST_MAX_TIER, DEFAULT_HYBRID_BOUNDARY_MARGIN, HEURISTIC_FIRST_MAX_TIER_KEYS, effectiveClassifierType, @@ -263,35 +261,7 @@ const ClassificationMethodConfig: React.FC = ({ const explicitlySupportedClassifierEfforts = effortOptionsByModel[classifierModel]; const handleClassifierTypeChange = (classifierType: ClassifierType) => { - const nextValue: ComplexityRouterConfigValue = { - ...value, - classifier_type: classifierType, - classifier_llm_config: usesLlmClassifier(classifierType) - ? value.classifier_llm_config ?? { - model: "", - timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS, - classification_rubric: NEW_CLASSIFIER_CLASSIFICATION_RUBRIC, - } - : undefined, - classifier_context_window_size: usesLlmClassifier(classifierType) - ? value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE - : undefined, - classifier_context_budget_chars: usesLlmClassifier(classifierType) - ? value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS - : undefined, - classifier_context_include_assistant_turns: usesLlmClassifier(classifierType) - ? value.classifier_context_include_assistant_turns - : undefined, - classifier_fallback: usesLlmClassifier(classifierType) ? value.classifier_fallback : undefined, - heuristic_first_max_tier: - classifierType === "heuristic_first" - ? value.heuristic_first_max_tier ?? DEFAULT_HEURISTIC_FIRST_MAX_TIER - : undefined, - hybrid_boundary_margin: - classifierType === "hybrid" ? value.hybrid_boundary_margin ?? DEFAULT_HYBRID_BOUNDARY_MARGIN : undefined, - ...nonReasoningTierFields(classifierType, value), - }; - onChange(nextValue); + onChange(transitionClassifierType(value, classifierType)); }; const handleHeuristicFirstMaxTierChange = (tier: string) => { @@ -433,27 +403,6 @@ const ClassificationMethodConfig: React.FC = ({ }); }; - if (classifierType === "capability") { - return ( -

- This router uses capability forecasting. Configure its classifier, threshold, and calibration through YAML or - the API. Saving preserves those settings -

- ); - } - - if (classifierType === "llm_v2") { - return ( -
- LLM V2 classifier (experimental) -

- Combines task demands and model capability in one forecast. Its solver profiles and quality allowance are - configured through the API. Saving this router preserves those settings -

-
- ); - } - return ( <> diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index c6b9a69e76a..f6b50ce20bc 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -1,8 +1,11 @@ +import RoutingOptions from "./RoutingOptions"; +import PlanModeOverrideControls from "./PlanModeOverrideControls"; +import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassifierConfig"; +import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; -import { SearchSelect } from "@/components/shared/SearchSelect"; +import DefaultModelField from "./DefaultModelField"; import { ChevronRight, Info, Plus, Trash2, X } from "lucide-react"; -import { Switch } from "@/components/ui/switch"; import { AffinityControls } from "./AffinityControls"; import NonReasoningTierToggle from "./NonReasoningTierToggle"; @@ -204,11 +207,6 @@ const rowOrigin = (row: TierRow, editing: boolean): string => { return isBuiltInTierName(row.name) ? "built-in" : "custom"; }; -const defaultModelPlaceholderFor = (derivedDefaultModel: string | undefined, isCustomSet: boolean): string => { - if (derivedDefaultModel) return `Derived from tiers: ${derivedDefaultModel}`; - return isCustomSet ? "Add a model to your fallback tier" : "Add a model to the Simple or Medium tier"; -}; - const builtInTierInfo = (rowId: string): { label: string; description: string; examples: string } | undefined => { const builtIn = ALL_BUILT_IN_TIERS.find((tier) => tier === rowId); return builtIn ? TIER_DESCRIPTIONS[builtIn] : undefined; @@ -376,6 +374,8 @@ export interface ComplexityRouterConfigValue { /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ default_model?: string; classifier_type: ClassifierType; + capability_classifier_config?: CapabilitySettings; + llm_v2_config?: FuseSettings; classifier_llm_config?: ClassifierLLMConfig; classifier_context_window_size?: number; classifier_context_budget_chars?: number; @@ -535,44 +535,6 @@ export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03; */ export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_ORDER.slice(0, -1); -const PlanModeOverrideControls: React.FC<{ - value: ComplexityRouterConfigValue; - onChange: (value: ComplexityRouterConfigValue) => void; - planModeTierOptions: { value: string; label: string }[]; -}> = ({ value, onChange, planModeTierOptions }) => ( - <> -
- - onChange({ - ...value, - plan_mode_min_tier: enabled ? planModeTierOptions.at(-1)?.value : undefined, - }) - } - aria-label="Route plan-mode requests to a minimum tier" - /> - Route plan-mode requests to a minimum tier -
- - Requests from coding agents in plan mode (Claude Code, GitHub Copilot) route to at least this tier. The classifier - still wins when it picks higher, and the override only lasts while plan mode is active. - {planModeTierOptions.length === 0 && " Add models to a tier to enable this."} - - {value.plan_mode_min_tier !== undefined && ( -
- onChange({ ...value, plan_mode_min_tier: tier })} - /> -
- )} - -); - const ComplexityRouterConfig: React.FC = ({ modelInfo, value, @@ -596,6 +558,7 @@ const ComplexityRouterConfig: React.FC = ({ onAutoRouterCompressionChange, showValidationErrors = false, }) => { + const forecast = isForecastClassifier(value.classifier_type); const customTierSet = value.custom_tier_set; const tierRows = activeTierRows(value); const tierRowsError = customTierSet ? getCustomTierRowsError(customTierSet) : null; @@ -605,8 +568,6 @@ const ComplexityRouterConfig: React.FC = ({ value: row.id, label: tierRowLabel(row, value.tier_labels), })); - const derivedDefaultModel = resolveComplexityDefaultModel(value); - const defaultModelPlaceholder = defaultModelPlaceholderFor(derivedDefaultModel, Boolean(customTierSet)); const defaultModel = resolveComplexityDefaultModel(value, value.default_model); const dispatch = (action: TierSetAction) => { @@ -641,298 +602,319 @@ const ComplexityRouterConfig: React.FC = ({ tier_model_params: setTierModelParam(value.tier_model_params, tier, model, change), }); - // Clearing the select drops the key entirely rather than storing "", so an emptied pin reads as - // "track the tiers" everywhere downstream instead of as a blank model name. - const handleDefaultModelChange = (model: string | null | undefined) => { - onChange({ ...value, default_model: model || undefined }); - }; - - const handleTierLabelChange = (tier: keyof ComplexityTiers, label: string) => { - onChange({ - ...value, - tier_labels: { ...value.tier_labels, [tier]: label }, - }); - }; + const handleTierLabelChange = (tier: keyof ComplexityTiers, label: string) => + onChange({ ...value, tier_labels: { ...value.tier_labels, [tier]: label } }); return (
-

Complexity Tier Configuration

- - - +

+ {forecast ? "Solver models" : "Complexity Tier Configuration"} +

+ {!forecast && ( + + + + )}
- - - - - {!customTierSet && ( - - )} - - {tierRows.map((row, index) => { - const tierInfo = builtInTierInfo(row.id); - const label = tierRowLabel(row, value.tier_labels); - const tierMissing = showValidationErrors && row.models.length === 0; - const needsDefinition = Boolean(customTierSet) && !row.definition.trim() && !isBuiltInTierName(row.name); - const definitionMissing = showValidationErrors && needsDefinition; - const showsDisplayName = !customTierSet && !editingTiers; - return ( -
- {index > 0 && } -
- removeTierRow(row.id)} - /> - {tierInfo && !customTierSet && ( - Examples: {tierInfo.examples} - )} - {editingTiers && ( - updateTierRow(row.id, patch)} - /> - )} - {showsDisplayName && tierInfo && ( - - handleTierLabelChange(row.id as keyof ComplexityTiers, event.target.value)} - placeholder={`Display name (default: ${tierInfo.label})`} - aria-label={`Display name for the ${tierInfo.label} tier`} - /> - {value.tier_labels?.[row.id as keyof ComplexityTiers] && ( - - handleTierLabelChange(row.id as keyof ComplexityTiers, "")} - > - - - - )} - - )} - setRowModels(row, models)} - placeholder={`Select model(s) for ${label.toLowerCase()} queries`} - emptyText="No models found" - className={tierMissing ? "w-full border-destructive" : "w-full"} - /> - - handleTierModelParamChange(row.id, model, ["reasoning_effort", effort]) - } - onFastModeChange={(model, enabled) => - handleTierModelParamChange(row.id, model, ["speed", enabled ? "fast" : undefined]) - } - /> - {row.models.length > 1 && ( - - Multiple models selected: the router randomly picks among them per request (or Thompson-samples - within the pool when adaptive routing is on). - - )} - {tierMissing && The {label} tier is required} -
-
- ); - })} - - + + + + ) : ( + <> + - {customTierSet && ( - onChange(setFallbackTier(value, fallbackTierId))} - /> - )} + + + {!customTierSet && ( + + )} - + {tierRows.map((row, index) => { + const tierInfo = builtInTierInfo(row.id); + const label = tierRowLabel(row, value.tier_labels); + const tierMissing = showValidationErrors && row.models.length === 0; + const needsDefinition = + Boolean(customTierSet) && !row.definition.trim() && !isBuiltInTierName(row.name); + const definitionMissing = showValidationErrors && needsDefinition; + const showsDisplayName = !customTierSet && !editingTiers; + return ( +
+ {index > 0 && } +
+ removeTierRow(row.id)} + /> + {tierInfo && !customTierSet && ( + Examples: {tierInfo.examples} + )} + {editingTiers && ( + updateTierRow(row.id, patch)} + /> + )} + {showsDisplayName && tierInfo && ( + + + handleTierLabelChange(row.id as keyof ComplexityTiers, event.target.value) + } + placeholder={`Display name (default: ${tierInfo.label})`} + aria-label={`Display name for the ${tierInfo.label} tier`} + /> + {value.tier_labels?.[row.id as keyof ComplexityTiers] && ( + + handleTierLabelChange(row.id as keyof ComplexityTiers, "")} + > + + + + )} + + )} + setRowModels(row, models)} + placeholder={`Select model(s) for ${label.toLowerCase()} queries`} + emptyText="No models found" + className={tierMissing ? "w-full border-destructive" : "w-full"} + /> + + handleTierModelParamChange(row.id, model, ["reasoning_effort", effort]) + } + onFastModeChange={(model, enabled) => + handleTierModelParamChange(row.id, model, ["speed", enabled ? "fast" : undefined]) + } + /> + {row.models.length > 1 && ( + + Multiple models selected: the router randomly picks among them per request (or + Thompson-samples within the pool when adaptive routing is on). + + )} + {tierMissing && The {label} tier is required} +
+
+ ); + })} -
-
- Default Model - - - -
- - - Used when the tier the request lands in has no model, and when the classifier fails with "Route to - the default model" selected. - -
- - + + {customTierSet && ( + onChange(setFallbackTier(value, fallbackTierId))} + /> + )} + + + + )} + {!forecast && } -
- {[ - { - key: "classifier", - label: Advanced: Classification Method, - children: ( - - ), - }, - { - key: "adaptive", - label: Advanced: Adaptive Routing, - children: ( - - - - ), - }, - { - key: "affinity", - label: Advanced: Affinity, - children: , - }, - { - key: "modality", - label: Advanced: Modality Routing, - children: , - }, - { - key: "plan-mode", - label: Advanced: Plan-Mode Override, - children: ( - - ), - }, - { - key: "context-window", - label: Advanced: Context Window Escalation, - children: , - }, - { - key: "stall-escalation", - label: Advanced: Stalled Task Escalation, - children: ( - - - - ), - }, - { - key: "response", - label: Advanced: Response Format, - children: , - }, - ...(onEscalationKeywordsChange - ? [ - { - key: "escalation", - label: Advanced: Escalation Keywords, - children: ( - - - - ), - }, - ] - : []), - ...(onAutoRouterCompressionChange - ? [ - { - key: "compression", - label: Advanced: Compression, - children: ( - - ), - }, - ] - : []), - ...(onKeywordTierRulesChange || onSemanticMatchingEnabledChange - ? [ - { - key: "keyword-semantic", - label: Advanced: Keyword/Semantic Matching, - children: ( - <> - {onKeywordTierRulesChange && ( - - )} - {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && } - {onSemanticMatchingEnabledChange && ( - - )} - - ), - }, - ] - : []), - ].map(({ key, label, children }) => ( - - - - {label} - - {children} - - ))} -
+ + {forecast && ( + <> + + + + )} +
+ {[ + ...(!forecast + ? [ + { + key: "classifier", + label: Advanced: Classification Method, + children: ( + + ), + }, + ] + : []), + { + key: "adaptive", + label: Advanced: Adaptive Routing, + children: ( + + + + ), + }, + { + key: "affinity", + label: Advanced: Affinity, + children: , + }, + { + key: "modality", + label: Advanced: Modality Routing, + children: , + }, + { + key: "plan-mode", + label: Advanced: Plan-Mode Override, + children: ( + + ), + }, + { + key: "context-window", + label: Advanced: Context Window Escalation, + children: , + }, + { + key: "stall-escalation", + label: Advanced: Stalled Task Escalation, + children: ( + + + + ), + }, + { + key: "response", + label: Advanced: Response Format, + children: , + }, + ...(onEscalationKeywordsChange + ? [ + { + key: "escalation", + label: Advanced: Escalation Keywords, + children: ( + + + + ), + }, + ] + : []), + ...(onAutoRouterCompressionChange + ? [ + { + key: "compression", + label: Advanced: Compression, + children: ( + + ), + }, + ] + : []), + ...(onKeywordTierRulesChange || onSemanticMatchingEnabledChange + ? [ + { + key: "keyword-semantic", + label: ( + Advanced: Keyword/Semantic Matching + ), + children: ( + <> + {onKeywordTierRulesChange && ( + + )} + {onKeywordTierRulesChange && onSemanticMatchingEnabledChange && } + {onSemanticMatchingEnabledChange && ( + + )} + + ), + }, + ] + : []), + ] + .filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key)) + .map(({ key, label, children }) => ( + + + + {label} + + {children} + + ))} +
+
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx index 34d14091bde..810289da79f 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx @@ -1,11 +1,12 @@ import userEvent from "@testing-library/user-event"; import React from "react"; import { describe, expect, it, vi } from "vitest"; -import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { renderWithProviders, screen, within } from "../../../tests/test-utils"; import { buildUpdatedComplexityRouterConfig, hydrateComplexityRouterConfig, } from "../edit_auto_router/edit_auto_router_modal"; +import type { KeywordTierRule } from "./KeywordTierRules"; import type { ModelGroup } from "../llm_calls/fetch_models"; import ComplexityRouterConfig, { type ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; @@ -50,8 +51,8 @@ it.each([false, true])("edits and round-trips independent model settings with cu const view = renderWithProviders(editor(initial)); const fast = () => screen.getByRole("switch", { name: `Fast mode for primary in the ${label} tier` }); - expect(screen.getAllByRole("switch", { name: /^Fast mode for/ })).toHaveLength(3); - expect(screen.queryByRole("switch", { name: /^Fast mode for (blocked|missing)/ })).not.toBeInTheDocument(); + expect(screen.getAllByRole("switch", { name: /^Fast mode for/ })).toHaveLength(4); + expect(screen.queryByRole("switch", { name: /^Fast mode for missing/ })).not.toBeInTheDocument(); expect(screen.queryByRole("combobox", { name: /^Reasoning effort for secondary/ })).not.toBeInTheDocument(); expect(screen.getByRole("switch", { name: `Fast mode for secondary in the ${label} tier` })).toBeChecked(); expect(fast()).not.toBeChecked(); @@ -125,19 +126,170 @@ it.each([false, true])("edits and round-trips independent model settings with cu ); }); -describe("Fast mode metadata", () => { - it("offers nothing before model capabilities load and leaves stored speed untouched", () => { - const value: ComplexityRouterConfigValue = { - tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: [] }, - classifier_type: "heuristic", - tier_model_params: { SIMPLE: { primary: { speed: "fast" } } }, - }; - const onChange = vi.fn(); - renderWithProviders(); - expect(screen.queryByRole("switch", { name: /^Fast mode for/ })).not.toBeInTheDocument(); - expect(onChange).not.toHaveBeenCalled(); - expect(buildUpdatedComplexityRouterConfig({}, value).tier_model_configs).toEqual({ - SIMPLE: [{ model_name: "primary", litellm_params: { speed: "fast" } }], - }); +it.each(["capability", "llm_v2"] as const)("preserves Fast mode controls for %s solvers", async (classifierType) => { + const user = userEvent.setup(); + const initial: ComplexityRouterConfigValue = { + classifier_type: classifierType, + classifier_llm_config: { model: "primary", timeout_ms: 3000 }, + tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: ["blocked"] }, + capability_classifier_config: { efficient_tier: "SIMPLE", capable_tier: "REASONING", base_threshold: 0.7 }, + llm_v2_config: { + efficient_profile: "Small solver", + capable_profile: "Large solver", + harness: "One attempt", + max_quality_gap: 0.05, + }, + tier_model_params: { SIMPLE: { primary: { reasoning_effort: "high", max_tokens: 1024 } } }, + }; + const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>(); + const editor = (value: ComplexityRouterConfigValue) => ( + + ); + const view = renderWithProviders(editor(initial)); + const fast = () => screen.getByRole("switch", { name: "Fast mode for primary in the Efficient solver tier" }); + expect(screen.getAllByRole("switch", { name: /^Fast mode for/ })).toHaveLength(1); + expect(fast()).not.toBeChecked(); + await user.click(fast()); + const enabled = onChange.mock.lastCall![0]; + expect(enabled.tier_model_params?.SIMPLE.primary).toEqual({ + reasoning_effort: "high", + max_tokens: 1024, + speed: "fast", + }); + const saved = buildUpdatedComplexityRouterConfig({}, enabled); + expect(saved.tier_model_configs).toEqual({ + SIMPLE: [{ model_name: "primary", litellm_params: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" } }], + }); + view.rerender(editor(hydrateComplexityRouterConfig(saved, undefined))); + expect(fast()).toBeChecked(); + await user.click(fast()); + expect(onChange.mock.lastCall![0].tier_model_params?.SIMPLE.primary).toEqual({ + reasoning_effort: "high", + max_tokens: 1024, }); }); + +describe("Fast mode metadata", () => { + it.each(["heuristic", "capability", "llm_v2"] as const)( + "can clear stored Fast mode without current capability metadata for %s", + async (classifier_type) => { + const user = userEvent.setup(); + const value: ComplexityRouterConfigValue = { + tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: ["secondary"] }, + classifier_type, + tier_model_params: { SIMPLE: { primary: { speed: "fast", max_tokens: 512 } } }, + }; + const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>(); + const editor = (current: ComplexityRouterConfigValue, info: ModelGroup[]) => ( + + ); + const view = renderWithProviders(editor(value, [])); + const fast = () => screen.getByRole("switch", { name: /^Fast mode for primary/ }); + expect(fast()).toBeChecked(); + expect(onChange).not.toHaveBeenCalled(); + view.rerender(editor(value, [{ model_group: "primary", supports_fast_mode: false }])); + expect(fast()).toBeChecked(); + await user.click(fast()); + const cleared = onChange.mock.lastCall![0]; + expect(cleared.tier_model_params?.SIMPLE.primary).toEqual({ max_tokens: 512 }); + const saved = buildUpdatedComplexityRouterConfig({}, cleared); + expect(saved.tier_model_configs).toEqual({ + SIMPLE: [{ model_name: "primary", litellm_params: { max_tokens: 512 } }], + }); + view.rerender(editor(hydrateComplexityRouterConfig(saved, undefined), [])); + expect(screen.queryByRole("switch", { name: /^Fast mode for primary/ })).not.toBeInTheDocument(); + view.rerender(editor(cleared, modelInfo)); + expect(fast()).not.toBeChecked(); + }, + ); +}); + +it.each(["MEDIUM", "REASONING"])("clears a legacy Capability pool while reconciling plan floor %s", async (floor) => { + const user = userEvent.setup(); + const stored = { + classifier_type: "capability" as const, + plan_mode_min_tier: floor, + tiers: { SIMPLE: ["primary"], MEDIUM: ["secondary"], COMPLEX: [], REASONING: ["blocked"] }, + tier_model_configs: { MEDIUM: [{ model_name: "secondary", litellm_params: { speed: "fast" } }] }, + }; + const value = hydrateComplexityRouterConfig(stored, undefined); + const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + expect(screen.getByRole("switch", { name: "Fast mode for secondary in the Medium routing pool tier" })).toBeChecked(); + expect(onChange).not.toHaveBeenCalled(); + await user.click(screen.getByRole("combobox", { name: "Select medium routing pool models" })); + await user.click(await screen.findByRole("option", { name: "secondary" })); + await user.keyboard("{Escape}"); + const cleared = onChange.mock.lastCall![0]; + expect(cleared.tiers.MEDIUM).toEqual([]); + expect(cleared.plan_mode_min_tier).toBe(floor === "MEDIUM" ? undefined : floor); + expect(cleared.tier_model_params).toBeUndefined(); + expect(buildUpdatedComplexityRouterConfig(stored, cleared).tiers).toEqual({ + SIMPLE: ["primary"], + REASONING: ["blocked"], + }); +}); + +it.each(["capability", "llm_v2"] as const)( + "shows and clears a persisted default model in %s", + async (classifier_type) => { + const user = userEvent.setup(); + const stored = { + classifier_type, + default_model: "legacy-default", + tiers: { SIMPLE: ["primary"], REASONING: ["secondary"] }, + }; + const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>(); + const editor = (value: ComplexityRouterConfigValue) => ( + + ); + const view = renderWithProviders(editor(hydrateComplexityRouterConfig(stored, undefined))); + await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + const select = () => screen.getByRole("combobox", { name: "Default model" }); + expect(select()).toHaveValue("legacy-default"); + expect(onChange).not.toHaveBeenCalled(); + await user.click(select()); + await user.click(await screen.findByRole("option", { name: "blocked" })); + const changed = onChange.mock.lastCall![0]; + expect(buildUpdatedComplexityRouterConfig(stored, changed).default_model).toBe("blocked"); + view.rerender(editor(changed)); + await user.click( + within(screen.getByRole("group", { name: "Default model configuration" })).getByRole("button", { name: "Clear" }), + ); + const cleared = onChange.mock.lastCall![0]; + expect(cleared.default_model).toBeUndefined(); + const saved = buildUpdatedComplexityRouterConfig(stored, cleared); + expect(saved).not.toHaveProperty("default_model"); + view.rerender(editor(hydrateComplexityRouterConfig(saved, undefined))); + expect(select()).toHaveValue(""); + expect(select()).toHaveAttribute("placeholder", expect.stringContaining("primary")); + }, +); + +it.each(["capability", "llm_v2"] as const)("offers only populated keyword targets for %s", async (classifier_type) => { + const user = userEvent.setup(); + const value: ComplexityRouterConfigValue = { + classifier_type, + tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: ["secondary"] }, + }; + const onRulesChange = vi.fn<(rules: KeywordTierRule[]) => void>(); + const editor = (rules: KeywordTierRule[]) => ( + + ); + const view = renderWithProviders(editor([])); + await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + await user.click(screen.getByRole("button", { name: "Add keyword rule" })); + const rules = onRulesChange.mock.lastCall![0]; + expect(rules[0].tier).toBe("SIMPLE"); + view.rerender(editor(rules)); + await user.click(screen.getByRole("combobox", { name: "Route keyword rule 1 to tier" })); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual(["Simple", "Reasoning"]); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/DefaultModelField.tsx b/ui/litellm-dashboard/src/components/add_model/DefaultModelField.tsx new file mode 100644 index 00000000000..a3ab85f7c13 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/DefaultModelField.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import { Info } from "lucide-react"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { isForecastClassifier } from "./forecast_classifier_config"; +import { resolveComplexityDefaultModel } from "./tier_rows"; + +interface DefaultModelFieldProps { + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + modelOptions: { value: string; label: string }[]; +} + +const defaultModelPlaceholderFor = (derivedDefaultModel: string | undefined, isCustomSet: boolean): string => { + if (derivedDefaultModel) return `Derived from tiers: ${derivedDefaultModel}`; + return isCustomSet ? "Add a model to your fallback tier" : "Add a model to the Simple or Medium tier"; +}; + +const DefaultModelField = ({ value, onChange, modelOptions }: DefaultModelFieldProps) => { + const defaultModelPlaceholder = defaultModelPlaceholderFor( + resolveComplexityDefaultModel(value), + Boolean(value.custom_tier_set), + ); + // Clearing the select drops the key entirely rather than storing "", so an emptied pin reads as + // "track the tiers" everywhere downstream instead of as a blank model name. + const handleDefaultModelChange = (model: string | null | undefined) => { + onChange({ ...value, default_model: model || undefined }); + }; + + return ( +
+
+ Default Model + + + +
+ + + {isForecastClassifier(value.classifier_type) + ? "Used when routing cannot find a suitable model. Classifier failures route to the capable solver." + : 'Used when the tier the request lands in has no model, and when the classifier fails with "Route to the default model" selected.'} + +
+ ); +}; + +export default DefaultModelField; diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx new file mode 100644 index 00000000000..4a574ac736d --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx @@ -0,0 +1,224 @@ +import React, { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +import ForecastClassifierConfig from "./ForecastClassifierConfig"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { getForecastConfigError, isForecastClassifier } from "./forecast_classifier_config"; +import { buildUpdatedComplexityRouterConfig } from "../edit_auto_router/edit_auto_router_modal"; + +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + getComplexityScorerDefaults: vi.fn(async () => ({ + tier_boundaries: {}, + token_thresholds: {}, + dimension_weights: {}, + })), +})); + +const initial: ComplexityRouterConfigValue = { + classifier_type: "capability", + classifier_llm_config: { model: "judge", timeout_ms: 20000 }, + tiers: { SIMPLE: ["efficient"], MEDIUM: [], COMPLEX: [], REASONING: ["capable"] }, + capability_classifier_config: { efficient_tier: "SIMPLE", capable_tier: "REASONING", base_threshold: 0.7 }, +}; +const fuseInitial: ComplexityRouterConfigValue = { + ...initial, + classifier_type: "llm_v2", + capability_classifier_config: undefined, + adaptive: false, + llm_v2_config: { + efficient_profile: "Small solver", + capable_profile: "Larger solver", + harness: "One attempt", + max_quality_gap: 0.05, + }, +}; +const options = ["judge", "efficient", "capable"].map((model) => ({ value: model, label: model })); + +function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) { + const [value, setValue] = useState(initialValue); + const [saved, setSaved] = useState(""); + return ( + <> + + {isForecastClassifier(value.classifier_type) ? ( + + ) : ( + + )} + + + {saved} + + ); +} + +describe("forecast classifier form", () => { + it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByRole("tab", { name: "Capability" })); + fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.7" } }); + expect(screen.getByRole("button", { name: "Save configuration" })).toBeEnabled(); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + const output = screen.getByRole("status", { name: "Saved configuration" }); + expect(output).toHaveTextContent('"classifier_type":"capability"'); + expect(output).toHaveTextContent('"SIMPLE":["efficient","second-efficient"]'); + expect(output).toHaveTextContent('"REASONING":["capable"]'); + expect(output).toHaveTextContent('"reasoning_effort":"low","speed":"fast","max_tokens":1024'); + expect(output).toHaveTextContent('"reasoning_effort":"high"'); + expect(output).toHaveTextContent('"adaptive":false'); + expect(output).not.toHaveTextContent("leftover-medium"); + expect(output).not.toHaveTextContent("leftover-complex"); + expect(output).not.toHaveTextContent('"plan_mode_min_tier"'); + }); + + it.each(["capability", "llm_v2"] as const)( + "carries non-default solver assignments when switching away from %s", + (source) => { + const pair = { efficient_tier: "MEDIUM", capable_tier: "COMPLEX" }; + const previous: ComplexityRouterConfigValue = { + ...(source === "capability" ? initial : fuseInitial), + tiers: { SIMPLE: [], MEDIUM: ["efficient"], COMPLEX: ["capable"], REASONING: [] }, + capability_classifier_config: + source === "capability" ? { ...initial.capability_classifier_config!, ...pair } : undefined, + llm_v2_config: source === "llm_v2" ? { ...fuseInitial.llm_v2_config!, ...pair } : undefined, + plan_mode_min_tier: "COMPLEX", + tier_model_params: { MEDIUM: { efficient: { max_tokens: 128 } }, COMPLEX: { capable: { speed: "fast" } } }, + }; + renderWithProviders(); + fireEvent.click(screen.getByRole("tab", { name: source === "capability" ? "Fuse v2" : "Capability" })); + if (source === "capability") { + fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Small solver" } }); + fireEvent.change(screen.getByLabelText("Capable solver profile"), { target: { value: "Large solver" } }); + fireEvent.change(screen.getByLabelText("Harness and budget"), { target: { value: "One attempt" } }); + fireEvent.change(screen.getByLabelText("Maximum quality gap"), { target: { value: "0.05" } }); + } else { + fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.7" } }); + } + expect(screen.getByRole("button", { name: "Save configuration" })).toBeEnabled(); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + const output = screen.getByRole("status", { name: "Saved configuration" }); + expect(output).toHaveTextContent('"efficient_tier":"MEDIUM","capable_tier":"COMPLEX"'); + expect(output).toHaveTextContent('"tiers":{"MEDIUM":["efficient"],"COMPLEX":["capable"]}'); + expect(output).toHaveTextContent('"plan_mode_min_tier":"COMPLEX"'); + expect(output).toHaveTextContent('"max_tokens":128'); + expect(output).toHaveTextContent('"speed":"fast"'); + }, + ); + + it("keeps decimal and negative numbers when entered one character at a time", async () => { + const user = userEvent.setup(); + renderWithProviders(); + const threshold = screen.getByLabelText("Solve probability threshold"); + await user.clear(threshold); + await user.type(threshold, "0.65"); + expect(threshold).toHaveValue(0.65); + await user.click(screen.getByRole("button", { name: "Classifier options" })); + await user.click(screen.getByRole("switch", { name: "Use fitted calibration" })); + await user.type(screen.getByLabelText("Efficient intercept"), "-0.3"); + expect(screen.getByLabelText("Efficient intercept")).toHaveValue(-0.3); + }); + + it.each([ + ["capability", "LLM Classifier"], + ["capability", "Heuristic first"], + ["capability", "Hybrid"], + ["llm_v2", "LLM Classifier"], + ["llm_v2", "Heuristic first"], + ["llm_v2", "Hybrid"], + ] as const)("restores the current rubric when switching %s through Complexity to %s", async (source, target) => { + const user = userEvent.setup(); + renderWithProviders(); + fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) })); + await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("option", { name: "judge", exact: true })); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + const output = screen.getByRole("status", { name: "Saved configuration" }); + expect(output).toHaveTextContent('"classification_rubric":"agentic"'); + expect(output).toHaveTextContent('"model":"judge"'); + expect(output).toHaveTextContent('"timeout_ms":3000'); + expect(output).not.toHaveTextContent('"capability_classifier_config"'); + expect(output).not.toHaveTextContent('"llm_v2_config"'); + }); + + it("saves capability threshold edits together with fitted calibration", () => { + renderWithProviders(); + fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.6" } }); + fireEvent.click(screen.getByRole("button", { name: "Classifier options" })); + fireEvent.click(screen.getByRole("switch", { name: "Use fitted calibration" })); + expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Calibration version"), { target: { value: "eval-a" } }); + fireEvent.change(screen.getByLabelText("Efficient slope"), { target: { value: "1.2" } }); + fireEvent.change(screen.getByLabelText("Efficient intercept"), { target: { value: "-0.3" } }); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + const output = screen.getByRole("status", { name: "Saved configuration" }); + expect(output).toHaveTextContent('"base_threshold":0.6'); + expect(output).toHaveTextContent('"calibration":{"version":"eval-a","slope":1.2,"intercept":-0.3}'); + fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "" } }); + expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); + }); + + it("switches to Fuse, requires solver context, and saves the filled fields", () => { + renderWithProviders(); + fireEvent.click(screen.getByRole("tab", { name: "Fuse v2" })); + expect(screen.queryByLabelText("Solve probability threshold")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Efficient solver profile"), { + target: { value: "Short reasoning budget" }, + }); + fireEvent.change(screen.getByLabelText("Capable solver profile"), { target: { value: "Larger reasoning budget" } }); + fireEvent.change(screen.getByLabelText("Harness and budget"), { + target: { value: "Shell and test runner, one attempt" }, + }); + fireEvent.change(screen.getByLabelText("Maximum quality gap"), { target: { value: "0.05" } }); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + const output = screen.getByRole("status", { name: "Saved configuration" }); + expect(output).toHaveTextContent('"classifier_type":"llm_v2"'); + expect(output).toHaveTextContent('"efficient_profile":"Short reasoning budget"'); + expect(output).toHaveTextContent('"capable_profile":"Larger reasoning budget"'); + expect(output).toHaveTextContent('"harness":"Shell and test runner, one attempt"'); + expect(output).toHaveTextContent('"max_quality_gap":0.05'); + expect(output).toHaveTextContent('"adaptive":false'); + expect(output).not.toHaveTextContent('"capability_classifier_config"'); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx new file mode 100644 index 00000000000..b901a509435 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx @@ -0,0 +1,433 @@ +import React from "react"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { ChevronRight } from "lucide-react"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Textarea } from "@/components/ui/textarea"; +import { Switch } from "@/components/ui/switch"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { + type ComplexityRouterConfigValue, + type ClassificationFrequency, + classificationFrequency, + withClassificationFrequency, + DEFAULT_CLASSIFIER_TIMEOUT_MS, +} from "./ComplexityRouterConfig"; +import { + forecastTierNames, + forecastModels, + getForecastConfigError, + newCapabilitySettings, + newFuseSettings, + type CapabilitySettings, + type FuseSettings, +} from "./forecast_classifier_config"; +import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect"; +import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; +import ClassifierVisionConfig from "./ClassifierVisionConfig"; +import TierModelEffortRows from "./TierModelEffortRows"; +import { activeTierRows } from "./tier_rows"; +import { setTierModels } from "./tier_set_actions"; +import { tierRowLabel, setTierModelParam, setTierModelReasoningEffort } from "./complexity_router_tiers"; + +interface Props { + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + modelOptions: { value: string; label: string }[]; + effortOptionsByModel: Record; +} + +const NumberField = ({ + label, + value, + onChange, + min, + max, + step = "any", + help, +}: { + label: string; + value: number; + onChange: (value: number) => void; + min?: number; + max?: number; + step?: number | "any"; + help?: string; +}) => { + const id = React.useId(); + return ( +
+ + onChange(event.target.value === "" ? Number.NaN : Number(event.target.value))} + /> + {help &&

{help}

} +
+ ); +}; + +export const ForecastSolverModels = ({ + value, + onChange, + modelOptions, + effortOptionsByModel, + fastModeByModel, + additionalPoolsOnly = false, +}: Props & { fastModeByModel: Record; additionalPoolsOnly?: boolean }) => { + const id = React.useId(); + const names = forecastTierNames(value); + const additionalRows = + value.classifier_type === "capability" + ? activeTierRows(value) + .filter((row) => !names.includes(row.id) && row.models.length > 0) + .map((row) => ({ tier: row.id, label: `${tierRowLabel(row, value.tier_labels)} routing pool` })) + : []; + const rows = additionalPoolsOnly + ? additionalRows + : names.map((tier, index) => ({ tier, label: index === 0 ? "Efficient solver" : "Capable solver" })); + if (rows.length === 0) return null; + return ( +
+ {rows.map(({ tier, label }) => { + const models = forecastModels(value.tiers, tier); + const setModels = (next: string[]) => onChange(setTierModels(value, tier, next)); + return ( +
+ + {value.classifier_type === "llm_v2" ? ( + setModels(model ? [model] : [])} + /> + ) : ( + + )} + [model, efforts ?? []]), + )} + paramsByModel={value.tier_model_params?.[tier] ?? {}} + fastModeByModel={fastModeByModel} + onFastModeChange={(model, enabled) => + onChange({ + ...value, + tier_model_params: setTierModelParam(value.tier_model_params, tier, model, [ + "speed", + enabled ? "fast" : undefined, + ]), + }) + } + onEffortChange={(model, effort) => + onChange({ + ...value, + tier_model_params: setTierModelReasoningEffort(value.tier_model_params, tier, model, effort), + }) + } + /> +
+ ); + })} + {!additionalPoolsOnly && ( +

+ Invalid forecasts and classifier failures route to the capable solver +

+ )} +
+ ); +}; + +const CalibrationFields = ({ + label, + value, + onChange, + bounded = false, +}: { + label: string; + bounded?: boolean; + value: { slope: number; intercept: number }; + onChange: (value: { slope: number; intercept: number }) => void; +}) => ( +
+ onChange({ ...value, slope })} + /> + onChange({ ...value, intercept })} + /> +
+); + +const emptyCoefficients = () => ({ slope: Number.NaN, intercept: Number.NaN }); + +const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel }: Props) => { + const id = React.useId(); + const isCapability = value.classifier_type === "capability"; + const capability = value.capability_classifier_config ?? newCapabilitySettings(); + const fuse = value.llm_v2_config ?? newFuseSettings(); + const config = isCapability ? capability : fuse; + const llm = value.classifier_llm_config ?? { model: "", timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS }; + const updateCapability = (next: CapabilitySettings) => onChange({ ...value, capability_classifier_config: next }); + const updateFuse = (next: FuseSettings) => onChange({ ...value, llm_v2_config: next }); + const updateTransport = (patch: { max_output_tokens?: number; response_format?: "json_schema" | "json_object" }) => + isCapability ? updateCapability({ ...capability, ...patch }) : updateFuse({ ...fuse, ...patch }); + const setCalibrationVersion = (version: string) => { + if (isCapability && capability.calibration) + updateCapability({ ...capability, calibration: { ...capability.calibration, version } }); + if (!isCapability && fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, version } }); + }; + const error = getForecastConfigError(value); + return ( +
+

+ {isCapability + ? "Forecasts whether the efficient solver can complete the task using the bundled capability card" + : "Forecasts success for both solvers and selects efficient when the estimated quality gap is within your allowance"} +

+
+ + { + if (model === llm.model) return; + onChange({ ...value, classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined } }); + }} + /> +
+ {isCapability ? ( + <> + updateCapability({ ...capability, base_threshold })} + /> + + ) : ( + <> + {(["efficient_profile", "capable_profile", "harness"] as const).map((field) => { + const label = { + efficient_profile: "Efficient solver profile", + capable_profile: "Capable solver profile", + harness: "Harness and budget", + }[field]; + return ( +
+ +