diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 08b0281b30f..7b88f23349a 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -9,7 +9,6 @@ fi suite="${1:?integration suite required}" results="test-results/integration-${suite}" mkdir -p "$results" -shard_timeout=11m integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')" upstream_pid="" proxy_pid="" @@ -131,6 +130,7 @@ start_proxy() { fi setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \ DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ + INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \ LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \ LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \ AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ @@ -176,7 +176,7 @@ if [ "$suite" = browser ]; then exit 0 fi -timeout --signal=TERM --kill-after=20s "$shard_timeout" env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ +env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ INTEGRATION_RUN_ID="$integration_identity" \ DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \ diff --git a/.github/workflows/compat-matrix-image.yml b/.github/workflows/compat-matrix-image.yml new file mode 100644 index 00000000000..c554096cd7e --- /dev/null +++ b/.github/workflows/compat-matrix-image.yml @@ -0,0 +1,33 @@ +name: Compat Matrix Image + +on: + pull_request: + paths: + - tests/e2e/claude_code/cron_vm/** + - .github/workflows/compat-matrix-image.yml + workflow_dispatch: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + compat-matrix-image: + name: compat-matrix-image + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Build the Render cron image + run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e + + - name: Run the pinned binaries as the cron user + run: | + docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version' diff --git a/.github/workflows/issue_fixed_comment.yml b/.github/workflows/issue_fixed_comment.yml index 92993d319a7..b98a86c4dfa 100644 --- a/.github/workflows/issue_fixed_comment.yml +++ b/.github/workflows/issue_fixed_comment.yml @@ -6,8 +6,12 @@ on: workflow_dispatch: inputs: issue_number: - description: "Closed issue number to comment on manually." - required: true + description: "Closed issue number to comment on and close the superseded pull requests of. Ignored by a sweep." + required: false + sweep: + description: "Close every open pull request whose linked issues were all fixed on the default branch. Reads every open pull request, so run it at most once an hour." + type: boolean + default: false pull_request: paths: - .github/workflows/issue_fixed_comment.yml @@ -39,16 +43,17 @@ jobs: with: bun-version: "1.4.0" - - name: Test the closer lookup, the release placement and the comment + - name: Test the closer lookup, the release placement, the comment and the superseded pull request close run: bun test scripts/comment-fixed-issue.test.ts comment-fixed-issue: if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 15 permissions: contents: read issues: write + pull-requests: write steps: - name: Checkout scripts uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 @@ -59,13 +64,16 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: - # Exact version, never latest: the next step holds an issues: write token + # Exact version, never latest: the next step holds issues: write and pull-requests: write tokens bun-version: "1.4.0" - - name: Name the release that carries the fix + - name: Name the release that carries the fix and close the pull requests it supersedes + shell: bash run: bun run scripts/comment-fixed-issue.ts | tee -a "${GITHUB_STEP_SUMMARY}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }} + SWEEP: ${{ github.event.inputs.sweep }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} DRY_RUN: ${{ vars.ISSUE_FIXED_COMMENT_ENABLED != 'true' }} + CLOSE_PRS_DRY_RUN: ${{ vars.ISSUE_FIXED_CLOSE_PRS_ENABLED != 'true' }} diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 4bf41dc249c..64e16b512e1 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -105,6 +105,16 @@ jobs: with: python-version: "3.12" + - uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install Python dependencies for the bridge tests + working-directory: . + run: | + uv sync --frozen --no-install-project + echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV" + - run: rustup toolchain install --no-self-update - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 diff --git a/Dockerfile b/Dockerfile index 759dac76795..4dcecf3ea3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/backend/Dockerfile b/backend/Dockerfile index 57e0a43a98d..59f836b55f8 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index b0bf935c616..61b6faae691 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 5d729046678..d4c07d56d90 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 729f3264706..8509600ad96 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.69" +version = "0.1.70" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.69" +version = "0.1.70" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 33d3791dbba..8045a8b64cb 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) ARG PGBOUNCER_VERSION=1.25.2 diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index fb9022f89a5..e9a4ff90b9e 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.100" +version = "0.4.101" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.100" +version = "0.4.101" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4427057a3b9..332d8b3dcf0 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2946,6 +2946,7 @@ name = "litellm-core-utils" version = "0.1.0" dependencies = [ "fancy-regex 0.19.2", + "litellm-tracing", "litellm-types", "rstest", "serde", @@ -3075,6 +3076,7 @@ dependencies = [ "aws-sdk-secretsmanager", "bytes", "criterion", + "fancy-regex 0.19.2", "futures-util", "litellm-auth", "litellm-auth-aws", @@ -3093,6 +3095,7 @@ dependencies = [ "litellm-callbacks-legacy-python", "litellm-core", "litellm-core-utils", + "litellm-host", "litellm-host-python", "litellm-http", "litellm-llms", @@ -3100,6 +3103,7 @@ dependencies = [ "litellm-secrets-aws", "litellm-secrets-types", "litellm-token-counter", + "litellm-tracing", "litellm-types", "pyo3", "pyo3-async-runtimes", @@ -3172,11 +3176,11 @@ dependencies = [ "litellm-auth-aws", "litellm-core-utils", "litellm-secrets-types", + "litellm-tracing", "rstest", "serde_json", "thiserror 2.0.19", "tokio", - "tracing", "veil", "wiremock", ] @@ -3208,6 +3212,7 @@ dependencies = [ "base64 0.22.1", "litellm-core-utils", "litellm-secrets-types", + "litellm-tracing", "moka", "percent-encoding", "reqwest 0.12.28", @@ -3216,7 +3221,6 @@ dependencies = [ "serde_json", "thiserror 2.0.19", "tokio", - "tracing", "veil", "wiremock", ] @@ -3331,6 +3335,19 @@ dependencies = [ "tiktoken-rs", ] +[[package]] +name = "litellm-tracing" +version = "0.1.0" +dependencies = [ + "fancy-regex 0.19.2", + "percent-encoding", + "rstest", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "litellm-types" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 65813a35214..9d05c8d2b98 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -9,6 +9,8 @@ license = "MIT" repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] +litellm-tracing = { path = "crates/tracing" } +tracing = "0.1" litellm-core = { path = "crates/core" } litellm-host = { path = "crates/host" } litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } diff --git a/litellm-rust/crates/core-utils/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml index eb353bc060c..baf5dd16707 100644 --- a/litellm-rust/crates/core-utils/Cargo.toml +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] fancy-regex.workspace = true +litellm-tracing.workspace = true litellm-types.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/core-utils/src/secret_redaction.rs b/litellm-rust/crates/core-utils/src/secret_redaction.rs index e3caee4799a..14a88eae288 100644 --- a/litellm-rust/crates/core-utils/src/secret_redaction.rs +++ b/litellm-rust/crates/core-utils/src/secret_redaction.rs @@ -1,109 +1 @@ -use fancy_regex::Regex; - -pub const REDACTED: &str = "REDACTED"; - -const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16; - -fn minimum_custom_key_length() -> usize { - std::env::var("MINIMUM_CUSTOM_KEY_LENGTH") - .ok() - .and_then(|value| value.trim().parse().ok()) - .unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH) -} - -fn secret_patterns(minimum_custom_key_length: usize) -> String { - let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len()); - [ - r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", - r"\bya29\.[A-Za-z0-9_.~+/-]+", - r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#, - r"(?:AKIA|ASIA)[0-9A-Z]{16}", - r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", - r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", - &format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"), - r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#, - r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#, - r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, - r"x-ak-[A-Za-z0-9\-_]{20,}", - r"AIza[0-9A-Za-z\-_]{35}", - r#"(?<=[?&])key=[^\s&'"]{8,}"#, - r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, - r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#, - r"dapi[0-9a-f]{32}", - r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, - r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#, - concat!( - r"(?:master_key|xai_key|database_url|db_url|connection_string|", - r"aws_secret_access_key|aws_session_token|aws_access_key_id|", - r"signing_key|encryption_key|", - r"auth_token|access_token|refresh_token|", - r"slack_webhook_url|webhook_url|", - r"database_connection_string|", - r"huggingface_token|jwt_secret)", - r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, - ), - r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", - r"(?<=[?&])sig=[A-Za-z0-9%+/=]+", - r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#, - ] - .join("|") -} - -/// Python's `_ENABLE_SECRET_REDACTION` pattern set, compiled once per configuration. -#[derive(Clone, Debug)] -pub struct SecretRedactor { - pattern: Regex, -} - -impl SecretRedactor { - pub fn new(minimum_custom_key_length: usize) -> Self { - let pattern = Regex::new(&format!( - "(?i){}", - secret_patterns(minimum_custom_key_length) - )) - .expect("secret redaction patterns compile"); - Self { pattern } - } - - /// `None` when `LITELLM_DISABLE_REDACT_SECRETS` turns redaction off. - pub fn from_env() -> Option { - let disabled = std::env::var("LITELLM_DISABLE_REDACT_SECRETS") - .is_ok_and(|value| value.eq_ignore_ascii_case("true")); - (!disabled).then(|| Self::new(minimum_custom_key_length())) - } - - pub fn redact(&self, value: &str) -> String { - self.pattern.replace_all(value, REDACTED).into_owned() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[rstest::rstest] - #[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")] - #[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")] - #[case::short_sk_key_is_kept("sk-abc", "sk-abc")] - #[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")] - #[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")] - #[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")] - #[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")] - #[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")] - #[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")] - #[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")] - #[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)] - fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) { - assert_eq!( - SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input), - expected - ); - } - - #[test] - fn sk_threshold_follows_the_minimum_custom_key_length() { - let redactor = SecretRedactor::new(8); - assert_eq!(redactor.redact("sk-abcde"), REDACTED); - assert_eq!(redactor.redact("sk-abcd"), "sk-abcd"); - } -} +pub use litellm_tracing::{REDACTED, SecretRedactor}; diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index dd0975219e4..85e6b9588dd 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -19,6 +19,9 @@ huggingface = ["litellm-token-counter/huggingface"] tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] +fancy-regex.workspace = true +litellm-tracing.workspace = true +litellm-host.workspace = true bytes.workspace = true futures-util.workspace = true litellm-cache.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/activation.rs b/litellm-rust/crates/python-bridge/src/cache/activation.rs index 8d2c340dec8..f77032c579d 100644 --- a/litellm-rust/crates/python-bridge/src/cache/activation.rs +++ b/litellm-rust/crates/python-bridge/src/cache/activation.rs @@ -1,6 +1,7 @@ +use crate::logger::run_sync_value; use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig}; use litellm_cache_redis_semantic::RedisSemanticConfig; -use litellm_host_python::{release_gil, run_sync_value}; +use litellm_host_python::release_gil; use litellm_http::ClientVariant; use pyo3::prelude::*; diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs index 56f1bff2853..fcf71e10427 100644 --- a/litellm-rust/crates/python-bridge/src/cache/binding.rs +++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs @@ -1,5 +1,6 @@ +use crate::logger::run_async; use litellm_cache_response::PartialHits; -use litellm_host_python::{ExecutionStep, from_py, release_gil, run_async, to_py}; +use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py}; use pyo3::{ PyTraverseError, PyVisit, exceptions::{PyRuntimeError, PyValueError}, diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs index dff8a771a23..e14916b25c6 100644 --- a/litellm-rust/crates/python-bridge/src/cache/handle.rs +++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs @@ -1,10 +1,11 @@ +use crate::logger::run_sync_value; use litellm_auth_aws::AwsAuthConfig; use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig}; use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization}; use litellm_cache_redis::{RedisNode, RedisTopology}; use litellm_cache_redis_semantic::RedisSemanticConfig; use litellm_cache_s3::{S3CacheConfig, S3Endpoint}; -use litellm_host_python::{release_gil, run_sync_value}; +use litellm_host_python::release_gil; use litellm_http::ClientVariant; use pyo3::{ PyTraverseError, PyVisit, diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 3c260e70843..0e279046812 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -470,7 +470,7 @@ impl NativeResponseCache { match self { Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); - litellm_host_python::run_async( + crate::logger::run_async( py, async move { service @@ -495,7 +495,7 @@ impl NativeResponseCache { match self { Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); - litellm_host_python::run_async( + crate::logger::run_async( py, async move { service.async_lookup(&request, now()).await }, super::cache_error, @@ -550,7 +550,7 @@ impl NativeResponseCache { match self { Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); - litellm_host_python::run_async( + crate::logger::run_async( py, async move { service.async_store(&request, response, now()).await }, super::cache_error, @@ -619,7 +619,7 @@ impl NativeResponseCache { match self { Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); - litellm_host_python::run_async( + crate::logger::run_async( py, async move { service.async_store_batch(entries, now()).await }, super::cache_error, diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic.rs b/litellm-rust/crates/python-bridge/src/cache/semantic.rs index bea58b88885..eb67f5594e2 100644 --- a/litellm-rust/crates/python-bridge/src/cache/semantic.rs +++ b/litellm-rust/crates/python-bridge/src/cache/semantic.rs @@ -1,7 +1,8 @@ +use crate::logger::run_async; use std::{collections::VecDeque, time::Duration}; use litellm_cache::Error; -use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; +use litellm_host_python::{Execution, ExecutionBody, ExecutionStep}; use pyo3::{ PyTraverseError, PyVisit, exceptions::{PyException, PyRuntimeError}, diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 2515b409c54..3dad3447f45 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -102,7 +102,7 @@ pub(crate) fn call_config( .without_missing_files(&|path: &Path| path.exists()); let resolution = Resolution::from(&settings); for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { - PythonSettings::warn(py, &unsupported.to_string())?; + crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}")); } Ok(resolution.config) } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index b1fc5244d6f..76de2ae2b7b 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -4,6 +4,7 @@ mod credentials; mod diagnostics; mod errors; mod http; +mod logger; mod marshal; mod python_settings; mod routes; @@ -25,6 +26,8 @@ mod _native { #[pymodule_export] use crate::errors::{RustBridgeDeclined, RustUpstreamError}; #[pymodule_export] + use crate::logger::NativeDiagnosticProcessor; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{ @@ -87,6 +90,7 @@ mod tests { "chat_completions", "achat_completions", "ResponsesWebSocketConnection", + "NativeDiagnosticProcessor", "TokenCounter", "Tokenizer", "gil_stats", diff --git a/litellm-rust/crates/python-bridge/src/logger/execution.rs b/litellm-rust/crates/python-bridge/src/logger/execution.rs new file mode 100644 index 00000000000..c8d5c0023e3 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/logger/execution.rs @@ -0,0 +1,46 @@ +use std::future::Future; + +use pyo3::prelude::*; +use serde::Serialize; + +pub(crate) fn run_sync( + py: Python<'_>, + future: F, + map_error: fn(E) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + E: Send + 'static, + F: Future> + Send + 'static, +{ + litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error) +} + +pub(crate) fn run_async( + py: Python<'_>, + future: F, + map_error: fn(E) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + E: Send + 'static, + F: Future> + Send + 'static, +{ + litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error) +} + +pub(crate) fn run_sync_value(py: Python<'_>, future: F) -> PyResult +where + T: Send + 'static, + F: Future> + Send + 'static, +{ + litellm_host_python::run_sync_value(py, super::capture(py).instrument(future)) +} + +pub(crate) fn run_async_value(py: Python<'_>, future: F) -> PyResult> +where + T: for<'py> IntoPyObject<'py> + Send + 'static, + F: Future> + Send + 'static, +{ + litellm_host_python::run_async_value(py, super::capture(py).instrument(future)) +} diff --git a/litellm-rust/crates/python-bridge/src/logger/machine.rs b/litellm-rust/crates/python-bridge/src/logger/machine.rs new file mode 100644 index 00000000000..7234308e67e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/logger/machine.rs @@ -0,0 +1,41 @@ +use std::sync::OnceLock; + +use litellm_host::{ + host::HostResult, + machine::{HostFailure, Interrupted, Machine, Step}, + route::Route, +}; +use litellm_tracing::Logger; +use pyo3::Python; + +pub(crate) struct LoggedMachine { + machine: M, + logger: OnceLock, +} + +impl LoggedMachine { + pub(crate) fn new(machine: M) -> Self { + Self { + machine, + logger: OnceLock::new(), + } + } +} + +impl Machine for LoggedMachine { + type Route = M::Route; + type Complete = M::Complete; + + fn resume(&mut self, result: Option>) -> Step<'_, Self> { + let logger = self.logger.get_or_init(|| Python::attach(super::capture)); + Box::pin(logger.instrument(logger.scope(|| self.machine.resume(result)))) + } + + fn interrupt( + &mut self, + failure: HostFailure<::Error>, + ) -> Interrupted<'_, Self> { + let logger = self.logger.get_or_init(|| Python::attach(super::capture)); + Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure)))) + } +} diff --git a/litellm-rust/crates/python-bridge/src/logger/mod.rs b/litellm-rust/crates/python-bridge/src/logger/mod.rs new file mode 100644 index 00000000000..6421fe1d554 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/logger/mod.rs @@ -0,0 +1,166 @@ +mod execution; +mod machine; + +pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value}; +pub(crate) use machine::LoggedMachine; + +use litellm_host_python::Pythonized; +use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; + +const MODULE: &str = "litellm.rust_bridge.logger"; +type NativeDiagnosticOutput = (String, Option, Option, Vec, bool); + +#[pyclass] +pub(crate) struct NativeDiagnosticProcessor { + inner: Processor, +} + +#[pymethods] +impl NativeDiagnosticProcessor { + #[new] + fn new(minimum_custom_key_length: usize) -> Self { + Self { + inner: Processor::new(minimum_custom_key_length), + } + } + + fn redact_text(&self, text: &str) -> PyResult { + self.inner.redact_text(text).map_err(processing_error) + } + + fn redact_structured_text(&self, key: Option<&str>, text: &str) -> PyResult { + self.inner + .redact_structured_text(key, text) + .map_err(processing_error) + } + + fn redact_client_message(&self, text: &str) -> PyResult { + self.inner + .redact_client_message(text) + .map_err(processing_error) + } + + #[pyo3(signature = (message, exception, stack, leaves, policy))] + fn process_diagnostic( + &self, + message: String, + exception: Option, + stack: Option, + leaves: Vec<(Option, String)>, + policy: (bool, i64, i64), + ) -> PyResult { + let input = DiagnosticInput { + message, + exception, + stack, + leaves, + }; + let policy = Policy { + redact: policy.0, + base64_limit: policy.1, + text_limit: policy.2, + }; + self.inner + .process_diagnostic(&input, policy) + .map(|output| { + ( + output.message, + output.exception, + output.stack, + output.leaves, + output.changed, + ) + }) + .map_err(processing_error) + } + + fn scrub_access_arguments(&self, arguments: Vec) -> PyResult> { + self.inner + .scrub_access_arguments(&arguments) + .map_err(processing_error) + } +} + +fn processing_error(_: fancy_regex::Error) -> PyErr { + PyRuntimeError::new_err("diagnostic processing failed") +} + +struct PythonSink { + correlation: (String, String), +} + +fn level(level: &Level) -> u8 { + match *level { + Level::ERROR => 40, + Level::WARN => 30, + Level::INFO => 20, + Level::DEBUG | Level::TRACE => 10, + } +} + +fn report(py: Python<'_>, result: PyResult) -> T { + match result { + Ok(value) => value, + Err(error) => { + error.write_unraisable(py, None); + T::default() + } + } +} + +impl Sink for PythonSink { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + if !metadata.target().starts_with("litellm_") && !metadata.target().starts_with("_native::") + { + return false; + } + Python::try_attach(|py| { + report( + py, + py.import(MODULE) + .and_then(|module| module.call_method1("enabled", (level(metadata.level()),))) + .and_then(|enabled| enabled.extract()), + ) + }) + .unwrap_or(false) + } + + fn emit(&self, record: &Record) { + Python::try_attach(|py| { + report( + py, + py.import(MODULE).and_then(|module| { + module + .call_method1( + "emit", + ( + level(record.metadata.level()), + &record.message, + record.metadata.file().unwrap_or_default(), + record.metadata.line().unwrap_or_default(), + record.metadata.target(), + Pythonized(&record.fields), + (&self.correlation.0, &self.correlation.1), + ), + ) + .map(|_| ()) + }), + ); + }); + } +} + +pub(crate) fn capture(py: Python<'_>) -> Logger { + report( + py, + py.import(MODULE) + .and_then(|module| module.call_method0("context")) + .and_then(|value| value.extract()) + .map(|correlation| Logger::new(PythonSink { correlation })), + ) +} + +#[cfg(test)] +mod tests; diff --git a/litellm-rust/crates/python-bridge/src/logger/tests.rs b/litellm-rust/crates/python-bridge/src/logger/tests.rs new file mode 100644 index 00000000000..9312d4c187c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/logger/tests.rs @@ -0,0 +1,295 @@ +use std::{process::Command, task::Poll}; + +use litellm_host::{ + host::HostResult, + machine::{HostFailure, Interrupted, Machine, MachineStep, Step}, + route::Route, +}; + +use pyo3::{prelude::*, types::PyDict}; + +struct DiagnosticMachine; + +impl Route for DiagnosticMachine { + type Response = (); + type Error = String; + type Op = (); + type OpResult = (); + type Chunk = (); + type StreamHead = (); +} + +impl Machine for DiagnosticMachine { + type Route = Self; + type Complete = (); + + fn resume(&mut self, _: Option>) -> Step<'_, Self> { + litellm_tracing::warn!("machine started"); + Box::pin(async { + tokio::task::yield_now().await; + litellm_tracing::warn!("machine warning"); + Ok(MachineStep::Complete(())) + }) + } + + fn interrupt(&mut self, _: HostFailure) -> Interrupted<'_, Self> { + Box::pin(async { + litellm_tracing::warn!("machine interrupted"); + Ok(()) + }) + } +} + +#[pyfunction] +fn machine_warning(py: Python<'_>) -> PyResult> { + let mut machine = super::LoggedMachine::new(DiagnosticMachine); + let mut future = Box::pin(async move { + machine + .resume(None) + .await + .map_err(pyo3::exceptions::PyValueError::new_err)?; + machine + .interrupt(HostFailure::Error("stop".into())) + .await + .map_err(pyo3::exceptions::PyValueError::new_err) + }); + assert!(matches!( + litellm_host_python::poll_async_value(py, future.as_mut())?, + Poll::Pending + )); + litellm_host_python::run_async_value(py, future) +} + +#[pyfunction] +fn warning(py: Python<'_>) { + super::capture(py).scope(|| { + litellm_tracing::warn!(attempt = 3, retry = true, "native warning"); + }); +} + +#[pyfunction] +fn levels(py: Python<'_>) { + super::capture(py).scope(|| { + litellm_tracing::trace!("trace"); + litellm_tracing::debug!("debug"); + litellm_tracing::info!("info"); + litellm_tracing::warn!("warn"); + litellm_tracing::error!("error"); + litellm_tracing::warn!(target: "unrelated_transport", "private wire data"); + }); +} + +#[pyfunction] +fn asynchronous_warning(py: Python<'_>) -> PyResult> { + super::run_async_value(py, async { + tokio::task::yield_now().await; + litellm_tracing::warn!("async warning"); + Ok(()) + }) +} + +#[pyfunction] +fn synchronous_warning(py: Python<'_>) -> PyResult<()> { + super::run_sync_value(py, async { + tokio::task::yield_now().await; + litellm_tracing::warn!("sync warning"); + Ok(()) + }) +} + +#[pyfunction] +fn synchronous_failure(py: Python<'_>) -> PyResult<()> { + super::run_sync_value(py, async { + litellm_tracing::warn!("failure diagnostic"); + Err(pyo3::exceptions::PyValueError::new_err("request failed")) + }) +} + +#[pyfunction] +fn http_warning(py: Python<'_>) -> PyResult<()> { + crate::http::call_config(py, &PyDict::new(py), false).map(|_| ()) +} + +#[test] +fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() { + if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + std::thread::current().name().unwrap(), + "--nocapture", + ]) + .env("LITELLM_LOGGER_TEST_PROCESS", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals + .set_item( + "repo_root", + concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."), + ) + .unwrap(); + locals + .set_item( + "machine_warning", + wrap_pyfunction!(machine_warning, py).unwrap(), + ) + .unwrap(); + locals + .set_item( + "synchronous_failure", + wrap_pyfunction!(synchronous_failure, py).unwrap(), + ) + .unwrap(); + locals + .set_item("levels", wrap_pyfunction!(levels, py).unwrap()) + .unwrap(); + locals + .set_item("warning", wrap_pyfunction!(warning, py).unwrap()) + .unwrap(); + locals + .set_item( + "asynchronous_warning", + wrap_pyfunction!(asynchronous_warning, py).unwrap(), + ) + .unwrap(); + locals + .set_item( + "synchronous_warning", + wrap_pyfunction!(synchronous_warning, py).unwrap(), + ) + .unwrap(); + locals + .set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap()) + .unwrap(); + let importable = py + .eval( + c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None", + Some(&locals), + Some(&locals), + ) + .unwrap() + .is_truthy() + .unwrap(); + if !importable { + eprintln!("SKIP: litellm package dependencies are not importable in this interpreter"); + return; + } + py.run(c" +import asyncio +import logging +import sys +sys.path.insert(0, repo_root) +import litellm +from litellm._logging import verbose_logger, session_id_var, trace_id_var + +class Capture(logging.Handler): + def __init__(self): + super().__init__() + self.records = [] + def emit(self, record): + self.records.append(record) + warning() + +class Broken(logging.Handler): + def emit(self, record): + raise ValueError('handler failed') + +capture = Capture() +old_handlers = verbose_logger.handlers +old_level = verbose_logger.level +old_correlation = litellm.request_correlation_in_logs +old_curve = litellm.ssl_ecdh_curve +old_unraisable = sys.unraisablehook +failures = [] +try: + verbose_logger.handlers = [capture] + litellm.request_correlation_in_logs = True + verbose_logger.setLevel(logging.ERROR) + warning() + assert capture.records == [] + verbose_logger.setLevel(logging.WARNING) + warning() + assert len(capture.records) == 1 + record = capture.records[0] + assert record.getMessage() == 'native warning' + assert record.levelno == logging.WARNING + assert record.rust_fields == {'attempt': 3, 'retry': True} + assert record.pathname.endswith('logger/tests.rs') + assert record.lineno > 0 + assert record.rust_target.endswith('logger::tests') + verbose_logger.setLevel(logging.ERROR) + warning() + assert len(capture.records) == 1 + verbose_logger.setLevel(logging.WARNING) + + async def request(name): + session = session_id_var.set(name) + trace = trace_id_var.set('trace-' + name) + try: + await asynchronous_warning() + await machine_warning() + synchronous_warning() + assert session_id_var.get() == name + assert trace_id_var.get() == 'trace-' + name + finally: + trace_id_var.reset(trace) + session_id_var.reset(session) + + async def concurrent(): + await asyncio.gather(request('first'), request('second')) + + asyncio.run(concurrent()) + assert sorted((r.getMessage(), r.session_id, r.trace_id) for r in capture.records[1:]) == sorted( + (message, name, 'trace-' + name) + for name in ('first', 'second') + for message in ('async warning', 'sync warning', 'machine started', 'machine warning', 'machine interrupted') + ) + + verbose_logger.setLevel(logging.DEBUG) + before_levels = len(capture.records) + levels() + assert [(r.getMessage(), r.levelno) for r in capture.records[before_levels:]] == [ + ('trace', logging.DEBUG), ('debug', logging.DEBUG), ('info', logging.INFO), + ('warn', logging.WARNING), ('error', logging.ERROR), + ] + + before = len(capture.records) + litellm.ssl_ecdh_curve = 'logger-test-unsupported-curve' + http_warning() + http_warning() + assert len(capture.records) == before + 1 + assert 'logger-test-unsupported-curve' in capture.records[-1].getMessage() + assert capture.records[-1].pathname.endswith('http.rs') + + verbose_logger.handlers = [Broken()] + sys.unraisablehook = failures.append + warning() + assert len(failures) == 1 + assert str(failures[0].exc_value) == 'handler failed' + try: + synchronous_failure() + except ValueError as error: + assert str(error) == 'request failed' + else: + raise AssertionError('request failure was lost') + assert len(failures) == 2 +finally: + sys.unraisablehook = old_unraisable + verbose_logger.handlers = old_handlers + verbose_logger.setLevel(old_level) + litellm.request_correlation_in_logs = old_correlation + litellm.ssl_ecdh_curve = old_curve +", Some(&locals), Some(&locals)).unwrap(); + }); +} diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index 111ac3bc259..abf664b795d 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -44,11 +44,6 @@ impl PythonSettings { pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> { Snapshot { group: self, value } } - - pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> { - py.import(MODULE)?.getattr("warn")?.call1((message,))?; - Ok(()) - } } #[cfg(test)] diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index d63e9a1feaf..dec4dcea21c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -1,7 +1,8 @@ +use crate::logger::{run_async, run_sync}; use litellm_core::audio_transcription::{ Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest, }; -use litellm_host_python::{from_py_argument, run_async, run_sync}; +use litellm_host_python::from_py_argument; use pyo3::prelude::*; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index 049a507dcdc..1fa2ca00c42 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -1,8 +1,9 @@ +use crate::logger::{run_async, run_sync}; use litellm_core::chat_completions::{ Error, chat_completions as run_chat_completions, chat_completions_decline_reason, types::ChatCompletionsRequest, }; -use litellm_host_python::{from_py_argument, run_async, run_sync}; +use litellm_host_python::from_py_argument; use litellm_types::utils::ChatCompletionsResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index b606293f79f..804d883e9ae 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -43,7 +43,7 @@ fn run_messages( py, SURFACE, PublicCall::capture(&request, &args, &kwargs)?, - messages_machine(), + crate::logger::LoggedMachine::new(messages_machine()), MessagesRouteHost::new(request.unbind()), asynchronous, ) diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 2dca6da66cd..ca5abf1f80e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -73,7 +73,7 @@ fn run_ocr( py, if asynchronous { ASYNC_SURFACE } else { SURFACE }, PublicCall::capture(&request, &args, &kwargs)?, - ocr_machine(client), + crate::logger::LoggedMachine::new(ocr_machine(client)), OcrRouteHost::new(request.unbind()), asynchronous, ) diff --git a/litellm-rust/crates/python-bridge/src/routes/responses.rs b/litellm-rust/crates/python-bridge/src/routes/responses.rs index 2e7e8fcbc21..ffbb945c415 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses.rs @@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection { ) -> PyResult> { let headers = marshal_headers(headers)?; let timeout = optional_timeout(timeout_seconds); - litellm_host_python::run_async_value(py, async move { + crate::logger::run_async_value(py, async move { let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout) .await .map_err(responses_error_to_pyerr)?; @@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection { fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult> { let inner = self.inner.clone(); - litellm_host_python::run_async_value(py, async move { + crate::logger::run_async_value(py, async move { inner .send_text(text) .await @@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection { fn recv_text<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); - litellm_host_python::run_async_value(py, async move { + crate::logger::run_async_value(py, async move { inner.recv_text().await.map_err(responses_error_to_pyerr) }) } fn close<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); - litellm_host_python::run_async_value(py, async move { + crate::logger::run_async_value(py, async move { inner.close().await.map_err(responses_error_to_pyerr) }) } diff --git a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs index 168c4883b0b..21589aa3fe9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs @@ -1,7 +1,8 @@ +use crate::logger::run_async; use std::sync::Arc; use std::{num::NonZero, thread::available_parallelism}; -use litellm_host_python::{enter_native, run_async}; +use litellm_host_python::enter_native; use litellm_token_counter::{ CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, }; diff --git a/litellm-rust/crates/secrets-aws/Cargo.toml b/litellm-rust/crates/secrets-aws/Cargo.toml index b1dc5b33cda..9c926bb441b 100644 --- a/litellm-rust/crates/secrets-aws/Cargo.toml +++ b/litellm-rust/crates/secrets-aws/Cargo.toml @@ -11,7 +11,7 @@ litellm-secrets-types.workspace = true litellm-core-utils.workspace = true serde_json.workspace = true thiserror.workspace = true -tracing = "0.1" +litellm-tracing.workspace = true veil.workspace = true aws-sdk-kms = "1.120.0" aws-sdk-secretsmanager = "1.117.0" diff --git a/litellm-rust/crates/secrets-aws/src/secret_manager.rs b/litellm-rust/crates/secrets-aws/src/secret_manager.rs index 76c61534861..5658b6d7e11 100644 --- a/litellm-rust/crates/secrets-aws/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/src/secret_manager.rs @@ -215,7 +215,7 @@ impl AwsSecretsManagerV2 { .await .is_err() { - tracing::warn!("secret created but replication failed"); + litellm_tracing::warn!("secret created but replication failed"); } Ok(response) } diff --git a/litellm-rust/crates/secrets-cyberark/Cargo.toml b/litellm-rust/crates/secrets-cyberark/Cargo.toml index 3c1159c40be..c1613f96403 100644 --- a/litellm-rust/crates/secrets-cyberark/Cargo.toml +++ b/litellm-rust/crates/secrets-cyberark/Cargo.toml @@ -14,7 +14,7 @@ reqwest.workspace = true serde_json.workspace = true thiserror.workspace = true veil.workspace = true -tracing = "0.1" +litellm-tracing.workspace = true percent-encoding = "2.3" tokio = { workspace = true, features = ["sync"] } diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs index 80e6fb12f9e..dd36cf183d8 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs @@ -92,7 +92,7 @@ impl CyberArkSecretManager { .unwrap_or(true); let mut builder = reqwest::Client::builder(); if !verify { - tracing::warn!( + litellm_tracing::warn!( "CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates." ); builder = builder.danger_accept_invalid_certs(true); @@ -259,11 +259,13 @@ impl CyberArkSecretManager { .endpoint .join(&format!("policies/{}/policy/root", self.account)); let Ok(policy_url) = policy_url else { - tracing::warn!("Could not build CyberArk policy endpoint"); + litellm_tracing::warn!("Could not build CyberArk policy endpoint"); return; }; let Ok(authorization) = self.authorization_header(context).await else { - tracing::warn!("Could not authenticate while ensuring CyberArk variable exists"); + litellm_tracing::warn!( + "Could not authenticate while ensuring CyberArk variable exists" + ); return; }; let body = format!( @@ -288,19 +290,19 @@ impl CyberArkSecretManager { reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY ) => { - tracing::debug!( + litellm_tracing::debug!( "CyberArk variable policy already exists or conflicts: {}", response.status() ); } Ok(response) => { - tracing::warn!( + litellm_tracing::warn!( "Could not ensure CyberArk variable exists: {}", response.status() ); } Err(error) => { - tracing::warn!("Error ensuring CyberArk variable exists: {error}"); + litellm_tracing::warn!("Error ensuring CyberArk variable exists: {error}"); } } } @@ -324,7 +326,7 @@ impl CyberArkSecretManager { _recovery_window_in_days: Option, _context: &SecretOperationContext, ) -> Result { - tracing::warn!( + litellm_tracing::warn!( "CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates." ); self.secrets.invalidate(name).await; diff --git a/litellm-rust/crates/tracing/Cargo.toml b/litellm-rust/crates/tracing/Cargo.toml new file mode 100644 index 00000000000..41ad20afb3e --- /dev/null +++ b/litellm-rust/crates/tracing/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "litellm-tracing" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +fancy-regex.workspace = true +percent-encoding.workspace = true +serde_json.workspace = true +tracing.workspace = true +tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/tracing/README.md b/litellm-rust/crates/tracing/README.md new file mode 100644 index 00000000000..7f086067f3e --- /dev/null +++ b/litellm-rust/crates/tracing/README.md @@ -0,0 +1,26 @@ +# Native diagnostic tracing + +`litellm-tracing` connects standard `tracing` events to a host-provided `Sink`. It has no Python dependency and does not install a global subscriber + +Use the exported `debug!`, `info!`, `warn!`, and `error!` macros in native code. A host creates a `Logger` with its sink, uses `scope` for synchronous operations, and wraps futures with `instrument`. Instrument spawned futures explicitly because thread-local subscribers do not automatically follow spawned work + +Bindings implement `litellm_tracing::Sink` to connect events to their host runtime: + +```rust +pub trait Sink: Send + Sync + 'static { + fn enabled(&self, metadata: &Metadata<'_>) -> bool; + fn emit(&self, record: &Record); +} +``` + +Pass the implementation to `litellm_tracing::Logger::new(sink)`, then call `logger.scope(|| litellm_tracing::info!(attempt = 1, "request started"))`. The sink owns host access, level mapping, correlation capture, and delivery failures. `enabled` runs before event fields are evaluated or formatted. `emit` borrows a record; an adapter that queues delivery must copy the data it needs into an owned value + +Records retain event metadata, the message, and typed event fields. Sink filtering runs for each event so runtime level changes take effect. Logging from inside a sink is suppressed to prevent recursion + +The Python bridge scopes native execution to a sink that uses LiteLLM's existing Python logger. It preserves request correlation, redacts before delivering to handlers, maps Rust trace events to Python debug, and reports handler failures through `sys.unraisablehook`. It accepts LiteLLM targets only, keeping dependency wire diagnostics out of the application logger + +Python consumers continue using `litellm._logging` and its existing loggers, filters, formatters, and context setters. Catalog dispatch selects the processing backend for both Python and native diagnostics. The pure `Processor` takes explicit settings and never emits events + +A future Node bridge can implement the same sink with runtime-specific delivery and expose the same processor through N-API. Node callback scheduling, queue limits, and shutdown belong in that bridge; this crate has no interpreter handles or output queue + +This is diagnostic logging. Request lifecycle hooks and `CustomLogger` dispatch remain separate diff --git a/litellm-rust/crates/tracing/src/lib.rs b/litellm-rust/crates/tracing/src/lib.rs new file mode 100644 index 00000000000..47f97d6db27 --- /dev/null +++ b/litellm-rust/crates/tracing/src/lib.rs @@ -0,0 +1,146 @@ +use std::{ + cell::Cell, + fmt, + future::{Future, poll_fn}, + pin::pin, +}; + +use serde_json::{Map, Value}; +use tracing::{ + Dispatch, Event, Subscriber, + field::{Field, Visit}, + subscriber::Interest, +}; +use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*}; + +mod processing; +mod redaction; + +pub use processing::{DiagnosticInput, DiagnosticOutput, Policy, Processor}; +pub use redaction::{REDACTED, SecretRedactor}; +pub use tracing::{Level, Metadata, debug, error, info, trace, warn}; + +pub trait Sink: Send + Sync + 'static { + fn enabled(&self, metadata: &Metadata<'_>) -> bool; + fn emit(&self, record: &Record); +} + +#[derive(Debug)] +pub struct Record { + pub metadata: &'static Metadata<'static>, + pub message: String, + pub fields: Map, +} + +#[derive(Clone, Default)] +pub struct Logger { + dispatch: Dispatch, +} + +impl Logger { + pub fn new(sink: impl Sink) -> Self { + Self { + dispatch: Dispatch::new(Registry::default().with(Output(sink))), + } + } + + pub fn scope(&self, operation: impl FnOnce() -> T) -> T { + if EMITTING.get() { + return operation(); + } + tracing::dispatcher::with_default(&self.dispatch, operation) + } + + pub fn instrument(&self, future: F) -> impl Future + use { + let logger = self.clone(); + async move { + let mut future = pin!(future); + poll_fn(|context| logger.scope(|| future.as_mut().poll(context))).await + } + } +} + +thread_local! { + static EMITTING: Cell = const { Cell::new(false) }; +} + +struct Emitting; + +impl Emitting { + fn enter() -> Option { + EMITTING.with(|active| (!active.replace(true)).then_some(Self)) + } +} + +impl Drop for Emitting { + fn drop(&mut self) { + EMITTING.set(false); + } +} + +struct Output(S); + +impl Layer for Output { + fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest { + Interest::sometimes() + } + + fn enabled(&self, metadata: &Metadata<'_>, _: Context<'_, R>) -> bool { + let Some(_guard) = Emitting::enter() else { + return false; + }; + self.0.enabled(metadata) + } + + fn on_event(&self, event: &Event<'_>, _: Context<'_, R>) { + let Some(_guard) = Emitting::enter() else { + return; + }; + let mut record = Record { + metadata: event.metadata(), + message: String::new(), + fields: Map::new(), + }; + event.record(&mut record); + self.0.emit(&record); + } +} + +impl Record { + fn field(&mut self, field: &Field, value: Value) { + if field.name() == "message" { + self.message = match value { + Value::String(message) => message, + value => value.to_string(), + }; + } else { + self.fields.insert(field.name().to_owned(), value); + } + } +} + +impl Visit for Record { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + self.field(field, format!("{value:?}").into()); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.field(field, value.into()); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.field(field, value.into()); + } + + fn record_i64(&mut self, field: &Field, value: i64) { + self.field(field, value.into()); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.field(field, value.into()); + } + + fn record_f64(&mut self, field: &Field, value: f64) { + self.field(field, value.into()); + } +} diff --git a/litellm-rust/crates/tracing/src/processing.rs b/litellm-rust/crates/tracing/src/processing.rs new file mode 100644 index 00000000000..29f5f6c71ea --- /dev/null +++ b/litellm-rust/crates/tracing/src/processing.rs @@ -0,0 +1,333 @@ +use fancy_regex::Result; +use percent_encoding::percent_decode_str; + +use crate::{REDACTED, SecretRedactor}; + +#[derive(Clone, Copy, Debug)] +pub struct Policy { + pub redact: bool, + pub base64_limit: i64, + pub text_limit: i64, +} + +#[derive(Clone, Debug)] +pub struct DiagnosticInput { + pub message: String, + pub exception: Option, + pub stack: Option, + pub leaves: Vec<(Option, String)>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticOutput { + pub message: String, + pub exception: Option, + pub stack: Option, + pub leaves: Vec, + pub changed: bool, +} + +pub struct Processor { + redactor: SecretRedactor, +} + +impl Processor { + pub fn new(minimum_custom_key_length: usize) -> Self { + Self { + redactor: SecretRedactor::new(minimum_custom_key_length), + } + } + + pub fn redact_text(&self, text: &str) -> Result { + self.redactor.try_redact(text) + } + + pub fn redact_structured_text(&self, key: Option<&str>, text: &str) -> Result { + self.redactor.try_redact_structured(key, text) + } + + pub fn redact_client_message(&self, text: &str) -> Result { + self.redactor.try_redact_internal(text) + } + + pub fn process_diagnostic( + &self, + input: &DiagnosticInput, + policy: Policy, + ) -> Result { + let message = self.process_text(&input.message, policy)?; + let exception = input + .exception + .as_deref() + .map(|text| self.process_text(text, policy)) + .transpose()?; + let stack = input + .stack + .as_deref() + .map(|text| { + if policy.redact { + self.redact_text(text) + } else { + Ok(text.to_owned()) + } + }) + .transpose()?; + let leaves = input + .leaves + .iter() + .map(|(key, text)| { + if policy.redact { + self.redact_structured_text(key.as_deref(), text) + } else { + Ok(text.clone()) + } + }) + .collect::>>()?; + let changed = message != input.message + || exception != input.exception + || stack != input.stack + || leaves + .iter() + .zip(&input.leaves) + .any(|(processed, (_, original))| processed != original); + Ok(DiagnosticOutput { + message, + exception, + stack, + leaves, + changed, + }) + } + + pub fn scrub_access_arguments(&self, arguments: &[String]) -> Result> { + arguments + .iter() + .map(|argument| self.scrub_access_arg(argument)) + .collect() + } + + fn process_text(&self, text: &str, policy: Policy) -> Result { + let collapsed = if policy.base64_limit > 0 { + collapse_base64(text, policy.base64_limit as usize) + } else { + text.to_owned() + }; + let redacted = if policy.redact { + self.redact_text(&collapsed)? + } else { + collapsed + }; + Ok( + if policy.text_limit > 0 && redacted.chars().count() > policy.text_limit as usize { + truncate_text(&redacted, policy.text_limit as usize) + } else { + redacted + }, + ) + } + + fn scrub_access_arg(&self, value: &str) -> Result { + let length = value.chars().count(); + let scanned = if length <= 512 { + value + } else { + let head = &value[..char_offset(value, 512)]; + if head.contains('?') { + &head[..head.rfind(['?', '&']).unwrap_or(0)] + } else { + head + } + }; + let scrubbed = self.redact_text(scanned)?; + let (path, query) = scrubbed + .split_once('?') + .map_or((scrubbed.as_str(), None), |(path, query)| { + (path, Some(query)) + }); + let safe = if self.hides_encoded_credential(path)? { + REDACTED.to_owned() + } else if query.is_some() && self.hides_encoded_credential(&scrubbed)? { + format!("{path}?{REDACTED}") + } else { + scrubbed + }; + Ok(if length > 512 { + format!( + "{safe}... ({} more chars truncated) ...", + length - scanned.chars().count() + ) + } else { + safe + }) + } + + fn hides_encoded_credential(&self, value: &str) -> Result { + if !value.as_bytes().contains(&b'%') { + return Ok(false); + } + let decoded = percent_decode_str(value).decode_utf8_lossy(); + Ok(self.redact_text(&decoded)? != decoded) + } +} + +fn char_offset(text: &str, count: usize) -> usize { + text.char_indices() + .nth(count) + .map_or(text.len(), |(index, _)| index) +} + +fn marker(skipped_chars: usize) -> String { + format!( + "... (litellm_truncated skipped {skipped_chars} chars. Truncation is a stdout logging safeguard. Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env.) ..." + ) +} + +fn truncate_text(text: &str, limit: usize) -> String { + let length = text.chars().count(); + let kept = limit.saturating_sub(marker(length).len()); + if kept == 0 { + return text[..char_offset(text, limit)].to_owned(); + } + let head = kept / 2; + let tail = kept - head; + format!( + "{}{}{}", + &text[..char_offset(text, head)], + marker(length - kept), + &text[char_offset(text, length - tail)..] + ) +} + +fn base64_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'+' || byte == b'/' +} + +fn looks_like_base64(run: &str) -> bool { + let unpadded = run.trim_end_matches('='); + let lower_hex = unpadded + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)); + let upper_hex = unpadded + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte)); + let repeated = unpadded.bytes().all(|byte| byte == unpadded.as_bytes()[0]); + (!lower_hex && !upper_hex) || repeated +} + +fn base64_size(chars: usize) -> String { + let bytes = chars as f64 * 3.0 / 4.0; + if bytes >= 1024.0 * 1024.0 { + return format!("{:.2}MB", bytes / (1024.0 * 1024.0)); + } + if bytes >= 1024.0 { + return format!("{:.1}KB", bytes / 1024.0); + } + format!("{}B", bytes as usize) +} + +fn collapse_base64(text: &str, limit: usize) -> String { + let bytes = text.as_bytes(); + let mut position = 0; + let mut previous = 0; + let mut output = String::new(); + while position < bytes.len() { + if !base64_byte(bytes[position]) || (position > 0 && base64_byte(bytes[position - 1])) { + position += 1; + continue; + } + let start = position; + while position < bytes.len() && base64_byte(bytes[position]) { + position += 1; + } + let run_end = position; + while position < bytes.len() && position - run_end < 2 && bytes[position] == b'=' { + position += 1; + } + let run = &text[start..position]; + if run_end - start > limit && looks_like_base64(run) { + output.push_str(&text[previous..start]); + output.push_str(&format!( + "[base64_data truncated: {}]", + base64_size(run.len()) + )); + previous = position; + } + } + output.push_str(&text[previous..]); + output +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redaction_precedes_the_text_bound_and_preserves_unicode_character_limits() { + let processor = Processor::new(16); + let secret = format!("sk-{}", "q".repeat(48)); + let text = format!("{}{}{}", "é".repeat(110), secret, "界".repeat(1000)); + let input = DiagnosticInput { + message: text, + exception: None, + stack: None, + leaves: vec![], + }; + let output = processor + .process_diagnostic( + &input, + Policy { + redact: true, + base64_limit: 0, + text_limit: 500, + }, + ) + .unwrap(); + assert!(output.message.chars().count() <= 500); + assert!(!output.message.contains("sk-qq")); + assert!(output.changed); + } + + #[test] + fn base64_collapse_applies_to_debug_and_exceptions_without_touching_hex() { + let processor = Processor::new(16); + let input = DiagnosticInput { + message: format!("image={} digest={}", "Q".repeat(100), "a1".repeat(50)), + exception: Some(format!("upload failed: {}", "Q".repeat(100))), + stack: Some("api_key=secret123".to_owned()), + leaves: vec![(Some("api_key".to_owned()), "secret123".to_owned())], + }; + let output = processor + .process_diagnostic( + &input, + Policy { + redact: true, + base64_limit: 20, + text_limit: 0, + }, + ) + .unwrap(); + assert!(output.message.contains("[base64_data truncated: 75B]")); + assert!(output.message.contains(&"a1".repeat(50))); + assert!( + output + .exception + .unwrap() + .contains("[base64_data truncated: 75B]") + ); + assert_eq!(output.stack.as_deref(), Some(REDACTED)); + assert_eq!(output.leaves, vec![REDACTED]); + } + + #[test] + fn access_arguments_keep_encoded_paths_and_drop_decoded_credentials() { + let processor = Processor::new(16); + let arguments = vec![ + "/v1/models?filter=gpt%2D4o&page=2".to_owned(), + "/v1/models?k%65y=sk%2Dabcdefghijklmnopqrstuvwxyz&page=2".to_owned(), + ]; + assert_eq!( + processor.scrub_access_arguments(&arguments).unwrap(), + vec![arguments[0].clone(), "/v1/models?REDACTED".to_owned()] + ); + } +} diff --git a/litellm-rust/crates/tracing/src/redaction.rs b/litellm-rust/crates/tracing/src/redaction.rs new file mode 100644 index 00000000000..4bef14148d3 --- /dev/null +++ b/litellm-rust/crates/tracing/src/redaction.rs @@ -0,0 +1,140 @@ +use fancy_regex::{NoExpand, Regex}; + +pub const REDACTED: &str = "REDACTED"; + +#[cfg(test)] +const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16; + +fn secret_patterns(minimum_custom_key_length: usize) -> String { + let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len()); + [ + r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", + r"\bya29\.[A-Za-z0-9_.~+/-]+", + r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#, + r"(?:AKIA|ASIA)[0-9A-Z]{16}", + r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", + r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", + &format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"), + r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#, + r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#, + r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + r"x-ak-[A-Za-z0-9\-_]{20,}", + r"AIza[0-9A-Za-z\-_]{35}", + r#"(?<=[?&])key=[^\s&'"]{8,}"#, + r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#, + r"dapi[0-9a-f]{32}", + r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#, + concat!( + r"(?:master_key|xai_key|database_url|db_url|connection_string|", + r"aws_secret_access_key|aws_session_token|aws_access_key_id|s3_secret_access_key|s3_access_key_id|", + r"signing_key|encryption_key|", + r"auth_token|access_token|refresh_token|", + r"slack_webhook_url|webhook_url|", + r"database_connection_string|", + r"huggingface_token|jwt_secret)", + r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + ), + r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", + r"(?<=[?&])sig=[A-Za-z0-9%+/=]+", + r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#, + ] + .join("|") +} + +#[derive(Clone, Debug)] +pub struct SecretRedactor { + pattern: Regex, + internal_pattern: Regex, +} + +impl SecretRedactor { + pub fn new(minimum_custom_key_length: usize) -> Self { + let pattern = Regex::new(&format!( + "(?i){}", + secret_patterns(minimum_custom_key_length) + )) + .expect("secret redaction patterns compile"); + let internal_pattern = Regex::new(concat!( + r#"(?i)/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'"\)\]}>,]+|"#, + r#"[A-Za-z]:\\[^\s'"\)\]}>,]+|"#, + r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|", + r"192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b|", + r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b", + )) + .expect("internal detail patterns compile"); + Self { + pattern, + internal_pattern, + } + } + + pub fn redact(&self, value: &str) -> String { + self.try_redact(value) + .unwrap_or_else(|_| REDACTED.to_owned()) + } + + pub fn try_redact(&self, value: &str) -> fancy_regex::Result { + self.pattern + .try_replacen(value, 0, NoExpand(REDACTED)) + .map(|value| value.into_owned()) + } + + pub fn try_redact_structured( + &self, + key: Option<&str>, + value: &str, + ) -> fancy_regex::Result { + let scrubbed = self.try_redact(value)?; + if scrubbed != value || key.is_none() { + return Ok(scrubbed); + } + let rendered = format!("'{}': '{value}'", key.unwrap_or_default()); + Ok(if self.try_redact(&rendered)? != rendered { + REDACTED.to_owned() + } else { + value.to_owned() + }) + } + + pub fn try_redact_internal(&self, value: &str) -> fancy_regex::Result { + let without_traceback = value + .split_once("Traceback (most recent call last):") + .map_or(value, |(prefix, _)| prefix.trim_end()); + self.internal_pattern + .try_replacen(&self.try_redact(without_traceback)?, 0, NoExpand(REDACTED)) + .map(|value| value.into_owned()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + #[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")] + #[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")] + #[case::short_sk_key_is_kept("sk-abc", "sk-abc")] + #[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")] + #[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")] + #[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")] + #[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")] + #[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")] + #[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")] + #[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")] + #[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)] + fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) { + assert_eq!( + SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input), + expected + ); + } + + #[test] + fn sk_threshold_follows_the_minimum_custom_key_length() { + let redactor = SecretRedactor::new(8); + assert_eq!(redactor.redact("sk-abcde"), REDACTED); + assert_eq!(redactor.redact("sk-abcd"), "sk-abcd"); + } +} diff --git a/litellm-rust/crates/tracing/tests/logging.rs b/litellm-rust/crates/tracing/tests/logging.rs new file mode 100644 index 00000000000..585e442dad1 --- /dev/null +++ b/litellm-rust/crates/tracing/tests/logging.rs @@ -0,0 +1,122 @@ +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc, +}; + +use litellm_tracing::{Level, Logger, Metadata, Record, Sink, info, warn}; +use serde_json::{Value, json}; + +struct Output { + enabled: Arc, + sender: mpsc::Sender<(String, Value, Level, &'static str, Option)>, +} + +impl Sink for Output { + fn enabled(&self, _: &Metadata<'_>) -> bool { + self.enabled.load(Ordering::Relaxed) + } + + fn emit(&self, record: &Record) { + self.sender + .send(( + record.message.clone(), + Value::Object(record.fields.clone()), + *record.metadata.level(), + record.metadata.target(), + record.metadata.line(), + )) + .unwrap(); + Logger::default().scope(|| warn!("a sink must not recursively emit")); + } +} + +fn emit() { + warn!( + attempt = 3_u64, + elapsed = 1.5, + retry = true, + reason = "timeout", + "retry {}", + 3 + ); +} + +#[test] +fn records_preserve_fields_metadata_and_dynamic_filtering_without_recursion() { + let (sender, receiver) = mpsc::channel(); + let enabled = Arc::new(AtomicBool::new(false)); + let logger = Logger::new(Output { + enabled: enabled.clone(), + sender, + }); + logger.scope(emit); + assert!(receiver.try_recv().is_err()); + enabled.store(true, Ordering::Relaxed); + logger.scope(emit); + let (message, fields, level, target, line) = receiver.try_recv().unwrap(); + assert_eq!(message, "retry 3"); + assert_eq!( + fields, + json!({"attempt": 3, "elapsed": 1.5, "retry": true, "reason": "timeout"}) + ); + assert_eq!(level, Level::WARN); + assert_eq!(target, module_path!()); + assert!(line.is_some()); + enabled.store(false, Ordering::Relaxed); + logger.scope(emit); + assert!(receiver.try_recv().is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_futures_keep_their_sinks_across_suspension_and_spawn() { + let tasks = (0..2) + .map(|id| { + let (sender, receiver) = mpsc::channel(); + let logger = Logger::new(Output { + enabled: Arc::new(AtomicBool::new(true)), + sender, + }); + let task = tokio::spawn(logger.instrument(async move { + tokio::task::yield_now().await; + info!(id, "worker"); + })); + (id, task, receiver) + }) + .collect::>(); + for (id, task, receiver) in tasks { + task.await.unwrap(); + let (message, fields, level, _, _) = receiver.try_recv().unwrap(); + assert_eq!(message, "worker"); + assert_eq!(fields, json!({"id": id})); + assert_eq!(level, Level::INFO); + assert!(receiver.try_recv().is_err()); + } +} + +#[test] +fn nested_scopes_restore_the_previous_sink() { + let (outer_sender, outer) = mpsc::channel(); + let (inner_sender, inner) = mpsc::channel(); + let logger = |sender| { + Logger::new(Output { + enabled: Arc::new(AtomicBool::new(true)), + sender, + }) + }; + let outside = logger(outer_sender); + let inside = logger(inner_sender); + outside.scope(|| { + info!("before"); + inside.scope(|| info!("inside")); + info!("after"); + }); + assert_eq!( + outer.try_iter().map(|event| event.0).collect::>(), + ["before", "after"] + ); + assert_eq!( + inner.try_iter().map(|event| event.0).collect::>(), + ["inside"] + ); +} diff --git a/litellm/_logging.py b/litellm/_logging.py index 644a79d8cbd..8bd7a86ecf5 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,11 +1,12 @@ import ast import contextvars import functools +import itertools import logging import os import re import sys -from collections.abc import Sequence +from collections.abc import Iterator from datetime import datetime from logging import Formatter from typing import Any, Final, TextIO @@ -22,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( + _python_redact_string, + _python_redact_structured_value, redact_internal_details, redact_string, - redact_structured_value, ) +from litellm.rust_bridge import diagnostics set_verbose = False @@ -49,7 +52,7 @@ def _sanitize_correlation_id(value: str) -> str: pass through credential redaction. """ stripped: Final = "".join(ch for ch in value if ch.isprintable()) - return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH]) + return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH] def set_session_id(session_id: str) -> "contextvars.Token[str]": @@ -74,12 +77,6 @@ def _redact_string(value: str) -> str: return redact_string(value) -def _redact_structured_value(key: str | None, value: str) -> str: - if not _ENABLE_SECRET_REDACTION: - return value - return redact_structured_value(key, value) - - _REDACTED_RECORD_ATTR: Final = "litellm_redacted" _REDACTED_STAMP: Final = object() _UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None)) @@ -103,14 +100,6 @@ def _plain_text(value: object) -> str: return UNSERIALIZABLE_OBJECT -def _redact_extra_value(key: str, value: object) -> object: - try: - scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key) - except Exception: - return _redact_string(_plain_text(value)) - return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed - - def redact_secrets(value: str) -> str: """Public API: redact known secret/credential patterns from an arbitrary string. @@ -150,7 +139,7 @@ def _substituted_color_message(record: logging.LogRecord) -> str | None: return None try: return color_message % record.args - except TypeError: + except Exception: return color_message @@ -162,42 +151,7 @@ class SecretRedactionFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: if not _ENABLE_SECRET_REDACTION or _is_redacted(record): return True - - # Runs before args are cleared, and before the extra-field loop below - # that redacts the substituted result. - substituted_color_message: Final = _substituted_color_message(record) - if substituted_color_message is not None: - record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place - - try: - record.msg = _redact_string(record.getMessage()) - record.args = None - except Exception: - if isinstance(record.msg, str): - record.msg = _redact_string(record.msg) - - # Redact exception tracebacks - if record.exc_info and record.exc_info[1] is not None: - try: - record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info)) - except Exception: - pass - - if isinstance(record.stack_info, str): - record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place - - # Redact extra fields passed via logger.debug("msg", extra={...}) - record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items()) - for key, value in record_items: - if key in _STANDARD_RECORD_ATTRS: - continue - if isinstance(value, str): - setattr(record, key, _redact_structured_value(key, value)) - elif not isinstance(value, _UNREDACTED_SCALAR_TYPES): - setattr(record, key, _redact_extra_value(key, value)) - - setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP) - return True + return _process_record(record, base64_limit=0, text_limit=0, redact=True) _secret_filter: Final = SecretRedactionFilter() @@ -211,7 +165,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED" def _hides_a_credential(value: str) -> bool: """Whether *value* only looks clean until it is percent-decoded.""" decoded: Final = unquote(value) - return _redact_string(decoded) != decoded + return _python_redact_string(decoded) != decoded def _drop_encoded_credential(scrubbed: str) -> str: @@ -239,10 +193,10 @@ def _scrub_access_arg(value: str) -> str: pattern and would then be logged raw. """ if len(value) <= _MAX_SCRUBBED_ACCESS_ARG: - return _drop_encoded_credential(_redact_string(value)) + return _drop_encoded_credential(_python_redact_string(value)) head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG] kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head - scrubbed: Final = _drop_encoded_credential(_redact_string(kept)) + scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept)) return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..." @@ -258,8 +212,17 @@ class AccessLogRedactionFilter(logging.Filter): if not _ENABLE_SECRET_REDACTION: return True if isinstance(record.args, tuple) and record.args: + strings: Final = tuple(arg for arg in record.args if isinstance(arg, str)) + candidate: Final = diagnostics.run( + lambda native: native.scrub_access_arguments(strings), + lambda: tuple(_scrub_access_arg(arg) for arg in strings), + ) + scrubbed: Final = ( + candidate if len(candidate) == len(strings) else tuple(_scrub_access_arg(arg) for arg in strings) + ) + values: Final = iter(scrubbed) record.args = tuple( # rebind-ok: a Filter scrubs records in place - _scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args + next(values) if isinstance(arg, str) else arg for arg in record.args ) return True # No positional args means everything is in msg, where collapsing is correct. @@ -365,6 +328,185 @@ def _collapse_base64_runs(text: str, limit: int) -> str: return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text) +def _extra_structure(key: str, value: object) -> object: + if isinstance(value, str): + return value + try: + return safe_json_structure(value, key=key) + except Exception: + return _plain_text(value) + + +def _string_leaves(key: str | None, value: object) -> Iterator[tuple[str | None, str]]: + if isinstance(value, str): + yield key, value + elif isinstance(value, dict): + yield from itertools.chain.from_iterable( + _string_leaves(child_key, child) for child_key, child in value.items() if isinstance(child_key, str) + ) + elif isinstance(value, (list, tuple)): + yield from itertools.chain.from_iterable(_string_leaves(key, child) for child in value) + + +def _replace_string_leaves(value: object, values: Iterator[str]) -> object: + if isinstance(value, str): + return next(values) + if isinstance(value, dict): + return { # mutable-ok: LogRecord extras must keep JSON dict shape for handlers + key: _replace_string_leaves(child, values) for key, child in value.items() + } + if isinstance(value, list): + return [ # mutable-ok: LogRecord extras must keep JSON list shape for handlers + _replace_string_leaves(child, values) for child in value + ] + if isinstance(value, tuple): + return tuple(_replace_string_leaves(child, values) for child in value) + return value + + +def _sort_processed_sets(original: object, processed: object) -> object: + if isinstance(original, set) and isinstance(processed, list): + return sorted(processed) + if isinstance(original, dict) and isinstance(processed, dict): + return { # mutable-ok: sorting nested sets must preserve the surrounding JSON dict + key: _sort_processed_sets(original.get(key), value) for key, value in processed.items() + } + if isinstance(original, list) and isinstance(processed, list): + return [ # mutable-ok: sorting nested sets must preserve the surrounding JSON list + _sort_processed_sets(before, after) for before, after in zip(original, processed) + ] + if isinstance(original, tuple) and isinstance(processed, tuple): + return tuple(_sort_processed_sets(before, after) for before, after in zip(original, processed)) + return processed + + +def _python_process_diagnostic( + message: str, + exception: str | None, + stack: str | None, + leaves: tuple[tuple[str | None, str], ...], + redact: bool, + base64_limit: int, + text_limit: int, +) -> tuple[str, str | None, str | None, tuple[str, ...], bool]: + def process_text(text: str) -> str: + collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text + scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed + return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed + + processed_message: Final = process_text(message) + processed_exception: Final = process_text(exception) if exception is not None else None + processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack + processed_leaves: Final = tuple( + _python_redact_structured_value(key, text) if redact else text for key, text in leaves + ) + changed: Final = ( + processed_message != message + or processed_exception != exception + or processed_stack != stack + or any(processed != original for processed, (_, original) in zip(processed_leaves, leaves)) + ) + return processed_message, processed_exception, processed_stack, processed_leaves, changed + + +def _render_message(record: logging.LogRecord) -> str: + try: + return record.getMessage() + except Exception: + return record.msg if isinstance(record.msg, str) else UNSERIALIZABLE_OBJECT + + +def _render_exception(record: logging.LogRecord) -> str | None: + if not isinstance(record.exc_info, tuple) or len(record.exc_info) < 2 or record.exc_info[1] is None: + return None + try: + return record.exc_text or SecretRedactionFilter._formatter.formatException(record.exc_info) + except Exception: + return "REDACTED" + + +def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: int, redact: bool) -> bool: + if _is_redacted(record): + return True + message: Final = _render_message(record) + exception: Final = _render_exception(record) + stack: Final = record.stack_info if isinstance(record.stack_info, str) else None + substituted_color: Final = _substituted_color_message(record) + extras: Final = ( + tuple( + ( + key, + value, + _extra_structure( + key, substituted_color if key == "color_message" and substituted_color is not None else value + ), + ) + for key, value in record.__dict__.items() + if key not in _STANDARD_RECORD_ATTRS + and key != _REDACTED_RECORD_ATTR + and not isinstance(value, _UNREDACTED_SCALAR_TYPES) + ) + if redact + else () + ) + extra_leaves: Final = tuple( + itertools.chain.from_iterable(_string_leaves(key, prepared) for key, _, prepared in extras) + ) + raw_template: Final = record.msg if redact and isinstance(record.msg, str) and record.args else None + color_template: Final = record.__dict__.get("color_message") + raw_color: Final = color_template if redact and isinstance(color_template, str) and record.args else None + leaves: Final = ( + extra_leaves + + (((None, raw_template),) if raw_template is not None else ()) + + (((None, raw_color),) if raw_color is not None else ()) + ) + candidate: Final = diagnostics.run( + lambda native: native.process_diagnostic(message, exception, stack, leaves, (redact, base64_limit, text_limit)), + lambda: _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit), + ) + processed_message, processed_exception, processed_stack, processed_leaves, _ = ( + candidate + if len(candidate[3]) == len(leaves) + else _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit) + ) + raw_template_changed: Final = raw_template is not None and processed_leaves[len(extra_leaves)] != raw_template + safe_message: Final = "REDACTED" if raw_template_changed and processed_message == message else processed_message + if redact or safe_message != message: + record.msg = safe_message # rebind-ok: the Filter interface mutates the record + record.args = None # rebind-ok: the rendered message replaces interpolation inputs + if processed_exception is not None: + record.exc_text = processed_exception # rebind-ok: the Filter interface mutates the record + if processed_stack is not None: + record.stack_info = processed_stack # rebind-ok: the Filter interface mutates the record + processed_values: Final = iter(processed_leaves[: len(extra_leaves)]) + for key, original, prepared in extras: + replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values)) + if not _scrubbing_changed_nothing(replacement, original): + setattr(record, key, replacement) + raw_color_changed: Final = ( + raw_color is not None and processed_leaves[len(extra_leaves) + int(raw_template is not None)] != raw_color + ) + if raw_color_changed and getattr(record, "color_message", None) == substituted_color: + setattr(record, "color_message", "REDACTED") + setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP) + return True + + +def _redact_json_record(value: object) -> object: + prepared: Final = safe_json_structure(value) + leaves: Final = tuple(_string_leaves(None, prepared)) + candidate: Final = diagnostics.run( + lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3], + lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves), + ) + replacements: Final = ( + candidate + if len(candidate) == len(leaves) + else tuple(_python_redact_structured_value(key, text) for key, text in leaves) + ) + return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements))) + + class StdoutLogTruncationFilter(logging.Filter): """Bounds how much of an oversized log line reaches stdout. @@ -412,7 +554,17 @@ class StdoutLogTruncationFilter(logging.Filter): return True -_stdout_truncation_filter: Final = StdoutLogTruncationFilter() +class DiagnosticProcessingFilter(StdoutLogTruncationFilter): + def filter(self, record: logging.LogRecord) -> bool: + return _process_record( + record, + base64_limit=_get_max_base64_length_stdout_log(), + text_limit=_get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0, + redact=_ENABLE_SECRET_REDACTION, + ) + + +_diagnostic_filter: Final = DiagnosticProcessingFilter() class CorrelationContextFilter(logging.Filter): @@ -633,7 +785,9 @@ class JsonFormatter(Formatter): if record.exc_info: json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info) - return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value) + return safe_dumps( + json_record if _is_redacted(record) or not _ENABLE_SECRET_REDACTION else _redact_json_record(json_record) + ) class CorrelationPlainFormatter(logging.Formatter): @@ -663,7 +817,7 @@ def _setup_json_exception_handlers(formatter): # Create a handler with JSON formatting for exceptions error_handler: Final = logging.StreamHandler() error_handler.setFormatter(formatter) - error_handler.addFilter(_stdout_truncation_filter) + error_handler.addFilter(_diagnostic_filter) error_handler.addFilter(_secret_filter) error_handler.addFilter(_correlation_filter) @@ -734,10 +888,10 @@ verbose_logger.addHandler(handler) # Filters attached to the logger, not the handler, survive callers swapping in their own # handlers (JSON mode, uvicorn log config, a host app's root handler). -verbose_router_logger.addFilter(_stdout_truncation_filter) -verbose_proxy_logger.addFilter(_stdout_truncation_filter) -verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter) -verbose_logger.addFilter(_stdout_truncation_filter) +verbose_router_logger.addFilter(_diagnostic_filter) +verbose_proxy_logger.addFilter(_diagnostic_filter) +verbose_proxy_stdout_logger.addFilter(_diagnostic_filter) +verbose_logger.addFilter(_diagnostic_filter) def _suppress_loggers(): diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d66d80a564b..7990832dc48 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2375,6 +2375,11 @@ def default_image_cost_calculator( model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name + provider_first_model_name_with_quality: Final = ( + f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" + if quality and custom_llm_provider + else None + ) # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" @@ -2386,6 +2391,7 @@ def default_image_cost_calculator( models_to_check: Final = ( model_name_with_quality, + provider_first_model_name_with_quality, base_model_name, model_name_with_v2_quality, model_with_quality_without_provider, diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 70b2cd08b4c..89df5e500db 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -10,6 +10,7 @@ import re from typing import Final from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH +from litellm.rust_bridge import diagnostics REDACTED: Final = "REDACTED" @@ -87,9 +88,13 @@ def _build_secret_patterns() -> "re.Pattern[str]": _SECRET_RE: Final = _build_secret_patterns() +def _python_redact_string(value: str) -> str: + return _SECRET_RE.sub(REDACTED, value) + + def redact_string(value: str) -> str: """Scrub known secret/credential patterns from *value* and return the result.""" - return _SECRET_RE.sub(REDACTED, value) + return diagnostics.run(lambda native: native.redact_text(value), lambda: _python_redact_string(value)) _UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+" @@ -105,15 +110,21 @@ _INTERNAL_DETAIL_RE: Final = re.compile( _TRACEBACK_MARKER: Final = "Traceback (most recent call last):" -def redact_internal_details(value: str) -> str: +def _python_redact_internal_details(value: str) -> str: """Drop an embedded traceback and scrub filesystem paths and internal hostnames, on top of redact_string(). For client-facing messages only: server logs keep this detail.""" marker_index: Final = value.find(_TRACEBACK_MARKER) without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value - return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback)) + return _INTERNAL_DETAIL_RE.sub(REDACTED, _python_redact_string(without_traceback)) -def redact_structured_value(key: str | None, value: str) -> str: +def redact_internal_details(value: str) -> str: + return diagnostics.run( + lambda native: native.redact_client_message(value), lambda: _python_redact_internal_details(value) + ) + + +def _python_redact_structured_value(key: str | None, value: str) -> str: """Scrub *value* as it appeared under *key* inside a structured record. redact_string() replaces a whole ``key: value`` span with REDACTED, which is @@ -122,8 +133,15 @@ def redact_structured_value(key: str | None, value: str) -> str: repr would, so the key-name patterns still fire, but collapses only the value so the caller's structure survives. """ - scrubbed: Final = redact_string(value) + scrubbed: Final = _python_redact_string(value) if scrubbed != value or key is None: return scrubbed rendered: Final = f"'{key}': '{value}'" - return REDACTED if redact_string(rendered) != rendered else value + return REDACTED if _python_redact_string(rendered) != rendered else value + + +def redact_structured_value(key: str | None, value: str) -> str: + return diagnostics.run( + lambda native: native.redact_structured_text(key, value), + lambda: _python_redact_structured_value(key, value), + ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 21830eb0d8e..497020c2836 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig): """ return self._is_deepseek_r1_model(model=model, base_model=base_model) + @classmethod + def _supports_sampling_params(cls, model: str) -> bool: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + base_model: Final = BedrockModelInfo.get_base_model(model) + if base_model.startswith("anthropic"): + return True + candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu."))) + for candidate in candidates: + if ( + flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API + candidate, "supports_sampling_params" + ) + ) is not None: + return flag + return True + def get_supported_openai_params(self, model: str) -> list[str]: from litellm.utils import supports_function_calling + supports_sampling: Final = self._supports_sampling_params(model) supported_params: Final = [ "max_tokens", "max_completion_tokens", "stream", "stream_options", "stop", - "temperature", - "top_p", + *(("temperature", "top_p") if supports_sampling else ()), "extra_headers", "response_format", "requestMetadata", @@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig): value = [value] optional_params["stopSequences"] = value if param == "temperature" or param == "top_p": - AnthropicConfig._apply_sampling_param( - optional_params=optional_params, - model=model, - param=param, - value=value, - drop_params=drop_params, - output_key="topP" if param == "top_p" else param, - ) + if base_model.startswith("anthropic"): + AnthropicConfig._apply_sampling_param( + optional_params=optional_params, + model=model, + param=param, + value=value, + drop_params=drop_params, + output_key="topP" if param == "top_p" else param, + ) + elif not self._supports_sampling_params(model): + if not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support {param}={value}. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + else: + optional_params["topP" if param == "top_p" else param] = value if param == "tools" and isinstance(value, list): self._apply_tool_call_transformation( tools=cast(list[OpenAIChatCompletionToolParam], value), diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 09219b805a2..c7b4018b80b 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -231,6 +231,12 @@ async def make_call( sync_stream=False, ) completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size)) + elif bedrock_invoke_provider == "moonshot": + decoder = AmazonOpenAICompatibleStreamDecoder( + model=model, + sync_stream=False, + ) + completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size)) else: decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size)) @@ -329,6 +335,12 @@ def make_sync_call( sync_stream=True, ) completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) + elif bedrock_invoke_provider == "moonshot": + decoder = AmazonOpenAICompatibleStreamDecoder( + model=model, + sync_stream=True, + ) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) else: decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) @@ -795,6 +807,24 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder): return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data) +class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder): + def __init__( + self, + model: str, + sync_stream: bool, + ) -> None: + super().__init__(model=model) + from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler + + self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler( + streaming_response=None, + sync_stream=sync_stream, + ) + + def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream: + return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data) + + class MockResponseIterator: # for returning ai21 streaming responses def __init__(self, model_response, json_mode: bool | None = False): self.model_response = model_response diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index f46edc766c7..14bd2bee6cf 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -770,9 +770,7 @@ class AmazonAnthropicClaudeMessagesConfig( aws_decoder: Final = AmazonAnthropicClaudeMessagesStreamDecoder( model=model, ) - completion_stream: Final = aws_decoder.aiter_bytes( - httpx_response.aiter_bytes(chunk_size=aws_decoder.DEFAULT_CHUNK_SIZE) - ) + completion_stream: Final = aws_decoder.aiter_bytes(httpx_response.aiter_bytes()) # Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients. return self.bedrock_sse_wrapper( completion_stream=completion_stream, @@ -919,16 +917,6 @@ class AmazonAnthropicClaudeMessagesConfig( class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder): - def __init__( - self, - model: str, - ) -> None: - """ - Iterator to return Bedrock invoke response in anthropic /messages format - """ - super().__init__(model=model) - self.DEFAULT_CHUNK_SIZE = 1024 - def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict: """ Parse the chunk data into anthropic /messages format diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 204b1ddd5ea..a50665c71df 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -287,7 +287,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Check if the model is Gemini 3 or newer. """ model_name = model.split("/")[-1].lower() - if not model_name: + is_vertex_fine_tuned_model: Final = model_name.isdigit() or ( + model.startswith("gemini/") and not model_name.startswith("gemini-") + ) + if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"): return False # Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ad66cb76f8b..ae02744b8e1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -81,7 +81,8 @@ "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 1.25e-05 + "output_cost_per_token": 1.25e-05, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.j2-ultra-v1": { "input_cost_per_token": 1.88e-05, @@ -90,7 +91,8 @@ "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 1.88e-05 + "output_cost_per_token": 1.88e-05, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.jamba-1-5-large-v1:0": { "deprecation_date": "2026-11-26", @@ -100,7 +102,8 @@ "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 8e-06 + "output_cost_per_token": 8e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.jamba-1-5-mini-v1:0": { "deprecation_date": "2026-11-26", @@ -110,7 +113,8 @@ "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 4e-07 + "output_cost_per_token": 4e-07, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.jamba-instruct-v1:0": { "input_cost_per_token": 5e-07, @@ -120,6 +124,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_system_messages": true }, "aiml/dall-e-2": { @@ -295,6 +300,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -306,6 +312,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -317,6 +324,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -328,6 +336,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -339,7 +348,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-vision-7b.html", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_vision": true }, "amazon.nova-lite-v1:0": { @@ -814,7 +823,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1048,7 +1057,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1083,7 +1093,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1118,7 +1129,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1153,7 +1165,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1188,7 +1201,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1223,7 +1237,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1255,7 +1270,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1309,7 +1324,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1347,7 +1362,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1385,12 +1400,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1422,12 +1438,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1695,7 +1712,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -2004,7 +2022,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2081,7 +2100,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2158,7 +2178,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2231,7 +2252,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2270,7 +2291,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2309,7 +2330,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2348,12 +2369,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2386,12 +2408,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2424,16 +2447,18 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -2460,11 +2485,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2619,7 +2645,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2657,7 +2684,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2695,7 +2723,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2834,7 +2863,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2868,7 +2898,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2902,7 +2933,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2934,7 +2966,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2971,7 +3004,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, - "output_cost_per_token_batches": 7.5e-06 + "output_cost_per_token_batches": 7.5e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -3213,7 +3247,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -3262,7 +3297,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -11845,6 +11881,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { @@ -11858,6 +11897,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { @@ -11872,6 +11914,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { @@ -11897,6 +11942,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { @@ -11910,6 +11957,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/moonshotai.kimi-k2-thinking": { @@ -11937,7 +11987,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, @@ -11969,6 +12021,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { @@ -11982,6 +12037,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { @@ -11996,6 +12054,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { @@ -12021,6 +12082,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { @@ -12034,6 +12097,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { @@ -12048,6 +12114,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.236e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { @@ -12062,6 +12131,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { @@ -12075,6 +12147,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { @@ -12089,6 +12164,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { @@ -12103,6 +12181,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { @@ -12116,6 +12196,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { @@ -12148,6 +12231,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { @@ -12161,6 +12247,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { @@ -12175,6 +12264,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { @@ -12189,6 +12281,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { @@ -12289,6 +12383,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { @@ -12303,6 +12400,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { @@ -12316,6 +12416,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { @@ -12347,6 +12450,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { @@ -12361,6 +12467,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { @@ -12374,6 +12483,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { @@ -12405,6 +12517,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { @@ -12419,6 +12534,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.86e-06 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { @@ -12449,6 +12567,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { @@ -12462,13 +12583,14 @@ "supports_tool_choice": true }, "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 1.04e-05, + "input_cost_per_token": 5.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 3.12e-05, + "output_cost_per_token": 1.56e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": { @@ -12492,6 +12614,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-south-1/minimax.minimax-m2.5": { @@ -12506,6 +12631,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { @@ -12519,6 +12647,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -12569,6 +12700,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { @@ -12582,6 +12716,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { @@ -12596,6 +12733,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { @@ -12621,6 +12761,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { @@ -12634,6 +12776,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { @@ -12753,13 +12898,14 @@ "supports_tool_choice": true }, "bedrock/us-east-1/mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 4e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 1.2e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": { @@ -12784,6 +12930,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.1": { @@ -12797,6 +12946,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.5": { @@ -12810,6 +12962,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { @@ -12835,6 +12990,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { @@ -12848,6 +13005,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/deepseek.v3.2": { @@ -12862,6 +13022,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.1": { @@ -12875,6 +13038,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.5": { @@ -12889,6 +13055,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.2e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { @@ -12914,6 +13083,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { @@ -12927,6 +13098,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { @@ -13354,13 +13528,14 @@ "supports_tool_choice": true }, "bedrock/us-west-2/mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 4e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 1.2e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": { @@ -13385,6 +13560,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.1": { @@ -13398,6 +13576,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.5": { @@ -13411,6 +13592,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { @@ -13436,6 +13620,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { @@ -13449,6 +13635,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { @@ -14656,16 +14845,18 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "cohere.command-text-v14": { - "input_cost_per_token": 1.5e-06, + "input_cost_per_token": 1e-06, "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "cohere.embed-english-v3": { @@ -14675,6 +14866,7 @@ "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_embedding_image_input": true }, "cohere.embed-multilingual-v3": { @@ -14684,6 +14876,7 @@ "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_embedding_image_input": true }, "cohere.embed-v4:0": { @@ -14694,6 +14887,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_embedding_image_input": true }, "us.cohere.embed-v4:0": { @@ -20867,6 +21061,7 @@ "max_tokens": 81920, "mode": "chat", "output_cost_per_token": 1.68e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -21342,7 +21537,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -21511,7 +21706,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -21548,7 +21744,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -25846,7 +26043,7 @@ "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/vertex_ai/live", "/v1/realtime" @@ -25882,7 +26079,6 @@ "input_cost_per_image_token": 3e-06 }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -25911,7 +26107,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -25927,7 +26123,6 @@ "input_cost_per_image_token": 3e-06 }, "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -25957,7 +26152,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -29709,7 +29904,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, - "output_cost_per_token_batches": 7.5e-06 + "output_cost_per_token_batches": 7.5e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -29741,7 +29937,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -29755,7 +29952,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34537,7 +34734,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34551,7 +34749,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -35128,7 +35326,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1e-06 + "output_cost_per_token": 1e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "meta.llama2-70b-chat-v1": { "input_cost_per_token": 1.95e-06, @@ -35137,7 +35336,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 2.56e-06 + "output_cost_per_token": 2.56e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, @@ -35837,16 +36037,18 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 4e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 1.2e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "mistral.mistral-large-2407-v1:0": { @@ -35878,12 +36080,15 @@ }, "mistral.mistral-small-2402-v1:0": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "mistral.mixtral-8x7b-instruct-v0:1": { @@ -35894,6 +36099,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 7e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "mistral.voxtral-mini-3b-2507": { @@ -35904,6 +36110,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true @@ -35916,6 +36123,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true @@ -39101,6 +39309,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -39114,6 +39323,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42315,7 +42525,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -42328,7 +42541,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.545e-07, @@ -42341,7 +42557,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -42354,7 +42573,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, @@ -42367,7 +42589,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -42380,7 +42605,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "qwen.qwen3-vl-235b-a22b": { "input_cost_per_token": 5.3e-07, @@ -44356,7 +44584,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -44467,7 +44695,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -44504,7 +44733,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -44534,7 +44764,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us-gov.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -44592,7 +44823,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -44718,7 +44949,10 @@ "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "us-gov.nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2.4e-07, @@ -44729,7 +44963,10 @@ "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true }, "us-gov.nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 7.2e-08, @@ -44739,7 +44976,11 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": false }, "us-gov.nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -44753,7 +44994,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -44821,7 +45065,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -44879,7 +45124,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -44911,19 +45157,21 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -44942,7 +45190,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -44974,7 +45223,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -44984,6 +45234,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.4e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": false, "supports_reasoning": true, "supports_tool_choice": false @@ -44999,7 +45250,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "eu.deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -45012,7 +45266,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "us.meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, @@ -45154,6 +45411,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_tool_choice": false }, @@ -46476,16 +46734,20 @@ "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -46501,16 +46763,20 @@ "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -46633,14 +46899,18 @@ "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -46657,20 +46927,25 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-5@20251101": { "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -46688,7 +46963,8 @@ "supports_vision": true, "supports_native_streaming": true, "supports_output_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-6": { "deprecation_date": "2027-02-05", @@ -46697,14 +46973,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46721,7 +47001,8 @@ "supports_vision": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-6@default": { "deprecation_date": "2027-02-05", @@ -46730,14 +47011,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46754,7 +47039,8 @@ "supports_vision": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-7": { "deprecation_date": "2027-04-16", @@ -46762,14 +47048,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46787,7 +47077,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-7@default": { "deprecation_date": "2027-04-16", @@ -46795,14 +47086,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46820,7 +47115,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5": { "deprecation_date": "2027-06-08", @@ -46828,14 +47124,18 @@ "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46855,14 +47155,17 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5-1": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "input_cost_per_token": 1e-05, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, @@ -46892,7 +47195,10 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512, - "deprecation_date": "2027-03-01" + "deprecation_date": "2027-03-01", + "input_cost_per_token_batches": 5e-06, + "output_cost_per_token_batches": 2.5e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -46900,14 +47206,18 @@ "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46927,14 +47237,17 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5-1@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "input_cost_per_token": 1e-05, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, @@ -46964,7 +47277,10 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512, - "deprecation_date": "2027-03-01" + "deprecation_date": "2027-03-01", + "input_cost_per_token_batches": 5e-06, + "output_cost_per_token_batches": 2.5e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -46973,14 +47289,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46998,7 +47318,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5@default": { "deprecation_date": "2027-01-24", @@ -47007,14 +47328,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47032,21 +47357,26 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5-5": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_creation_input_token_cost_batches": 2.5e-06, "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47068,21 +47398,26 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_creation_input_token_cost_batches": 2.5e-06, "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47104,7 +47439,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-8": { "deprecation_date": "2027-05-28", @@ -47113,14 +47449,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47138,7 +47478,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-8@default": { "deprecation_date": "2027-05-28", @@ -47147,14 +47488,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47172,18 +47517,21 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-5": { "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, @@ -47202,7 +47550,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-5": { "deprecation_date": "2026-12-24", @@ -47212,12 +47561,14 @@ "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47236,7 +47587,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-6": { "regional_endpoint_uplift_multiplier": 1.1, @@ -47244,14 +47596,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_batches": 1.88e-06, "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -47268,18 +47624,21 @@ "search_context_size_medium": 0.01 }, "supports_output_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-5@20250929": { "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, @@ -47299,7 +47658,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -47309,6 +47669,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47320,6 +47681,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47331,6 +47693,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47342,6 +47705,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47379,14 +47743,17 @@ "supports_tool_choice": true }, "vertex_ai/deepseek-ai/deepseek-v3.1-maas": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.7e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 8.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "us-central1" ], @@ -47397,6 +47764,7 @@ "supports_tool_choice": true }, "vertex_ai/deepseek-ai/deepseek-v3.2-maas": { + "cache_read_input_token_cost": 5.6e-08, "input_cost_per_token": 5.6e-07, "input_cost_per_token_batches": 2.8e-07, "litellm_provider": "vertex_ai-deepseek_models", @@ -47406,7 +47774,7 @@ "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -47418,13 +47786,15 @@ }, "vertex_ai/deepseek-ai/deepseek-r1-0528-maas": { "input_cost_per_token": 1.35e-06, + "input_cost_per_token_batches": 6.75e-07, "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 65336, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 2.7e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "us-central1" ], @@ -48000,13 +48370,15 @@ }, "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 5.75e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_modalities": [ "text", "image" @@ -48060,13 +48432,15 @@ }, "vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 3.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_modalities": [ "text", "image" @@ -48112,6 +48486,7 @@ "supports_tool_choice": true }, "vertex_ai/minimaxai/minimax-m2-maas": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-minimax_models", "max_input_tokens": 196608, @@ -48119,11 +48494,12 @@ "max_tokens": 196608, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, "vertex_ai/moonshotai/kimi-k2-thinking-maas": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 6e-07, "litellm_provider": "vertex_ai-moonshot_models", "max_input_tokens": 256000, @@ -48131,12 +48507,13 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true, "supports_web_search": true }, "vertex_ai/zai-org/glm-4.7-maas": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 6e-07, "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, @@ -48144,7 +48521,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48161,7 +48538,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48178,6 +48555,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48189,6 +48567,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48200,6 +48579,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48211,6 +48591,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48291,7 +48672,7 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/mistral-small-2503@001": { "input_cost_per_token": 1e-07, @@ -48303,7 +48684,7 @@ "output_cost_per_token": 3e-07, "supports_function_calling": true, "supports_tool_choice": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/mistral-ocr-2505": { "litellm_provider": "vertex_ai", @@ -48320,7 +48701,7 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "ocr_cost_per_page": 0.0003, - "source": "https://cloud.google.com/vertex-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "us-central1" ] @@ -48343,13 +48724,15 @@ }, "vertex_ai/openai/gpt-oss-120b-maas": { "input_cost_per_token": 9e-08, + "input_cost_per_token_batches": 4.5e-08, "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.6e-07, - "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas", + "output_cost_per_token_batches": 1.8e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_reasoning": true }, "vertex_ai/openai/gpt-oss-20b-maas": { @@ -48360,9 +48743,11 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_reasoning": true, - "cache_read_input_token_cost": 7e-09 + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token_batches": 3.5e-08, + "output_cost_per_token_batches": 1.25e-07 }, "vertex_ai/xai/grok-4.1-fast-non-reasoning": { "cache_read_input_token_cost": 5e-08, @@ -48373,7 +48758,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/developers/models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -48390,7 +48775,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/developers/models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -48482,13 +48867,15 @@ }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "output_cost_per_token_batches": 4.4e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global", "us-south1" @@ -48497,14 +48884,17 @@ "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { + "cache_read_input_token_cost": 2.2e-08, "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "output_cost_per_token_batches": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48519,7 +48909,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48534,7 +48924,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -54020,7 +54410,7 @@ "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, - "source": "https://platform.openai.com/docs/api-reference/videos", + "source": "https://developers.openai.com/api/docs/pricing", "supported_modalities": [ "text", "image" @@ -54735,12 +55125,14 @@ "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -54759,7 +55151,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-6@default": { "regional_endpoint_uplift_multiplier": 1.1, @@ -54767,14 +55160,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_batches": 1.88e-06, "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -54791,7 +55188,8 @@ "search_context_size_medium": 0.01 }, "supports_output_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -54984,14 +55382,14 @@ "supports_vision": true }, "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": { - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -55074,6 +55472,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55087,7 +55486,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "global.openai.gpt-5.6-sol": { "input_cost_per_token": 4e-06, @@ -55103,6 +55503,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55116,7 +55517,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "us.openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -55132,6 +55534,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55145,7 +55548,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "global.openai.gpt-5.6-terra": { "input_cost_per_token": 2e-06, @@ -55161,6 +55565,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55174,7 +55579,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "us.openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -55190,6 +55596,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55203,7 +55610,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "global.openai.gpt-5.6-luna": { "input_cost_per_token": 2e-07, @@ -55219,6 +55627,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55232,7 +55641,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "bedrock_mantle/openai.gpt-6-astra": { "input_cost_per_token": 1.1e-05, @@ -55302,7 +55712,7 @@ "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.openai.gpt-6-astra": { "input_cost_per_token": 1e-05, @@ -55334,7 +55744,7 @@ "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -55548,6 +55958,7 @@ "max_output_tokens": 500000, "max_tokens": 500000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, @@ -55563,6 +55974,7 @@ "max_output_tokens": 500000, "max_tokens": 500000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, @@ -55766,6 +56178,9 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/zai.glm-5": { @@ -55781,6 +56196,9 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -57107,10 +57525,10 @@ }, "gemini/gemini-robotics-er-2-streaming-preview": { "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 2e-06, + "input_cost_per_token": 1e-06, "litellm_provider": "gemini", "mode": "chat", - "output_cost_per_token": 1e-05, + "output_cost_per_token": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, @@ -61005,7 +61423,10 @@ "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2.4e-07, @@ -61016,7 +61437,10 @@ "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 7.2e-08, @@ -61026,7 +61450,11 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -61040,7 +61468,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -61122,7 +61553,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -61246,7 +61677,10 @@ "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2.4e-07, @@ -61257,7 +61691,10 @@ "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 7.2e-08, @@ -61267,7 +61704,11 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -61281,7 +61722,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -61363,7 +61807,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -65777,6 +66221,32 @@ "output_cost_per_token": 9e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/gemini-omni-1.1-flash-preview": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "vertex_ai", + "max_output_tokens": 57920, + "max_tokens": 57920, + "mode": "chat", + "output_cost_per_reasoning_token": 9e-06, + "output_cost_per_token": 9e-06, + "output_cost_per_video_token": 1.75e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1beta/interactions" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "video" + ], + "supports_reasoning": true, + "supports_video_input": true, + "supports_vision": true + }, "vertex_ai/gemma-4-26b-a4b-it": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -72375,5 +72845,46 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "vertex_ai/gemini-2.0-flash": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_character": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, + "litellm_provider": "vertex_ai", + "mode": "chat", + "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + }, + "vertex_ai/gemini-2.0-flash-lite": { + "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_audio_token_batches": 3.75e-08, + "input_cost_per_character": 1.875e-08, + "input_cost_per_token": 7.5e-08, + "input_cost_per_token_batches": 3.75e-08, + "litellm_provider": "vertex_ai", + "mode": "chat", + "output_cost_per_token": 3e-07, + "output_cost_per_token_batches": 1.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + }, + "vertex_ai/zai-org/glm-5.2-maas": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_regions": ["global"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py index c90f9b535ea..435db632026 100644 --- a/litellm/models/managed_files.py +++ b/litellm/models/managed_files.py @@ -61,3 +61,4 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase): litellm_params: dict[str, Any] | None = None team_id: str | None = None user_id: str | None = None + is_config: bool = False diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 87f28624d58..71dbf0da239 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -45790,6 +45790,10 @@ "title": "Custom Llm Provider", "type": "string" }, + "is_config": { + "title": "Is Config", + "type": "boolean" + }, "litellm_credential_name": { "anyOf": [ { @@ -45962,6 +45966,11 @@ "title": "Custom Llm Provider", "type": "string" }, + "is_config": { + "default": false, + "title": "Is Config", + "type": "boolean" + }, "litellm_credential_name": { "anyOf": [ { @@ -46203,7 +46212,7 @@ "paths": { "/v1/vector_store/list": { "get": { - "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth - deleted stores are removed from memory, updated stores sync to memory.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", + "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth for stores it owns: deleted stores are removed from memory, updated stores\nsync to memory. Stores declared in the config file are owned by the config file, are always listed, and are\nnever overwritten by database rows.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", "operationId": "list_vector_stores_v1_vector_store_list_get", "parameters": [ { @@ -46354,7 +46363,7 @@ }, "/vector_store/list": { "get": { - "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth - deleted stores are removed from memory, updated stores sync to memory.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", + "description": "List all available vector stores with optional filtering and pagination.\nCombines both in-memory vector stores and those stored in the database.\nDatabase is the source of truth for stores it owns: deleted stores are removed from memory, updated stores\nsync to memory. Stores declared in the config file are owned by the config file, are always listed, and are\nnever overwritten by database rows.\n\nParameters:\n- page: int - Page number for pagination (default: 1)\n- page_size: int - Number of items per page (default: 100)", "operationId": "list_vector_stores_vector_store_list_get", "parameters": [ { diff --git a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py index dbe11882b3c..2bb53c7723d 100644 --- a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py +++ b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py @@ -18,6 +18,10 @@ if TYPE_CHECKING: AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" _POLL_TIMEOUT_SECONDS: Final = 1.0 +_MAX_PENDING_PUBLISHES: Final = 1024 +_MAX_IN_FLIGHT_PUBLISHES: Final = 16 +_pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs keep background publishes alive +_in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES) _BACKOFF_INITIAL_SECONDS: Final = 5.0 _BACKOFF_MAX_SECONDS: Final = 60.0 @@ -67,6 +71,21 @@ def _message_from_data(data: object) -> _CacheInvalidationMessage | None: ) +async def _publish_to_redis(redis_cache: "RedisCache", cache_key: str, message: str) -> None: + try: + client: Final = _pubsub_capable_client(redis_cache) + if client is None: + verbose_proxy_logger.debug( + "auth cache invalidation publish for %s skipped: cluster redis client has no pub/sub support", + cache_key, + ) + return + async with _in_flight_publishes: + await client.publish(auth_cache_invalidation_channel(redis_cache), message) + except Exception as e: # noqa: BLE001 # best-effort publish; mutations must never fail on redis errors + verbose_proxy_logger.warning("auth cache invalidation publish for %s failed: %s", cache_key, e) + + async def publish_auth_cache_invalidation( cache_key: str, new_value: float | None = None, ttl: float | None = None ) -> None: @@ -80,24 +99,34 @@ async def publish_auth_cache_invalidation( writes the value into its additional in-memory caches rather than deleting the key. A spend reset uses this so the handler's self-delivered message cannot erase the freshly-written post-reset counter or floor marker. + + The Redis round trip runs as a background task: this call returns once the + publish has been handed to the event loop, so a Redis that accepts + connections but never replies costs the caller nothing. The DB write has + already committed and the local eviction already happened, so the caller + has nothing to do with the publish result. At most 16 publishes hold a + Redis connection at once; the rest wait in the task set, so a wedge cannot + drain the shared connection pool. """ redis_cache: Final = coordination_redis_cache() if redis_cache is None: return - try: - client: Final = _pubsub_capable_client(redis_cache) - if client is None: - verbose_proxy_logger.debug( - "auth cache invalidation publish for %s skipped: cluster redis client has no pub/sub support", - cache_key, - ) - return - await client.publish( - auth_cache_invalidation_channel(redis_cache), - _cache_invalidation_message_json(cache_key, new_value=new_value, ttl=ttl), + _pending_publishes.difference_update({task for task in _pending_publishes if task.done()}) + if len(_pending_publishes) >= _MAX_PENDING_PUBLISHES: + verbose_proxy_logger.warning( + "auth cache invalidation publish for %s dropped: %d publishes already waiting on redis; " + "other workers keep their cached copy until its TTL expires", + cache_key, + len(_pending_publishes), ) - except Exception as e: # noqa: BLE001 # best-effort publish; mutations must never fail on redis errors - verbose_proxy_logger.warning("auth cache invalidation publish for %s failed: %s", cache_key, e) + return + task: Final = asyncio.create_task( + _publish_to_redis( + redis_cache, cache_key, _cache_invalidation_message_json(cache_key, new_value=new_value, ttl=ttl) + ) + ) + _pending_publishes.add(task) + await asyncio.sleep(0) async def evict_and_broadcast(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache") -> None: @@ -106,8 +135,8 @@ async def evict_and_broadcast(cache_keys: Sequence[str], user_api_key_cache: "Us Every endpoint that mutates a cached object must call this: auth serves those objects cache-first with no freshness check, so a mutation that leaves the entry in place keeps the - stale object enforced until its TTL expires (LIT-3803). Best-effort on both steps: the DB write - has already committed, so a cache backend error must not fail the endpoint. + stale object enforced until its TTL expires (LIT-3803). Best-effort: the DB write has already + committed, so a cache backend error must not fail the endpoint. """ for cache_key in cache_keys: try: diff --git a/litellm/proxy/policy_engine/policy_matcher.py b/litellm/proxy/policy_engine/policy_matcher.py index e0f558b5085..2ea2def8331 100644 --- a/litellm/proxy/policy_engine/policy_matcher.py +++ b/litellm/proxy/policy_engine/policy_matcher.py @@ -12,6 +12,7 @@ from typing import Final from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.types.proxy.policy_engine import Policy, PolicyMatchContext, PolicyScope @@ -136,14 +137,46 @@ class PolicyMatcher: context: PolicyMatchContext, policies: dict[str, Policy] | None = None, ) -> Callable[[str], bool]: - """Predicate telling whether a policy exists and its condition matches the context.""" + """ + Predicate telling whether a policy exists and any policy in its + inheritance chain applies to the context. Admissions where the + policy's own condition missed but an ancestor applies are logged at + INFO, once per attachment scan. + """ resolved: Final = policies if policies is not None else PolicyMatcher._registry_policies() - return lambda policy_name: bool( - PolicyMatcher.get_policies_with_matching_conditions( - policy_names=(policy_name,), - context=context, - policies=resolved, + + def applies(policy_name: str) -> bool: + applying: Final = PolicyMatcher._applying_chain_members( + policy_name=policy_name, context=context, policies=resolved ) + if applying and policy_name not in applying: + verbose_proxy_logger.info( + "Policy '%s' applied through ancestor '%s' although its own condition did not match " + "(team_alias=%s, key_alias=%s, model=%s)", + policy_name, + applying[0], + context.team_alias, + context.key_alias, + context.model, + ) + return bool(applying) + + return applies + + @staticmethod + def _applying_chain_members( + policy_name: str, + context: PolicyMatchContext, + policies: dict[str, Policy], + ) -> tuple[str, ...]: + from litellm.proxy.policy_engine.condition_evaluator import ConditionEvaluator + + chain: Final = PolicyResolver.resolve_inheritance_chain(policy_name=policy_name, policies=policies) + return tuple( + name + for name in chain + if (policy := policies.get(name)) is not None + and (policy.condition is None or ConditionEvaluator.evaluate(policy.condition, context)) ) @staticmethod @@ -160,11 +193,14 @@ class PolicyMatcher: policies: dict[str, Policy] | None = None, ) -> list[str]: """ - Filter policies to only those whose conditions match the context. + Filter policies to only those that apply to the given context. - A policy's condition matches if: - - The policy has no condition (condition is None), OR - - The policy's condition evaluates to True for the given context + A policy applies when any policy in its inheritance chain has no + condition or a condition that evaluates to True for the context. The + resolver then drops only the chain members whose own condition fails, + so a child whose condition misses still contributes the guardrails of + its unconditional ancestors. A missing policy resolves to an empty + chain and does not apply. Args: policy_names: List of policy names to filter @@ -172,19 +208,11 @@ class PolicyMatcher: policies: Dictionary of all policies (if None, uses global registry) Returns: - List of policy names whose conditions match the context + List of policy names that apply to the context """ - from litellm.proxy.policy_engine.condition_evaluator import ConditionEvaluator - resolved: Final = policies if policies is not None else PolicyMatcher._registry_policies() - - matching_policies: Final = [] - for policy_name in policy_names: - policy = resolved.get(policy_name) - if policy is None: - continue - # Policy matches if it has no condition OR condition evaluates to True - if policy.condition is None or ConditionEvaluator.evaluate(policy.condition, context): - matching_policies.append(policy_name) - - return matching_policies + return [ + policy_name + for policy_name in policy_names + if PolicyMatcher._applying_chain_members(policy_name, context, resolved) + ] diff --git a/litellm/proxy/policy_engine/policy_resolver.py b/litellm/proxy/policy_engine/policy_resolver.py index 70503f85b03..e1422d79e15 100644 --- a/litellm/proxy/policy_engine/policy_resolver.py +++ b/litellm/proxy/policy_engine/policy_resolver.py @@ -210,6 +210,7 @@ class PolicyResolver: Returns: List of (policy_name, GuardrailPipeline) tuples """ + from litellm.proxy.policy_engine.condition_evaluator import ConditionEvaluator from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -230,6 +231,11 @@ class PolicyResolver: policy = policies.get(policy_name) if policy is None: continue + if policy.condition is not None and not ConditionEvaluator.evaluate( + condition=policy.condition, context=context + ): + verbose_proxy_logger.debug("Policy '%s' condition did not match, skipping pipeline", policy_name) + continue if policy.pipeline is not None: pipelines.append((policy_name, policy.pipeline)) verbose_proxy_logger.debug( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c7ce9d081d6..ce2f97d6d55 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1004,6 +1004,8 @@ def _stamp_deployment_attribution( return attribution metadata.setdefault("model_info", attribution["model_info"]) metadata.setdefault("deployment", attribution["deployment"]) + if isinstance(model_group, str): + metadata.setdefault("model_group", model_group) return attribution diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 2fb6813a471..cae144bb266 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -13,6 +13,7 @@ import json from typing import TYPE_CHECKING, Any, Final from fastapi import APIRouter, Depends, HTTPException +from typing_extensions import ReadOnly, TypedDict if TYPE_CHECKING: from prisma.models import LiteLLM_ManagedVectorStoresTable as _VectorStoreRow @@ -56,6 +57,32 @@ def _row_to_vector_store(row: "_VectorStoreRow") -> LiteLLM_ManagedVectorStore: return LiteLLM_ManagedVectorStore(**row.model_dump()) +class _ConfigOwnedDetail(TypedDict): + error: ReadOnly[str] + vector_store_id: ReadOnly[str] + + +def _raise_if_config_owned(vector_store_id: str) -> None: + if litellm.vector_store_registry is None or not litellm.vector_store_registry.is_config_vector_store( + vector_store_id + ): + return + detail: Final[_ConfigOwnedDetail] = { + "error": ( + f"Vector store {vector_store_id} is defined in the config file, so the config file owns it and it " + "cannot be changed here. Edit the config file to change it, or remove it from the file to let the " + "database own it." + ), + "vector_store_id": vector_store_id, + } + raise HTTPException(status_code=400, detail=detail) + + +def _with_ownership(vector_store: LiteLLM_ManagedVectorStore) -> LiteLLM_ManagedVectorStore: + ownership: Final = LiteLLM_ManagedVectorStore(is_config=vector_store.get("is_config", False)) + return vector_store | ownership + + _LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=frozenset(("connection",))) @@ -274,6 +301,7 @@ async def new_vector_store( status_code=400, detail="vector_store_id and custom_llm_provider are required", ) + _raise_if_config_owned(vector_store_id) # Extract and validate metadata metadata: Final = vector_store.get("vector_store_metadata") @@ -306,6 +334,8 @@ async def new_vector_store( "message": f"Vector store {vector_store.get('vector_store_id')} created successfully", "vector_store": response_vs, } + except HTTPException: + raise except Exception as e: verbose_proxy_logger.exception("Error creating vector store: %s", e) raise HTTPException(status_code=500, detail=str(e)) @@ -331,7 +361,9 @@ async def list_vector_stores( """ List all available vector stores with optional filtering and pagination. Combines both in-memory vector stores and those stored in the database. - Database is the source of truth - deleted stores are removed from memory, updated stores sync to memory. + Database is the source of truth for stores it owns: deleted stores are removed from memory, updated stores + sync to memory. Stores declared in the config file are owned by the config file, are always listed, and are + never overwritten by database rows. Parameters: - page: int - Page number for pagination (default: 1) @@ -366,8 +398,10 @@ async def list_vector_stores( if not vector_store_id: continue + if vector_store.get("is_config", False): + vector_store_map[vector_store_id] = vector_store # If vector store is in memory but NOT in database, it was deleted - if vector_store_id not in db_vector_store_ids: + elif vector_store_id not in db_vector_store_ids: verbose_proxy_logger.info( "Vector store %s exists in memory but not in database - marking for deletion from cache", vector_store_id, @@ -394,7 +428,7 @@ async def list_vector_stores( # Filter vector stores based on access control accessible_vector_stores: Final = [] for vs in await filter_listable_vector_stores(vector_store_map.values(), user_api_key_dict): - redacted = LiteLLM_ManagedVectorStore(**vs) + redacted = _with_ownership(vs) redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params")) accessible_vector_stores.append(redacted) @@ -467,6 +501,7 @@ async def delete_vector_store( status_code=404, detail=f"Vector store with ID {data.vector_store_id} not found", ) + _raise_if_config_owned(data.vector_store_id) # Check access control if vector_store_to_check and not await _check_vector_store_access(vector_store_to_check, user_api_key_dict): @@ -545,6 +580,7 @@ async def get_vector_store_info( litellm_params=_redact_sensitive_litellm_params(vector_store.get("litellm_params")), team_id=vector_store.get("team_id") or None, user_id=vector_store.get("user_id") or None, + is_config=vector_store.get("is_config", False), ) return {"vector_store": vector_store_pydantic_obj} @@ -591,6 +627,7 @@ async def update_vector_store( update_data: Final = data.model_dump(exclude_unset=True) vector_store_id: Final[str] = data.vector_store_id update_data.pop("vector_store_id") + _raise_if_config_owned(vector_store_id) # Per-store access control: anyone authenticated who passes the # premium-feature gate could otherwise update *any* vector store — diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index f794dc4a99e..355a5e5062e 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -12,6 +12,22 @@ class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... +@final +class NativeDiagnosticProcessor: + def __new__(cls, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ... + def redact_text(self, text: str) -> str: ... + def redact_structured_text(self, key: str | None, text: str) -> str: ... + def redact_client_message(self, text: str) -> str: ... + def process_diagnostic( + self, + message: str, + exception: str | None, + stack: str | None, + leaves: Sequence[tuple[str | None, str]], + policy: tuple[bool, int, int], + ) -> tuple[str, str | None, str | None, list[str], bool]: ... + def scrub_access_arguments(self, arguments: Sequence[str]) -> list[str]: ... + def ocr( request: LiteLLMOcrRequest, args: tuple[object, ...], @@ -335,6 +351,7 @@ def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", + "NativeDiagnosticProcessor", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index d6cbe17d790..1a2d153871d 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -86,11 +86,25 @@ class SecretManagerRule: return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems) -Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext -Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule +@dataclass(frozen=True, slots=True) +class LoggerContext: + pass + + +@dataclass(frozen=True, slots=True) +class LoggerRule: + rollout: Rollout + + def matches(self, context: Context) -> bool: + return isinstance(context, LoggerContext) + + +Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext | LoggerContext +Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule | LoggerRule Rules: TypeAlias = tuple[Rule, ...] RULES: Final[Rules] = ( + LoggerRule(Rollout.RUST_OPT_IN), RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), diff --git a/litellm/rust_bridge/diagnostics.py b/litellm/rust_bridge/diagnostics.py new file mode 100644 index 00000000000..fbe1b8122d7 --- /dev/null +++ b/litellm/rust_bridge/diagnostics.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from functools import lru_cache +from typing import Final, Protocol, TypeVar, cast + +from litellm.rust_bridge.bindings import NativeBinding + +ResultT: Final = TypeVar("ResultT") + + +class NativeDiagnosticProcessor(Protocol): + def redact_text(self, text: str) -> str: ... + def redact_structured_text(self, key: str | None, text: str) -> str: ... + def redact_client_message(self, text: str) -> str: ... + def process_diagnostic( + self, + message: str, + exception: str | None, + stack: str | None, + leaves: tuple[tuple[str | None, str], ...], + policy: tuple[bool, int, int], + ) -> tuple[str, str | None, str | None, Sequence[str], bool]: ... + def scrub_access_arguments(self, arguments: tuple[str, ...]) -> Sequence[str]: ... + + +class NativeDiagnosticFactory(Protocol): + def __call__(self, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ... + + +def _as_factory(value: object) -> NativeDiagnosticFactory | None: + if not isinstance(value, type): + return None + return cast(NativeDiagnosticFactory, value) # cast-ok: PyO3 factory must be a type + + +PROCESSOR: Final = NativeBinding("NativeDiagnosticProcessor", validate=_as_factory) + + +@lru_cache(maxsize=4) +def _construct(factory: NativeDiagnosticFactory, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: + return factory(minimum_custom_key_length) + + +def run(native: Callable[[NativeDiagnosticProcessor], ResultT], python: Callable[[], ResultT]) -> ResultT: + from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH + from litellm.rust_bridge.catalog import LoggerContext, decision + from litellm.rust_bridge.configuration import Decision + + selected: Final = decision(LoggerContext()) + if selected is Decision.PYTHON: + return python() + factory: Final = PROCESSOR.load() + if factory is None: + return python() + try: + return native(_construct(factory, MINIMUM_CUSTOM_KEY_LENGTH)) + except Exception: + return python() diff --git a/litellm/rust_bridge/logger.py b/litellm/rust_bridge/logger.py new file mode 100644 index 00000000000..bcd53852a2f --- /dev/null +++ b/litellm/rust_bridge/logger.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +from pydantic import JsonValue + +from litellm._logging import ( + CorrelationContextFilter, + DiagnosticProcessingFilter, + session_id_var, + set_session_id, + set_trace_id, + trace_id_var, + verbose_logger, +) + +_REDACTION: Final = DiagnosticProcessingFilter() +_CORRELATION: Final = CorrelationContextFilter() + + +def context() -> tuple[str, str]: + return session_id_var.get(), trace_id_var.get() + + +def enabled(level: int) -> bool: + return verbose_logger.isEnabledFor(level) + + +def emit( + level: int, + message: str, + pathname: str, + lineno: int, + target: str, + fields: Mapping[str, JsonValue], + correlation: tuple[str, str], +) -> None: + if not enabled(level): + return + session_token: Final = set_session_id(correlation[0]) + trace_token: Final = set_trace_id(correlation[1]) + try: + record: Final = verbose_logger.makeRecord( + verbose_logger.name, + level, + pathname, + lineno, + message, + (), + None, + func=target, + extra={ + "rust_target": target, + "rust_fields": dict(fields), + }, # mutable-ok: LogRecord requires JSON dict extras + ) + _REDACTION.filter(record) + _CORRELATION.filter(record) + verbose_logger.handle(record) + finally: + trace_id_var.reset(trace_token) + session_id_var.reset(session_token) diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index 866f2fce989..46e96b60620 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -58,12 +58,6 @@ class SecretManagerBinding: settings_object: object -def warn(message: str) -> None: - from litellm._logging import verbose_logger - - verbose_logger.warning("%s", message) - - def secret_manager() -> SecretManager: from litellm.secret_managers.main import ( _should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index 6d2ca308798..34c14e6b042 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -44,6 +44,8 @@ class LiteLLM_ManagedVectorStore(TypedDict, total=False): team_id: str | None user_id: str | None + is_config: ReadOnly[bool] + class LiteLLM_ManagedVectorStoreListResponse(TypedDict, total=False): """Response format for listing vector stores""" diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index c7aed77286c..e78d2aa5f6a 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -340,7 +340,7 @@ class VectorStoreRegistry: # Verify vector store still exists in database (if we have DB access) # This ensures deleted vector stores are removed from cache - if vector_store is not None and prisma_client is not None: + if vector_store is not None and prisma_client is not None and not vector_store.get("is_config", False): try: # Check if it still exists in database db_vector_store = await ManagedVectorStoresRepository(prisma_client).table.find_unique( @@ -426,6 +426,7 @@ class VectorStoreRegistry: vector_store_metadata=vector_store_litellm_params.get("vector_store_metadata"), created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc), + is_config=True, ) self.vector_stores.append(litellm_managed_vector_store) @@ -452,6 +453,10 @@ class VectorStoreRegistry: return response + def is_config_vector_store(self, vector_store_id: str) -> bool: + vector_store: Final = self.get_litellm_managed_vector_store_from_registry(vector_store_id=vector_store_id) + return vector_store is not None and vector_store.get("is_config", False) + def add_vector_store_to_registry(self, vector_store: LiteLLM_ManagedVectorStore): """ Add a vector store to the registry @@ -475,10 +480,11 @@ class VectorStoreRegistry: ] def update_vector_store_in_registry(self, vector_store_id: str, updated_data: LiteLLM_ManagedVectorStore): - """Update or add a vector store in the registry""" + """Update or add a vector store in the registry. Config-defined stores are left untouched""" for i, vector_store in enumerate(self.vector_stores): if vector_store.get("vector_store_id") == vector_store_id: - self.vector_stores[i] = updated_data + if not vector_store.get("is_config", False): + self.vector_stores[i] = updated_data return self.vector_stores.append(updated_data) diff --git a/migrations/Dockerfile b/migrations/Dockerfile index f34940c0ce0..255b94b0ea8 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ad66cb76f8b..ae02744b8e1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -81,7 +81,8 @@ "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 1.25e-05 + "output_cost_per_token": 1.25e-05, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.j2-ultra-v1": { "input_cost_per_token": 1.88e-05, @@ -90,7 +91,8 @@ "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 1.88e-05 + "output_cost_per_token": 1.88e-05, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.jamba-1-5-large-v1:0": { "deprecation_date": "2026-11-26", @@ -100,7 +102,8 @@ "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 8e-06 + "output_cost_per_token": 8e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.jamba-1-5-mini-v1:0": { "deprecation_date": "2026-11-26", @@ -110,7 +113,8 @@ "max_output_tokens": 256000, "max_tokens": 256000, "mode": "chat", - "output_cost_per_token": 4e-07 + "output_cost_per_token": 4e-07, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "ai21.jamba-instruct-v1:0": { "input_cost_per_token": 5e-07, @@ -120,6 +124,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_system_messages": true }, "aiml/dall-e-2": { @@ -295,6 +300,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -306,6 +312,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -317,6 +324,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -328,6 +336,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true }, @@ -339,7 +348,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-vision-7b.html", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_vision": true }, "amazon.nova-lite-v1:0": { @@ -814,7 +823,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -1048,7 +1057,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1083,7 +1093,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1118,7 +1129,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1153,7 +1165,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1188,7 +1201,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1223,7 +1237,8 @@ "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1255,7 +1270,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1309,7 +1324,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1347,7 +1362,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1385,12 +1400,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1422,12 +1438,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1695,7 +1712,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -2004,7 +2022,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2081,7 +2100,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2158,7 +2178,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2231,7 +2252,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2270,7 +2291,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2309,7 +2330,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2348,12 +2369,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2386,12 +2408,13 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2424,16 +2447,18 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -2460,11 +2485,12 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2619,7 +2645,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2657,7 +2684,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2695,7 +2723,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2834,7 +2863,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2868,7 +2898,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2902,7 +2933,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2934,7 +2966,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2971,7 +3004,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, - "output_cost_per_token_batches": 7.5e-06 + "output_cost_per_token_batches": 7.5e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -3213,7 +3247,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -3262,7 +3297,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -11845,6 +11881,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { @@ -11858,6 +11897,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { @@ -11872,6 +11914,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { @@ -11897,6 +11942,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { @@ -11910,6 +11957,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/moonshotai.kimi-k2-thinking": { @@ -11937,7 +11987,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, @@ -11969,6 +12021,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { @@ -11982,6 +12037,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { @@ -11996,6 +12054,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { @@ -12021,6 +12082,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { @@ -12034,6 +12097,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { @@ -12048,6 +12114,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.236e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { @@ -12062,6 +12131,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { @@ -12075,6 +12147,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { @@ -12089,6 +12164,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { @@ -12103,6 +12181,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { @@ -12116,6 +12196,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { @@ -12148,6 +12231,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { @@ -12161,6 +12247,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { @@ -12175,6 +12264,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { @@ -12189,6 +12281,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { @@ -12289,6 +12383,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { @@ -12303,6 +12400,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { @@ -12316,6 +12416,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { @@ -12347,6 +12450,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { @@ -12361,6 +12467,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { @@ -12374,6 +12483,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { @@ -12405,6 +12517,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { @@ -12419,6 +12534,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.86e-06 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { @@ -12449,6 +12567,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { @@ -12462,13 +12583,14 @@ "supports_tool_choice": true }, "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 1.04e-05, + "input_cost_per_token": 5.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 3.12e-05, + "output_cost_per_token": 1.56e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": { @@ -12492,6 +12614,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/eu-south-1/minimax.minimax-m2.5": { @@ -12506,6 +12631,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { @@ -12519,6 +12647,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { @@ -12569,6 +12700,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { @@ -12582,6 +12716,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { @@ -12596,6 +12733,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.44e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { @@ -12621,6 +12761,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { @@ -12634,6 +12776,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { @@ -12753,13 +12898,14 @@ "supports_tool_choice": true }, "bedrock/us-east-1/mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 4e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 1.2e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": { @@ -12784,6 +12930,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.1": { @@ -12797,6 +12946,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/minimax.minimax-m2.5": { @@ -12810,6 +12962,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { @@ -12835,6 +12990,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { @@ -12848,6 +13005,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/deepseek.v3.2": { @@ -12862,6 +13022,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.1": { @@ -12875,6 +13038,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/minimax.minimax-m2.5": { @@ -12889,6 +13055,9 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "output_cost_per_token": 1.2e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { @@ -12914,6 +13083,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { @@ -12927,6 +13098,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { @@ -13354,13 +13528,14 @@ "supports_tool_choice": true }, "bedrock/us-west-2/mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 4e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 1.2e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": { @@ -13385,6 +13560,9 @@ "supports_reasoning": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.1": { @@ -13398,6 +13576,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/minimax.minimax-m2.5": { @@ -13411,6 +13592,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { @@ -13436,6 +13620,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, + "supports_audio_input": false, + "supports_response_schema": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { @@ -13449,6 +13635,9 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { @@ -14656,16 +14845,18 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "cohere.command-text-v14": { - "input_cost_per_token": 1.5e-06, + "input_cost_per_token": 1e-06, "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "cohere.embed-english-v3": { @@ -14675,6 +14866,7 @@ "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_embedding_image_input": true }, "cohere.embed-multilingual-v3": { @@ -14684,6 +14876,7 @@ "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_embedding_image_input": true }, "cohere.embed-v4:0": { @@ -14694,6 +14887,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_embedding_image_input": true }, "us.cohere.embed-v4:0": { @@ -20867,6 +21061,7 @@ "max_tokens": 81920, "mode": "chat", "output_cost_per_token": 1.68e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -21342,7 +21537,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -21511,7 +21706,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -21548,7 +21744,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -25846,7 +26043,7 @@ "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/vertex_ai/live", "/v1/realtime" @@ -25882,7 +26079,6 @@ "input_cost_per_image_token": 3e-06 }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -25911,7 +26107,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -25927,7 +26123,6 @@ "input_cost_per_image_token": 3e-06 }, "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025": { - "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -25957,7 +26152,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -29709,7 +29904,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, - "output_cost_per_token_batches": 7.5e-06 + "output_cost_per_token_batches": 7.5e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -29741,7 +29937,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -29755,7 +29952,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -34537,7 +34734,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34551,7 +34749,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -35128,7 +35326,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1e-06 + "output_cost_per_token": 1e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "meta.llama2-70b-chat-v1": { "input_cost_per_token": 1.95e-06, @@ -35137,7 +35336,8 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 2.56e-06 + "output_cost_per_token": 2.56e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, @@ -35837,16 +36037,18 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "mistral.mistral-large-2402-v1:0": { - "input_cost_per_token": 8e-06, + "input_cost_per_token": 4e-06, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", - "output_cost_per_token": 2.4e-05, + "output_cost_per_token": 1.2e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "mistral.mistral-large-2407-v1:0": { @@ -35878,12 +36080,15 @@ }, "mistral.mistral-small-2402-v1:0": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 8191, "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true }, "mistral.mixtral-8x7b-instruct-v0:1": { @@ -35894,6 +36099,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 7e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_tool_choice": true }, "mistral.voxtral-mini-3b-2507": { @@ -35904,6 +36110,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true @@ -35916,6 +36123,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true @@ -39101,6 +39309,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -39114,6 +39323,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42315,7 +42525,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -42328,7 +42541,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.545e-07, @@ -42341,7 +42557,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -42354,7 +42573,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, @@ -42367,7 +42589,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -42380,7 +42605,10 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_native_structured_output": true, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "qwen.qwen3-vl-235b-a22b": { "input_cost_per_token": 5.3e-07, @@ -44356,7 +44584,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -44467,7 +44695,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -44504,7 +44733,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, - "output_cost_per_token_batches": 8.25e-06 + "output_cost_per_token_batches": 8.25e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -44534,7 +44764,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us-gov.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -44592,7 +44823,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -44718,7 +44949,10 @@ "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "us-gov.nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2.4e-07, @@ -44729,7 +44963,10 @@ "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true }, "us-gov.nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 7.2e-08, @@ -44739,7 +44976,11 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": false }, "us-gov.nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -44753,7 +44994,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -44821,7 +45065,8 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -44879,7 +45124,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -44911,19 +45157,21 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -44942,7 +45190,8 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -44974,7 +45223,8 @@ "supports_tool_choice": true, "supports_vision": true, "bedrock_converse_supports_strict_tools": false, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://aws.amazon.com/bedrock/pricing/" }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -44984,6 +45234,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.4e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": false, "supports_reasoning": true, "supports_tool_choice": false @@ -44999,7 +45250,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "eu.deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -45012,7 +45266,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "us.meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, @@ -45154,6 +45411,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_tool_choice": false }, @@ -46476,16 +46734,20 @@ "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -46501,16 +46763,20 @@ "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, + "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -46633,14 +46899,18 @@ "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -46657,20 +46927,25 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-5@20251101": { "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -46688,7 +46963,8 @@ "supports_vision": true, "supports_native_streaming": true, "supports_output_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-6": { "deprecation_date": "2027-02-05", @@ -46697,14 +46973,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46721,7 +47001,8 @@ "supports_vision": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-6@default": { "deprecation_date": "2027-02-05", @@ -46730,14 +47011,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46754,7 +47039,8 @@ "supports_vision": true, "supports_output_config": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-7": { "deprecation_date": "2027-04-16", @@ -46762,14 +47048,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46787,7 +47077,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-7@default": { "deprecation_date": "2027-04-16", @@ -46795,14 +47086,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46820,7 +47115,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 2048 + "prompt_cache_min_tokens": 2048, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5": { "deprecation_date": "2027-06-08", @@ -46828,14 +47124,18 @@ "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46855,14 +47155,17 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5-1": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "input_cost_per_token": 1e-05, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, @@ -46892,7 +47195,10 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512, - "deprecation_date": "2027-03-01" + "deprecation_date": "2027-03-01", + "input_cost_per_token_batches": 5e-06, + "output_cost_per_token_batches": 2.5e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", @@ -46900,14 +47206,18 @@ "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, "input_cost_per_token": 1e-05, + "input_cost_per_token_batches": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, + "output_cost_per_token_batches": 2.5e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46927,14 +47237,17 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-fable-5-1@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_creation_input_token_cost_batches": 6.25e-06, "cache_read_input_token_cost": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "input_cost_per_token": 1e-05, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, @@ -46964,7 +47277,10 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512, - "deprecation_date": "2027-03-01" + "deprecation_date": "2027-03-01", + "input_cost_per_token_batches": 5e-06, + "output_cost_per_token_batches": 2.5e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5": { "deprecation_date": "2027-01-24", @@ -46973,14 +47289,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -46998,7 +47318,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5@default": { "deprecation_date": "2027-01-24", @@ -47007,14 +47328,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47032,21 +47357,26 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5-5": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_creation_input_token_cost_batches": 2.5e-06, "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47068,21 +47398,26 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_creation_input_token_cost_batches": 2.5e-06, "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47104,7 +47439,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-8": { "deprecation_date": "2027-05-28", @@ -47113,14 +47449,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47138,7 +47478,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-opus-4-8@default": { "deprecation_date": "2027-05-28", @@ -47147,14 +47488,18 @@ "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_batches": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "output_cost_per_token_batches": 1.25e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47172,18 +47517,21 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-5": { "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, @@ -47202,7 +47550,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-5": { "deprecation_date": "2026-12-24", @@ -47212,12 +47561,14 @@ "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -47236,7 +47587,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-6": { "regional_endpoint_uplift_multiplier": 1.1, @@ -47244,14 +47596,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_batches": 1.88e-06, "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -47268,18 +47624,21 @@ "search_context_size_medium": 0.01 }, "supports_output_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-5@20250929": { "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, @@ -47299,7 +47658,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -47309,6 +47669,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47320,6 +47681,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47331,6 +47693,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47342,6 +47705,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -47379,14 +47743,17 @@ "supports_tool_choice": true }, "vertex_ai/deepseek-ai/deepseek-v3.1-maas": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.7e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 8.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "us-central1" ], @@ -47397,6 +47764,7 @@ "supports_tool_choice": true }, "vertex_ai/deepseek-ai/deepseek-v3.2-maas": { + "cache_read_input_token_cost": 5.6e-08, "input_cost_per_token": 5.6e-07, "input_cost_per_token_batches": 2.8e-07, "litellm_provider": "vertex_ai-deepseek_models", @@ -47406,7 +47774,7 @@ "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -47418,13 +47786,15 @@ }, "vertex_ai/deepseek-ai/deepseek-r1-0528-maas": { "input_cost_per_token": 1.35e-06, + "input_cost_per_token_batches": 6.75e-07, "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 65336, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 2.7e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "us-central1" ], @@ -48000,13 +48370,15 @@ }, "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 5.75e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_modalities": [ "text", "image" @@ -48060,13 +48432,15 @@ }, "vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "output_cost_per_token_batches": 3.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_modalities": [ "text", "image" @@ -48112,6 +48486,7 @@ "supports_tool_choice": true }, "vertex_ai/minimaxai/minimax-m2-maas": { + "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-minimax_models", "max_input_tokens": 196608, @@ -48119,11 +48494,12 @@ "max_tokens": 196608, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, "vertex_ai/moonshotai/kimi-k2-thinking-maas": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 6e-07, "litellm_provider": "vertex_ai-moonshot_models", "max_input_tokens": 256000, @@ -48131,12 +48507,13 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true, "supports_web_search": true }, "vertex_ai/zai-org/glm-4.7-maas": { + "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 6e-07, "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, @@ -48144,7 +48521,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48161,7 +48538,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48178,6 +48555,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48189,6 +48567,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48200,6 +48579,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48211,6 +48591,7 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_tool_choice": true }, @@ -48291,7 +48672,7 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/mistral-small-2503@001": { "input_cost_per_token": 1e-07, @@ -48303,7 +48684,7 @@ "output_cost_per_token": 3e-07, "supports_function_calling": true, "supports_tool_choice": true, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/mistral-ocr-2505": { "litellm_provider": "vertex_ai", @@ -48320,7 +48701,7 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "ocr_cost_per_page": 0.0003, - "source": "https://cloud.google.com/vertex-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "us-central1" ] @@ -48343,13 +48724,15 @@ }, "vertex_ai/openai/gpt-oss-120b-maas": { "input_cost_per_token": 9e-08, + "input_cost_per_token_batches": 4.5e-08, "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.6e-07, - "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas", + "output_cost_per_token_batches": 1.8e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_reasoning": true }, "vertex_ai/openai/gpt-oss-20b-maas": { @@ -48360,9 +48743,11 @@ "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_reasoning": true, - "cache_read_input_token_cost": 7e-09 + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token_batches": 3.5e-08, + "output_cost_per_token_batches": 1.25e-07 }, "vertex_ai/xai/grok-4.1-fast-non-reasoning": { "cache_read_input_token_cost": 5e-08, @@ -48373,7 +48758,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/developers/models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, @@ -48390,7 +48775,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/developers/models", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -48482,13 +48867,15 @@ }, "vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "output_cost_per_token_batches": 4.4e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global", "us-south1" @@ -48497,14 +48884,17 @@ "supports_tool_choice": true }, "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": { + "cache_read_input_token_cost": 2.2e-08, "input_cost_per_token": 2.2e-07, + "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "output_cost_per_token_batches": 9e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48519,7 +48909,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -48534,7 +48924,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_regions": [ "global" ], @@ -54020,7 +54410,7 @@ "litellm_provider": "openai", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, - "source": "https://platform.openai.com/docs/api-reference/videos", + "source": "https://developers.openai.com/api/docs/pricing", "supported_modalities": [ "text", "image" @@ -54735,12 +55125,14 @@ "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -54759,7 +55151,8 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/claude-sonnet-4-6@default": { "regional_endpoint_uplift_multiplier": 1.1, @@ -54767,14 +55160,18 @@ "supports_legacy_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_batches": 1.88e-06, "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, + "output_cost_per_token_batches": 7.5e-06, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -54791,7 +55188,8 @@ "search_context_size_medium": 0.01 }, "supports_output_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -54984,14 +55382,14 @@ "supports_vision": true }, "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": { - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_272k_tokens": 1.1e-05, - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, - "output_cost_per_token": 3.3e-05, - "output_cost_per_token_above_272k_tokens": 4.95e-05, + "input_cost_per_token": 4.4e-06, + "input_cost_per_token_above_272k_tokens": 8.8e-06, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.1e-05, + "cache_read_input_token_cost": 4.4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8.8e-07, + "output_cost_per_token": 2.2e-05, + "output_cost_per_token_above_272k_tokens": 3.3e-05, "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -55074,6 +55472,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55087,7 +55486,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "global.openai.gpt-5.6-sol": { "input_cost_per_token": 4e-06, @@ -55103,6 +55503,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55116,7 +55517,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "us.openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, @@ -55132,6 +55534,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55145,7 +55548,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "global.openai.gpt-5.6-terra": { "input_cost_per_token": 2e-06, @@ -55161,6 +55565,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55174,7 +55579,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "us.openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, @@ -55190,6 +55596,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55203,7 +55610,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "global.openai.gpt-5.6-luna": { "input_cost_per_token": 2e-07, @@ -55219,6 +55627,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supported_modalities": [ "text", "image" @@ -55232,7 +55641,8 @@ "supports_tool_choice": true, "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, - "supports_vision": true + "supports_vision": true, + "supports_sampling_params": false }, "bedrock_mantle/openai.gpt-6-astra": { "input_cost_per_token": 1.1e-05, @@ -55302,7 +55712,7 @@ "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "global.openai.gpt-6-astra": { "input_cost_per_token": 1e-05, @@ -55334,7 +55744,7 @@ "supports_reasoning": true, "supports_xhigh_reasoning_effort": true, "supports_vision": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html" + "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -55548,6 +55958,7 @@ "max_output_tokens": 500000, "max_tokens": 500000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, @@ -55563,6 +55974,7 @@ "max_output_tokens": 500000, "max_tokens": 500000, "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, @@ -55766,6 +56178,9 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-west-2/zai.glm-5": { @@ -55781,6 +56196,9 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false, "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { @@ -57107,10 +57525,10 @@ }, "gemini/gemini-robotics-er-2-streaming-preview": { "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 2e-06, + "input_cost_per_token": 1e-06, "litellm_provider": "gemini", "mode": "chat", - "output_cost_per_token": 1e-05, + "output_cost_per_token": 5e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, @@ -61005,7 +61423,10 @@ "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2.4e-07, @@ -61016,7 +61437,10 @@ "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 7.2e-08, @@ -61026,7 +61450,11 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -61040,7 +61468,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -61122,7 +61553,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -61246,7 +61677,10 @@ "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2.4e-07, @@ -61257,7 +61691,10 @@ "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 7.2e-08, @@ -61267,7 +61704,11 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, - "supports_system_messages": true + "supports_system_messages": true, + "supports_audio_input": false, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -61281,7 +61722,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_audio_input": false, + "supports_response_schema": true, + "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -61363,7 +61807,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -65777,6 +66221,32 @@ "output_cost_per_token": 9e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/gemini-omni-1.1-flash-preview": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "vertex_ai", + "max_output_tokens": 57920, + "max_tokens": 57920, + "mode": "chat", + "output_cost_per_reasoning_token": 9e-06, + "output_cost_per_token": 9e-06, + "output_cost_per_video_token": 1.75e-05, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1beta/interactions" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "video" + ], + "supports_reasoning": true, + "supports_video_input": true, + "supports_vision": true + }, "vertex_ai/gemma-4-26b-a4b-it": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -72375,5 +72845,46 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "vertex_ai/gemini-2.0-flash": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_batches": 5e-07, + "input_cost_per_character": 3.75e-08, + "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, + "litellm_provider": "vertex_ai", + "mode": "chat", + "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + }, + "vertex_ai/gemini-2.0-flash-lite": { + "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_audio_token_batches": 3.75e-08, + "input_cost_per_character": 1.875e-08, + "input_cost_per_token": 7.5e-08, + "input_cost_per_token_batches": 3.75e-08, + "litellm_provider": "vertex_ai", + "mode": "chat", + "output_cost_per_token": 3e-07, + "output_cost_per_token_batches": 1.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + }, + "vertex_ai/zai-org/glm-5.2-maas": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_regions": ["global"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/pyproject.toml b/pyproject.toml index f447343ff33..ba72378989a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.103.0" +version = "1.104.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -75,8 +75,8 @@ proxy = [ "mcp>=2.2.0,<3", "httpx2>=2.5.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.100", - "litellm-enterprise==0.1.69", + "litellm-proxy-extras==0.4.101", + "litellm-enterprise==0.1.70", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", @@ -355,7 +355,7 @@ litellm-enterprise = { workspace = true } members = ["enterprise", "litellm-proxy-extras"] [tool.commitizen] -version = "1.103.0" +version = "1.104.0" version_files = [ "pyproject.toml:^version", ] diff --git a/scripts/comment-fixed-issue.test.ts b/scripts/comment-fixed-issue.test.ts index f9cd41d96d8..fef5c0f4007 100644 --- a/scripts/comment-fixed-issue.test.ts +++ b/scripts/comment-fixed-issue.test.ts @@ -3,62 +3,140 @@ import { describe, expect, test } from "bun:test"; import type { Comment, GitHubApi } from "./auto-close-duplicates"; import { FIXED_MARKER, + OPEN_PULL_REQUESTS_QUERY, + SUPERSEDED_MARKER, + closeVerdict, closerOf, - commentFixedIssue, + describeIssue, + describeSweep, + fixOf, fixedBody, + handleFixedIssue, nextMinor, parseVersion, placement, readConfig, releaseCandidate, + supersededBody, + sweep, type ClosedIssue, type FixedConfig, + type IssueClosure, + type LinkedIssue, + type LinkedPullRequest, + type PullRequestsPage, } from "./comment-fixed-issue"; const MERGE_COMMIT = "68c4c82ac977b48b2b81ee8d633d5771307c6162"; +const ISSUE = 41750; const mergedPr = { __typename: "PullRequest" as const, number: 41767, merged: true, baseRefName: "main", + repository: { nameWithOwner: "BerriAI/litellm" }, mergeCommit: { oid: MERGE_COMMIT }, }; -type Closer = ClosedIssue["timelineItems"]["nodes"][number]["closer"]; +const commitCloser = { __typename: "Commit" as const, oid: MERGE_COMMIT, repository: { nameWithOwner: "BerriAI/litellm" } }; -const closedBy = (closer: Closer, state: ClosedIssue["state"] = "CLOSED"): ClosedIssue => ({ +type Closer = IssueClosure["timelineItems"]["nodes"][number]["closer"]; + +const closure = (closer: Closer, state: IssueClosure["state"] = "CLOSED"): IssueClosure => ({ state, timelineItems: { nodes: [{ closer }] }, }); +const page = (pages: readonly (readonly LinkedPullRequest[])[], index: number): PullRequestsPage => ({ + pageInfo: { hasNextPage: index + 1 < pages.length, endCursor: String(index + 1) }, + nodes: pages[index] ?? [], +}); + +const cursorIndex = (after: string | null): number => (after === null ? 0 : Number(after)); + +const closedBy = (closer: Closer, state?: IssueClosure["state"], linked: readonly LinkedPullRequest[] = []): ClosedIssue => ({ + ...closure(closer, state), + closedByPullRequestsReferences: page([linked], 0), +}); + +const reopenedAt = (createdAt: string): LinkedPullRequest["reopens"] => ({ nodes: [{ createdAt }] }); + +const linkedIssue = (number: number, closer: Closer = mergedPr, state?: IssueClosure["state"]): LinkedIssue => ({ + number, + repository: { nameWithOwner: "BerriAI/litellm" }, + ...closure(closer, state), +}); + +const links = (...issues: readonly LinkedIssue[]): LinkedPullRequest["closingIssuesReferences"] => ({ totalCount: issues.length, nodes: issues }); + +const openPr = (number: number, overrides: Partial = {}): LinkedPullRequest => ({ + number, + state: "OPEN", + baseRefName: "main", + repository: { nameWithOwner: "BerriAI/litellm" }, + closingIssuesReferences: links(linkedIssue(ISSUE)), + reopens: { nodes: [] }, + ...overrides, +}); + const pyproject = (version: string): string => `[project]\nname = "litellm"\nversion = "${version}"\n\n[tool.commitizen]\nversion = "${version}"\n`; -const config: FixedConfig = { repo: "BerriAI/litellm", issueNumber: 41750, defaultBranch: "main", dryRun: false }; +const config: FixedConfig = { repo: "BerriAI/litellm", defaultBranch: "main", commentDryRun: false, closeDryRun: false }; + +const prFix = (issue = ISSUE, number = 41767) => ({ issue, source: { kind: "pull_request" as const, number, oid: MERGE_COMMIT } }); +const commitFix = (issue = ISSUE) => ({ issue, source: { kind: "commit" as const, oid: MERGE_COMMIT } }); +const oneFixBody = supersededBody([prFix()], "main"); + +const noPause = async (): Promise => {}; + +const supersededComment: Comment = { + id: 2, + body: `${SUPERSEDED_MARKER}\n#41750 was fixed by #41767 on main, so this pull request is closed. Reopen it if something was missed.`, + created_at: "2026-09-18T00:00:00Z", + user: { type: "Bot", login: "github-actions[bot]" }, +}; interface World { readonly issue?: ClosedIssue | null; readonly comments?: readonly Comment[]; + readonly pullRequestComments?: Readonly>; + readonly openPullRequests?: readonly (readonly LinkedPullRequest[])[]; + readonly linkedPullRequests?: readonly (readonly LinkedPullRequest[])[]; readonly version?: string; // Which existing rc.1 tags contain the merge commit; a tag absent from the map does not exist readonly tags?: Readonly>; + readonly reachable?: Readonly>; } function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes: string[] } { const writes: string[] = []; const tags = world.tags ?? {}; + const reachable = world.reachable ?? { main: [MERGE_COMMIT] }; + const pages = world.openPullRequests ?? []; const api: GitHubApi = { request: async (method: string, path: string, body?: object): Promise => { if (method === "POST" && path === "/graphql") { - return { data: { repository: { issue: world.issue === undefined ? closedBy(mergedPr) : world.issue } } } as T; + const { query, variables } = body as { query: string; variables: { after: string | null } }; + if (query === OPEN_PULL_REQUESTS_QUERY) { + return { data: { repository: { pullRequests: page(pages, cursorIndex(variables.after)) } } } as T; + } + const issue = world.issue === undefined ? closedBy(mergedPr) : world.issue; + if (issue === null || world.linkedPullRequests === undefined) { + return { data: { repository: { issue } } } as T; + } + const closedByPullRequestsReferences = page(world.linkedPullRequests, cursorIndex(variables.after)); + return { data: { repository: { issue: { ...issue, closedByPullRequestsReferences } } } } as T; } if (method !== "GET") { writes.push(`${method} ${path} ${JSON.stringify(body)}`); return {} as T; } - if (path.startsWith("/repos/BerriAI/litellm/issues/41750/comments")) { - return (world.comments ?? []) as T; + const comments = /^\/repos\/BerriAI\/litellm\/issues\/(\d+)\/comments/.exec(path); + if (comments !== null) { + const number = Number(comments[1]); + return ((number === ISSUE ? world.comments : world.pullRequestComments?.[number]) ?? []) as T; } if (path === `/repos/BerriAI/litellm/contents/pyproject.toml?ref=${MERGE_COMMIT}`) { return { content: btoa(pyproject(world.version ?? "1.103.0")).replace(/(.{60})/g, "$1\n") } as T; @@ -68,6 +146,10 @@ function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes: return (matching[1] in tags ? [{ ref: `refs/tags/${matching[1]}` }] : []) as T; } const compare = /^\/repos\/BerriAI\/litellm\/compare\/(.+)\.\.\.(.+)$/.exec(path); + const branch = compare === null ? undefined : reachable[compare[1] ?? ""]; + if (compare !== null && branch !== undefined) { + return { status: branch.includes(compare[2] ?? "") ? "behind" : "diverged" } as T; + } if (compare !== null && compare[2] === MERGE_COMMIT) { return { status: tags[compare[1]] ? "behind" : "ahead" } as T; } @@ -79,23 +161,28 @@ function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes: describe("closerOf", () => { test("a pull request merged into the default branch is the fix", () => { - expect(closerOf(closedBy(mergedPr), "main")).toEqual({ kind: "pull_request", number: 41767, mergeCommit: MERGE_COMMIT }); + expect(closerOf(closedBy(mergedPr), "BerriAI/litellm", "main")).toEqual({ kind: "pull_request", number: 41767, mergeCommit: MERGE_COMMIT }); }); test("an issue closed by hand, by a commit, or by an unmerged pull request gets no comment", () => { - expect(closerOf(closedBy(null), "main")).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" }); - expect(closerOf(closedBy({ __typename: "Commit", oid: MERGE_COMMIT }), "main").kind).toBe("skip"); - expect(closerOf(closedBy({ ...mergedPr, merged: false }), "main").kind).toBe("skip"); - expect(closerOf(closedBy({ ...mergedPr, mergeCommit: null }), "main").kind).toBe("skip"); + expect(closerOf(closedBy(null), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" }); + expect(closerOf(closedBy(commitCloser), "BerriAI/litellm", "main").kind).toBe("skip"); + expect(closerOf(closedBy({ ...mergedPr, merged: false }), "BerriAI/litellm", "main").kind).toBe("skip"); + expect(closerOf(closedBy({ ...mergedPr, mergeCommit: null }), "BerriAI/litellm", "main").kind).toBe("skip"); }); test("a pull request merged into a release branch is not a fix on main", () => { - const verdict = closerOf(closedBy({ ...mergedPr, baseRefName: "release/1.102.0rc2" }), "main"); + const verdict = closerOf(closedBy({ ...mergedPr, baseRefName: "release/1.102.0rc2" }), "BerriAI/litellm", "main"); expect(verdict).toEqual({ kind: "skip", reason: "#41767 merged into release/1.102.0rc2, not main" }); }); test("an issue reopened after the close event is left alone", () => { - expect(closerOf(closedBy(mergedPr, "OPEN"), "main")).toEqual({ kind: "skip", reason: "the issue is open again" }); + expect(closerOf(closedBy(mergedPr, "OPEN"), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "the issue is open again" }); + }); + + test("a pull request merged in a fork closes the issue on GitHub but is no fix here", () => { + const forkPr = { ...mergedPr, number: 9, repository: { nameWithOwner: "someone/litellm" } }; + expect(closerOf(closedBy(forkPr), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "closed by someone/litellm#9, a pull request in another repository" }); }); }); @@ -173,44 +260,327 @@ describe("fixedBody", () => { }); }); -describe("commentFixedIssue", () => { +describe("fixOf", () => { + test("a merged pull request or a commit is a fix, whatever branch it was merged into", () => { + expect(fixOf(closure(mergedPr), "BerriAI/litellm")).toEqual({ kind: "pull_request", number: 41767, oid: MERGE_COMMIT }); + expect(fixOf(closure({ ...mergedPr, baseRefName: "litellm_internal_staging" }), "BerriAI/litellm")).toEqual({ kind: "pull_request", number: 41767, oid: MERGE_COMMIT }); + expect(fixOf(closure(commitCloser), "BerriAI/litellm")).toEqual({ kind: "commit", oid: MERGE_COMMIT }); + }); + + test("a hand close, a fork's pull request, an unmerged pull request, or a reopened issue is no fix", () => { + expect(fixOf(closure(null), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed by hand" }); + expect(fixOf(closure({ ...mergedPr, repository: { nameWithOwner: "someone/litellm" } }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed from someone/litellm" }); + expect(fixOf(closure({ ...commitCloser, repository: { nameWithOwner: "someone/litellm" } }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed from someone/litellm" }); + expect(fixOf(closure({ ...mergedPr, merged: false }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed by #41767, which is not merged" }); + expect(fixOf(closure({ ...mergedPr, mergeCommit: null }), "BerriAI/litellm").kind).toBe("skip"); + expect(fixOf(closure(mergedPr, "OPEN"), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "is open again" }); + }); +}); + +describe("closeVerdict", () => { + test("an open pull request whose only linked issue was fixed by a merged pull request is a candidate", () => { + expect(closeVerdict(openPr(41760), config)).toEqual({ kind: "candidate", fixes: [prFix()] }); + }); + + test("every linked issue has to be fixed, and each fix is named", () => { + const both = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, commitCloser)) }); + expect(closeVerdict(both, config)).toEqual({ kind: "candidate", fixes: [prFix(), commitFix(41751)] }); + }); + + test("a pull request that still links an open issue keeps its work", () => { + const stillOpen = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, null, "OPEN")) }); + expect(closeVerdict(stillOpen, config)).toEqual({ kind: "skip", reason: "still linked to open #41751" }); + }); + + test("a linked issue closed by hand or by an unmerged pull request is not a fix that supersedes the pull request", () => { + expect(closeVerdict(openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, null)) }), config)).toEqual({ + kind: "skip", + reason: `#${ISSUE} was closed by hand`, + }); + const unmerged = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, { ...mergedPr, merged: false })) }); + expect(closeVerdict(unmerged, config)).toEqual({ kind: "skip", reason: `#${ISSUE} was closed by #41767, which is not merged` }); + }); + + test("a pull request linking an issue in another repository, or more issues than the query reads, is left alone", () => { + const foreign = { ...linkedIssue(41751), repository: { nameWithOwner: "mlflow/mlflow" } }; + expect(closeVerdict(openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), foreign) }), config)).toEqual({ + kind: "skip", + reason: "links mlflow/mlflow#41751", + }); + const truncated = openPr(41760, { closingIssuesReferences: { totalCount: 11, nodes: [linkedIssue(ISSUE)] } }); + expect(closeVerdict(truncated, config)).toEqual({ kind: "skip", reason: "links 11 issues, more than the 10 this workflow reads" }); + }); + + test("a pull request against a release line is a backport and stays open, one against a retired development branch does not", () => { + for (const base of ["release/v1.102.0-rc.2", "stable/v1.83.14", "v_1_83_3_stable_patch"]) { + expect(closeVerdict(openPr(41760, { baseRefName: base }), config)).toEqual({ kind: "skip", reason: `targets the release line ${base}` }); + } + expect(closeVerdict(openPr(41760, { baseRefName: "litellm_internal_staging" }), config).kind).toBe("candidate"); + }); + + test("a pull request in a fork, one that is not open, or one linking no issue is left alone", () => { + expect(closeVerdict(openPr(1, { repository: { nameWithOwner: "someone/litellm" } }), config)).toEqual({ kind: "skip", reason: "lives in someone/litellm" }); + expect(closeVerdict(openPr(41767, { state: "MERGED" }), config)).toEqual({ kind: "skip", reason: "is merged" }); + expect(closeVerdict(openPr(41760, { closingIssuesReferences: links() }), config)).toEqual({ kind: "skip", reason: "links no issue" }); + }); +}); + +describe("supersededBody", () => { + test("names each issue with the pull request or commit that fixed it, invites a reopen, and carries the marker the rerun looks for", () => { + expect(oneFixBody.startsWith(SUPERSEDED_MARKER)).toBe(true); + expect(oneFixBody).toContain("#41750 was fixed by #41767 on main"); + expect(oneFixBody).toContain("Reopen it"); + expect(supersededBody([prFix(), commitFix(41751)], "main")).toContain("#41750 was fixed by #41767 and #41751 by commit 68c4c82ac9 on main"); + }); + + test("stays within the 25-word comment rule for one and two fixes", () => { + for (const fixes of [[prFix()], [commitFix()], [prFix(), commitFix(41751)]]) { + const words = supersededBody(fixes, "main").replace(SUPERSEDED_MARKER, "").trim().split(/\s+/); + expect(words.length).toBeGreaterThanOrEqual(15); + expect(words.length).toBeLessThanOrEqual(25); + } + }); +}); + +describe("handleFixedIssue", () => { test("a real run posts one comment naming the pull request and the release", async () => { const { api, writes } = fakeApi(); - const verdict = await commentFixedIssue(api, config); - expect(verdict).toMatchObject({ kind: "commented", pullRequest: 41767, tag: "v1.103.0-rc.1" }); + const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(comment).toMatchObject({ kind: "commented", pullRequest: 41767, tag: "v1.103.0-rc.1" }); + expect(pullRequests).toEqual([]); expect(writes).toHaveLength(1); expect(writes[0]).toContain("POST /repos/BerriAI/litellm/issues/41750/comments"); expect(writes[0]).toContain("Fixed by #41767. This ships in v1.103.0-rc.1 and up"); }); - test("a dry run renders the comment and writes nothing", async () => { - const { api, writes } = fakeApi(); - const verdict = await commentFixedIssue(api, { ...config, dryRun: true }); - expect(verdict.kind).toBe("commented"); - expect(writes).toEqual([]); + test("every other open pull request linked to the fixed issue is commented on and then closed, a pause before every write", async () => { + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760), openPr(41761)]) }); + let pauses = 0; + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, async () => { + pauses += 1; + }); + expect(pullRequests.map((pullRequest) => pullRequest.kind)).toEqual(["closed", "closed"]); + expect(writes).toEqual([ + expect.stringContaining("POST /repos/BerriAI/litellm/issues/41750/comments"), + `POST /repos/BerriAI/litellm/issues/41760/comments ${JSON.stringify({ body: oneFixBody })}`, + 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}', + expect.stringContaining("POST /repos/BerriAI/litellm/issues/41761/comments"), + 'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}', + ]); + expect(pauses).toBe(4); }); - test("an issue that already carries the comment is not commented twice", async () => { + test("the fixed-in comment and the closing are gated separately: a close dry run still posts the comment and closes nothing", async () => { + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) }); + let pauses = 0; + const outcome = await handleFixedIssue(api, { ...config, closeDryRun: true }, ISSUE, async () => { + pauses += 1; + }); + expect(outcome.comment.kind).toBe("commented"); + expect(outcome.pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]); + expect(writes).toHaveLength(1); + expect(writes[0]).toContain("/issues/41750/comments"); + expect(pauses).toBe(0); + }); + + test("a comment dry run still closes the pull requests for real", async () => { + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) }); + const outcome = await handleFixedIssue(api, { ...config, commentDryRun: true }, ISSUE, noPause); + expect(outcome.comment.kind).toBe("commented"); + expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']); + }); + + test("an issue that already carries the fixed-in comment is not commented twice, and its linked pull requests still get closed", async () => { const existing: Comment = { id: 1, body: `${FIXED_MARKER}\nFixed by #41767. This ships in v1.103.0-rc.1 and up.`, created_at: "2026-09-18T00:00:00Z", user: { type: "Bot", login: "github-actions[bot]" }, }; - const { api, writes } = fakeApi({ comments: [existing] }); - expect(await commentFixedIssue(api, config)).toEqual({ kind: "skip", reason: "already carries a fixed-in comment" }); + const { api, writes } = fakeApi({ comments: [existing], issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) }); + const outcome = await handleFixedIssue(api, config, ISSUE, noPause); + expect(outcome.comment).toEqual({ kind: "skip", reason: "already carries a fixed-in comment" }); + expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']); + }); + + test("a fix merged into the retired development branch or pushed as a commit counts once it is on the default branch", async () => { + const stagingPr = { ...mergedPr, baseRefName: "litellm_internal_staging" }; + const staging = openPr(41760, { baseRefName: "litellm_internal_staging", closingIssuesReferences: links(linkedIssue(ISSUE, stagingPr)) }); + const byCommit = openPr(41761, { closingIssuesReferences: links(linkedIssue(41751, commitCloser)) }); + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [staging, byCommit]) }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([ + { kind: "closed", number: 41760, body: oneFixBody }, + { kind: "closed", number: 41761, body: supersededBody([commitFix(41751)], "main") }, + ]); + expect(writes).toHaveLength(5); + expect(writes[3]).toContain("#41751 was fixed by commit 68c4c82ac9 on main"); + }); + + test("a fix whose commit never reached the default branch supersedes nothing", async () => { + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]), reachable: { main: [] } }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([{ kind: "skip", number: 41760, reason: "#41750 was fixed by #41767, which is not on main" }]); + expect(writes).toHaveLength(1); + expect(writes[0]).toContain("/issues/41750/comments"); + }); + + test("the default branch comes from the config for the containment check and the comment alike", async () => { + const stagingConfig = { ...config, defaultBranch: "litellm_internal_staging" }; + const closer = { ...mergedPr, baseRefName: "litellm_internal_staging" }; + const { api, writes } = fakeApi({ + issue: closedBy(closer, "CLOSED", [openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, closer)) })]), + reachable: { litellm_internal_staging: [MERGE_COMMIT] }, + }); + const { pullRequests } = await handleFixedIssue(api, stagingConfig, ISSUE, noPause); + expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: supersededBody([prFix()], "litellm_internal_staging") }]); + expect(writes[1]).toContain("on litellm_internal_staging, so this pull request is closed"); + }); + + test("the closer sits in the linked list as merged and gets neither a line nor a write", async () => { + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41767, { state: "MERGED" }), openPr(41760)]) }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]); + expect(writes.map((write) => write.split(" ")[1])).toEqual([ + "/repos/BerriAI/litellm/issues/41750/comments", + "/repos/BerriAI/litellm/issues/41760/comments", + "/repos/BerriAI/litellm/pulls/41760", + ]); + }); + + test("a pull request this workflow closed once and its author reopened stays open", async () => { + const reopened = openPr(41760, { reopens: reopenedAt("2026-09-19T00:00:00Z") }); + const { api, writes } = fakeApi({ + issue: closedBy(mergedPr, "CLOSED", [reopened, openPr(41761)]), + pullRequestComments: { 41760: [supersededComment] }, + }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests[0]).toEqual({ kind: "skip", number: 41760, reason: "was closed by this workflow once and reopened" }); + expect(pullRequests[1]?.kind).toBe("closed"); + expect(writes.filter((write) => write.includes("41760"))).toEqual([]); + }); + + test("a pull request whose comment landed but whose close failed is closed on the next run without a second comment", async () => { + const reopenedBeforeTheComment = openPr(41761, { reopens: reopenedAt("2026-09-17T00:00:00Z") }); + const { api, writes } = fakeApi({ + issue: closedBy(mergedPr, "CLOSED", [openPr(41760), reopenedBeforeTheComment]), + pullRequestComments: { 41760: [supersededComment], 41761: [supersededComment] }, + }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([ + { kind: "closed", number: 41760, body: supersededComment.body }, + { kind: "closed", number: 41761, body: supersededComment.body }, + ]); + expect(writes.filter((write) => write.includes("/4176"))).toEqual([ + 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}', + 'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}', + ]); + }); + + test("a superseded marker pasted by anyone but the workflow neither keeps a pull request open nor replaces its comment", async () => { + const forged: Comment = { ...supersededComment, id: 3, user: { type: "User", login: "someone" } }; + const { api, writes } = fakeApi({ + issue: closedBy(mergedPr, "CLOSED", [openPr(41760, { reopens: reopenedAt("2026-09-19T00:00:00Z") })]), + pullRequestComments: { 41760: [forged] }, + }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]); + expect(writes.filter((write) => write.includes("/41760"))).toEqual([ + `POST /repos/BerriAI/litellm/issues/41760/comments ${JSON.stringify({ body: oneFixBody })}`, + 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}', + ]); + }); + + test("every page of linked pull requests is read, not just the first", async () => { + const { api } = fakeApi({ linkedPullRequests: [[openPr(41760)], [openPr(41761)], [openPr(41762)]] }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([ + { kind: "closed", number: 41760, body: oneFixBody }, + { kind: "closed", number: 41761, body: oneFixBody }, + { kind: "closed", number: 41762, body: oneFixBody }, + ]); + }); + + test("a linked pull request from a fork or one still tied to another open issue is reported, not closed", async () => { + const fork = openPr(1, { repository: { nameWithOwner: "someone/litellm" } }); + const busy = openPr(41762, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, null, "OPEN")) }); + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [fork, busy]) }); + const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(pullRequests).toEqual([ + { kind: "skip", number: 1, reason: "lives in someone/litellm" }, + { kind: "skip", number: 41762, reason: "still linked to open #41751" }, + ]); + expect(writes).toHaveLength(1); + }); + + test("a hand-closed issue gets no comment and leaves its linked pull requests open with the reason on each", async () => { + const byHand = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, null)) }); + const { api, writes } = fakeApi({ issue: closedBy(null, "CLOSED", [byHand]) }); + const outcome = await handleFixedIssue(api, config, ISSUE, noPause); + expect(outcome.comment).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" }); + expect(outcome.pullRequests).toEqual([{ kind: "skip", number: 41760, reason: `#${ISSUE} was closed by hand` }]); expect(writes).toEqual([]); }); - test("a hand-closed issue never reaches the release lookup or the API writes", async () => { - const { api, writes } = fakeApi({ issue: closedBy(null) }); - expect((await commentFixedIssue(api, config)).kind).toBe("skip"); + test("an issue closed by a commit on the default branch gets no comment but still closes its linked pull requests", async () => { + const byCommit = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, commitCloser)) }); + const { api, writes } = fakeApi({ issue: closedBy(commitCloser, "CLOSED", [byCommit]) }); + const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(comment).toEqual({ kind: "skip", reason: "closed by commit 68c4c82ac9, not by a pull request" }); + expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: supersededBody([commitFix()], "main") }]); + expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']); + expect(writes[0]).toContain("#41750 was fixed by commit 68c4c82ac9 on main"); + }); + + test("an issue closed from the retired development branch gets no comment but still closes its linked pull requests once the fix is on the default branch", async () => { + const stagingPr = { ...mergedPr, baseRefName: "litellm_internal_staging" }; + const staging = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, stagingPr)) }); + const { api, writes } = fakeApi({ issue: closedBy(stagingPr, "CLOSED", [staging]) }); + const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause); + expect(comment).toEqual({ kind: "skip", reason: "#41767 merged into litellm_internal_staging, not main" }); + expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]); + expect(writes.map((write) => write.split(" ")[1])).toEqual(["/repos/BerriAI/litellm/issues/41760/comments", "/repos/BerriAI/litellm/pulls/41760"]); + }); + + test("an issue that is open again gets no comment and its linked pull requests are neither read nor touched", async () => { + const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "OPEN", [openPr(41760)]) }); + const outcome = await handleFixedIssue(api, config, ISSUE, noPause); + expect(outcome).toEqual({ comment: { kind: "skip", reason: "the issue is open again" }, pullRequests: [] }); expect(writes).toEqual([]); }); test("a number that is not an issue in the repository is a skip", async () => { const { api, writes } = fakeApi({ issue: null }); - expect(await commentFixedIssue(api, config)).toEqual({ kind: "skip", reason: "not an issue in this repository" }); + const outcome = await handleFixedIssue(api, config, ISSUE, noPause); + expect(outcome.comment).toEqual({ kind: "skip", reason: "not an issue in this repository" }); + expect(writes).toEqual([]); + }); +}); + +describe("sweep", () => { + test("walks every page of open pull requests and closes the ones whose linked issues were all fixed", async () => { + const unlinked = openPr(41700, { closingIssuesReferences: links() }); + const busy = openPr(41701, { closingIssuesReferences: links(linkedIssue(41751, null, "OPEN")) }); + const { api, writes } = fakeApi({ openPullRequests: [[unlinked, openPr(41760)], [busy, openPr(41761)]] }); + const outcome = await sweep(api, { ...config, commentDryRun: true }, noPause); + expect(outcome.considered).toBe(4); + expect(outcome.pullRequests).toEqual([ + { kind: "closed", number: 41760, body: oneFixBody }, + { kind: "skip", number: 41701, reason: "still linked to open #41751" }, + { kind: "closed", number: 41761, body: oneFixBody }, + ]); + expect(writes).toEqual([ + expect.stringContaining("POST /repos/BerriAI/litellm/issues/41760/comments"), + 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}', + expect.stringContaining("POST /repos/BerriAI/litellm/issues/41761/comments"), + 'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}', + ]); + }); + + test("a sweep dry run lists what it would close and writes nothing", async () => { + const { api, writes } = fakeApi({ openPullRequests: [[openPr(41760)]] }); + const outcome = await sweep(api, { ...config, closeDryRun: true }, noPause); + expect(outcome.pullRequests.map((pullRequest) => pullRequest.kind)).toEqual(["closed"]); expect(writes).toEqual([]); }); }); @@ -218,10 +588,24 @@ describe("commentFixedIssue", () => { describe("readConfig", () => { const env = { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm", ISSUE_NUMBER: "41750", DEFAULT_BRANCH: "main" }; - test("reads the four inputs and treats anything but the literal true as a real run", () => { - expect(readConfig(env)).toEqual({ token: "t", repo: "BerriAI/litellm", issueNumber: 41750, defaultBranch: "main", dryRun: false }); - expect(readConfig({ ...env, DRY_RUN: "true" }).dryRun).toBe(true); - expect(readConfig({ ...env, DRY_RUN: "false" }).dryRun).toBe(false); + test("reads the inputs and treats anything but the literal true as a real run for each gate", () => { + expect(readConfig(env)).toEqual({ + token: "t", + repo: "BerriAI/litellm", + defaultBranch: "main", + commentDryRun: false, + closeDryRun: false, + run: { kind: "issue", number: 41750 }, + }); + expect(readConfig({ ...env, DRY_RUN: "true" })).toMatchObject({ commentDryRun: true, closeDryRun: false }); + expect(readConfig({ ...env, CLOSE_PRS_DRY_RUN: "true" })).toMatchObject({ commentDryRun: false, closeDryRun: true }); + expect(readConfig({ ...env, DRY_RUN: "false", CLOSE_PRS_DRY_RUN: "false" })).toMatchObject({ commentDryRun: false, closeDryRun: false }); + }); + + test("a sweep needs no issue number and anything but the literal true is an issue run", () => { + expect(readConfig({ ...env, ISSUE_NUMBER: undefined, SWEEP: "true" }).run).toEqual({ kind: "sweep" }); + expect(readConfig({ ...env, SWEEP: "false" }).run).toEqual({ kind: "issue", number: 41750 }); + expect(() => readConfig({ ...env, ISSUE_NUMBER: "", SWEEP: "false" })).toThrow("dispatch with an issue_number or with sweep ticked"); }); test("refuses a missing token, repo, branch or a bad issue number", () => { @@ -232,3 +616,31 @@ describe("readConfig", () => { expect(() => readConfig({ ...env, ISSUE_NUMBER: "abc" })).toThrow("ISSUE_NUMBER"); }); }); + +describe("step summary", () => { + const closed = { kind: "closed" as const, number: 41760, body: oneFixBody }; + const left = { kind: "skip" as const, number: 41762, reason: "still linked to open #41751" }; + const commented = { kind: "commented" as const, pullRequest: 41767, tag: "v1.103.0-rc.1", body: fixedBody(41767, { tag: "v1.103.0-rc.1", shipped: false }) }; + + test("an issue run names the comment, each close, and each pull request left open", () => { + const summary = describeIssue(config, ISSUE, { comment: commented, pullRequests: [closed, left] }); + expect(summary).toContain("#41750: commented, fixed by #41767 in v1.103.0-rc.1"); + expect(summary).toContain("#41760: closed with: #41750 was fixed by #41767 on main"); + expect(summary).toContain("#41762: left open, still linked to open #41751"); + expect(summary).not.toContain("DRY RUN"); + }); + + test("a close dry run names the repo variable that turns closing on", () => { + const summary = describeIssue({ ...config, closeDryRun: true }, ISSUE, { comment: commented, pullRequests: [closed] }); + expect(summary).toContain("ISSUE_FIXED_CLOSE_PRS_ENABLED"); + expect(summary).toContain("#41760: DRY RUN, would close with: #41750 was fixed by #41767 on main"); + }); + + test("a sweep summary counts what it saw and lists only the closes", () => { + const summary = describeSweep(config, { considered: 3522, pullRequests: [closed, left] }); + expect(summary).toContain("Swept 3522 open pull requests, 2 linked to an issue, 1 closed"); + expect(summary).toContain("#41760: closed with:"); + expect(summary).not.toContain("#41762"); + expect(describeSweep({ ...config, closeDryRun: true }, { considered: 3522, pullRequests: [closed] })).toContain("1 would be closed, DRY RUN, set the ISSUE_FIXED_CLOSE_PRS_ENABLED"); + }); +}); diff --git a/scripts/comment-fixed-issue.ts b/scripts/comment-fixed-issue.ts index 480b5e90249..450d530d90c 100644 --- a/scripts/comment-fixed-issue.ts +++ b/scripts/comment-fixed-issue.ts @@ -6,35 +6,66 @@ declare const process: { readonly env: Readonly base.startsWith("release/") || base.includes("stable"); + +const CLOSURE_FRAGMENT = `fragment Closure on Issue { + state + timelineItems(last: 1, itemTypes: [CLOSED_EVENT]) { + nodes { + ... on ClosedEvent { + closer { + __typename + ... on PullRequest { number merged baseRefName repository { nameWithOwner } mergeCommit { oid } } + ... on Commit { oid repository { nameWithOwner } } } } } } }`; +const LINKED_PULL_REQUEST_FRAGMENT = `fragment Linked on PullRequest { + number + state + baseRefName + repository { nameWithOwner } + closingIssuesReferences(first: ${MAX_LINKED_ISSUES}) { totalCount nodes { number repository { nameWithOwner } ...Closure } } + reopens: timelineItems(last: 1, itemTypes: [REOPENED_EVENT]) { nodes { ... on ReopenedEvent { createdAt } } } +}`; + +export const CLOSER_QUERY = `query($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + issue(number: $number) { + ...Closure + closedByPullRequestsReferences(first: ${MAX_LINKED_PULL_REQUESTS}, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { ...Linked } + } + } + } +} +${CLOSURE_FRAGMENT} +${LINKED_PULL_REQUEST_FRAGMENT}`; + +export const OPEN_PULL_REQUESTS_QUERY = `query($owner: String!, $name: String!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, first: ${SWEEP_PAGE_SIZE}, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { ...Linked } + } + } +} +${CLOSURE_FRAGMENT} +${LINKED_PULL_REQUEST_FRAGMENT}`; + const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason }); -export function closerOf(issue: ClosedIssue, defaultBranch: string): Closer { +export function closerOf(issue: IssueClosure, repo: string, defaultBranch: string): Closer { if (issue.state !== "CLOSED") { - return skip("the issue is open again"); + return skip(OPEN_AGAIN); } const closer = issue.timelineItems.nodes[0]?.closer ?? null; if (closer === null) { @@ -94,6 +192,9 @@ export function closerOf(issue: ClosedIssue, defaultBranch: string): Closer { if (closer.__typename === "Commit") { return skip(`closed by commit ${closer.oid.slice(0, 10)}, not by a pull request`); } + if (closer.repository.nameWithOwner !== repo) { + return skip(`closed by ${closer.repository.nameWithOwner}#${closer.number}, a pull request in another repository`); + } if (!closer.merged || closer.mergeCommit === null) { return skip(`closed by #${closer.number}, which is not merged`); } @@ -121,8 +222,8 @@ async function tagExists(api: GitHubApi, repo: string, tag: string): Promise ref.ref === `refs/tags/${tag}`); } -async function tagContains(api: GitHubApi, repo: string, tag: string, sha: string): Promise { - const comparison = await api.request("GET", `/repos/${repo}/compare/${tag}...${sha}`); +async function refContains(api: GitHubApi, repo: string, ref: string, sha: string): Promise { + const comparison = await api.request("GET", `/repos/${repo}/compare/${ref}...${sha}`); return comparison.status === "behind" || comparison.status === "identical"; } @@ -139,7 +240,7 @@ async function firstReleaseWith( if (!(await tagExists(api, repo, tag))) { return { kind: "release", tag, shipped: false }; } - if (await tagContains(api, repo, tag, sha)) { + if (await refContains(api, repo, tag, sha)) { return { kind: "release", tag, shipped: true }; } if (bumpsLeft === 0) { @@ -164,21 +265,13 @@ export function fixedBody(pullRequest: number, release: { readonly tag: string; return `${FIXED_MARKER}\nFixed by #${pullRequest}. ${availability}`; } -export async function commentFixedIssue(api: GitHubApi, config: FixedConfig): Promise { - const [owner, name] = config.repo.split("/"); - const response = await api.request("POST", "/graphql", { - query: CLOSER_QUERY, - variables: { owner, name, number: config.issueNumber }, - }); - const issue = response.data?.repository?.issue ?? null; - if (issue === null) { - return skip("not an issue in this repository"); - } - const closer = closerOf(issue, config.defaultBranch); - if (closer.kind === "skip") { - return closer; - } - const issuePath = `/repos/${config.repo}/issues/${config.issueNumber}`; +export async function commentFixedIssue( + api: GitHubApi, + config: FixedConfig, + issueNumber: number, + closer: { readonly number: number; readonly mergeCommit: string }, +): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; const comments = await listAll(api, `${issuePath}/comments`); if (comments.some((comment) => comment.body.includes(FIXED_MARKER))) { return skip("already carries a fixed-in comment"); @@ -188,37 +281,259 @@ export async function commentFixedIssue(api: GitHubApi, config: FixedConfig): Pr return release; } const body = fixedBody(closer.number, release); - if (!config.dryRun) { + if (!config.commentDryRun) { await api.request("POST", `${issuePath}/comments`, { body }); } return { kind: "commented", pullRequest: closer.number, tag: release.tag, body }; } -export function readConfig(env: Readonly>): FixedConfig & { readonly token: string } { +export function fixOf(issue: IssueClosure, repo: string): FixVerdict { + if (issue.state !== "CLOSED") { + return skip("is open again"); + } + const closer = issue.timelineItems.nodes[0]?.closer ?? null; + if (closer === null) { + return skip("was closed by hand"); + } + if (closer.repository.nameWithOwner !== repo) { + return skip(`was closed from ${closer.repository.nameWithOwner}`); + } + if (closer.__typename === "Commit") { + return { kind: "commit", oid: closer.oid }; + } + if (!closer.merged || closer.mergeCommit === null) { + return skip(`was closed by #${closer.number}, which is not merged`); + } + return { kind: "pull_request", number: closer.number, oid: closer.mergeCommit.oid }; +} + +export function closeVerdict(pullRequest: LinkedPullRequest, config: FixedConfig): CloseVerdict { + if (pullRequest.repository.nameWithOwner !== config.repo) { + return skip(`lives in ${pullRequest.repository.nameWithOwner}`); + } + if (pullRequest.state !== "OPEN") { + return skip(`is ${pullRequest.state.toLowerCase()}`); + } + if (isReleaseLine(pullRequest.baseRefName)) { + return skip(`targets the release line ${pullRequest.baseRefName}`); + } + const { totalCount, nodes: linked } = pullRequest.closingIssuesReferences; + if (linked.length === 0) { + return skip("links no issue"); + } + if (totalCount > linked.length) { + return skip(`links ${totalCount} issues, more than the ${MAX_LINKED_ISSUES} this workflow reads`); + } + const foreign = linked.find((issue) => issue.repository.nameWithOwner !== config.repo); + if (foreign !== undefined) { + return skip(`links ${foreign.repository.nameWithOwner}#${foreign.number}`); + } + const stillOpen = linked.find((issue) => issue.state === "OPEN"); + if (stillOpen !== undefined) { + return skip(`still linked to open #${stillOpen.number}`); + } + const verdicts = linked.map((issue) => ({ issue: issue.number, fix: fixOf(issue, config.repo) })); + for (const { issue, fix } of verdicts) { + if (fix.kind === "skip") { + return skip(`#${issue} ${fix.reason}`); + } + } + return { + kind: "candidate", + fixes: verdicts.flatMap(({ issue, fix }) => (fix.kind === "skip" ? [] : [{ issue, source: fix }])), + }; +} + +function describeSource(source: FixSource): string { + return source.kind === "pull_request" ? `#${source.number}` : `commit ${source.oid.slice(0, 10)}`; +} + +async function fixOffDefaultBranch(api: GitHubApi, config: FixedConfig, fixes: readonly Fix[]): Promise { + const onBranch = await Promise.all(fixes.map((fix) => refContains(api, config.repo, config.defaultBranch, fix.source.oid))); + return fixes.find((_, index) => !onBranch[index]); +} + +export function supersededBody(fixes: readonly Fix[], defaultBranch: string): string { + const pairs = fixes + .map((fix, index) => `#${fix.issue} ${index === 0 ? "was fixed by" : "by"} ${describeSource(fix.source)}`) + .join(" and "); + return `${SUPERSEDED_MARKER}\n${pairs} on ${defaultBranch}, so this pull request is closed. Reopen it if something was missed.`; +} + +function reopenedAfter(pullRequest: LinkedPullRequest, comment: Comment): boolean { + const reopen = pullRequest.reopens.nodes[0]; + return reopen !== undefined && Date.parse(reopen.createdAt) > Date.parse(comment.created_at); +} + +async function closePullRequest( + api: GitHubApi, + config: FixedConfig, + pullRequest: LinkedPullRequest, + pause: () => Promise, +): Promise { + const verdict = closeVerdict(pullRequest, config); + if (verdict.kind === "skip") { + return { kind: "skip", number: pullRequest.number, reason: verdict.reason }; + } + const offBranch = await fixOffDefaultBranch(api, config, verdict.fixes); + if (offBranch !== undefined) { + const fix = describeSource(offBranch.source); + return { kind: "skip", number: pullRequest.number, reason: `#${offBranch.issue} was fixed by ${fix}, which is not on ${config.defaultBranch}` }; + } + const issuePath = `/repos/${config.repo}/issues/${pullRequest.number}`; + const comments = await listAll(api, `${issuePath}/comments`); + const marker = comments.find((comment) => comment.user.login === WORKFLOW_LOGIN && comment.body.includes(SUPERSEDED_MARKER)); + if (marker !== undefined && reopenedAfter(pullRequest, marker)) { + return { kind: "skip", number: pullRequest.number, reason: "was closed by this workflow once and reopened" }; + } + const body = marker?.body ?? supersededBody(verdict.fixes, config.defaultBranch); + if (config.closeDryRun) { + return { kind: "closed", number: pullRequest.number, body }; + } + if (marker === undefined) { + await pause(); + await api.request("POST", `${issuePath}/comments`, { body }); + } + await pause(); + await api.request("PATCH", `/repos/${config.repo}/pulls/${pullRequest.number}`, { state: "closed" }); + return { kind: "closed", number: pullRequest.number, body }; +} + +export function closePullRequests( + api: GitHubApi, + config: FixedConfig, + candidates: readonly LinkedPullRequest[], + pause: () => Promise, +): Promise { + return candidates.reduce>( + async (previous, candidate) => [...(await previous), await closePullRequest(api, config, candidate, pause)], + Promise.resolve([]), + ); +} + +type NextPage = (after: string | null) => Promise; + +async function collectPages(page: PullRequestsPage, nextPage: NextPage): Promise { + if (!page.pageInfo.hasNextPage) { + return page.nodes; + } + return [...page.nodes, ...(await collectPages(await nextPage(page.pageInfo.endCursor), nextPage))]; +} + +async function closedIssue(api: GitHubApi, config: FixedConfig, issueNumber: number, after: string | null): Promise { + const [owner, name] = config.repo.split("/"); + const response = await api.request("POST", "/graphql", { + query: CLOSER_QUERY, + variables: { owner, name, number: issueNumber, after }, + }); + return response.data?.repository?.issue ?? null; +} + +export async function handleFixedIssue( + api: GitHubApi, + config: FixedConfig, + issueNumber: number, + pause: () => Promise, +): Promise { + const issue = await closedIssue(api, config, issueNumber, null); + if (issue === null) { + return { comment: skip("not an issue in this repository"), pullRequests: [] }; + } + if (issue.state !== "CLOSED") { + return { comment: skip(OPEN_AGAIN), pullRequests: [] }; + } + const closer = closerOf(issue, config.repo, config.defaultBranch); + const comment = closer.kind === "skip" ? closer : await commentFixedIssue(api, config, issueNumber, closer); + const nextPage: NextPage = async (after) => { + const more = await closedIssue(api, config, issueNumber, after); + if (more === null) { + throw new Error(`#${issueNumber} came back without data while reading its linked pull requests after cursor ${after}`); + } + return more.closedByPullRequestsReferences; + }; + const linked = await collectPages(issue.closedByPullRequestsReferences, nextPage); + const open = linked.filter((pullRequest) => pullRequest.state === "OPEN"); + const pullRequests = await closePullRequests(api, config, open, pause); + return { comment, pullRequests }; +} + +async function openPullRequests(api: GitHubApi, config: FixedConfig): Promise { + const [owner, name] = config.repo.split("/"); + const nextPage: NextPage = async (after) => { + const response = await api.request("POST", "/graphql", { + query: OPEN_PULL_REQUESTS_QUERY, + variables: { owner, name, after }, + }); + const page = response.data?.repository?.pullRequests; + if (page === undefined) { + throw new Error(`open pull requests after cursor ${after} came back without data: ${JSON.stringify(response)}`); + } + return page; + }; + return collectPages(await nextPage(null), nextPage); +} + +export async function sweep(api: GitHubApi, config: FixedConfig, pause: () => Promise): Promise { + const open = await openPullRequests(api, config); + const linked = open.filter((pullRequest) => pullRequest.closingIssuesReferences.nodes.length > 0); + return { considered: open.length, pullRequests: await closePullRequests(api, config, linked, pause) }; +} + +export function readConfig( + env: Readonly>, +): FixedConfig & { readonly token: string; readonly run: Run } { const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const defaultBranch = env.DEFAULT_BRANCH; if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo) || !defaultBranch) { throw new Error("GITHUB_TOKEN, GITHUB_REPOSITORY (owner/repo) and DEFAULT_BRANCH are required"); } + const config = { token, repo, defaultBranch, commentDryRun: env.DRY_RUN === "true", closeDryRun: env.CLOSE_PRS_DRY_RUN === "true" }; + if (env.SWEEP === "true") { + return { ...config, run: { kind: "sweep" } }; + } const issueNumber = Number(env.ISSUE_NUMBER); if (!Number.isInteger(issueNumber) || issueNumber <= 0) { - throw new Error(`ISSUE_NUMBER must be a positive integer, got "${env.ISSUE_NUMBER}"`); + throw new Error(`ISSUE_NUMBER must be a positive integer, got "${env.ISSUE_NUMBER}": dispatch with an issue_number or with sweep ticked`); } - return { token, repo, issueNumber, defaultBranch, dryRun: env.DRY_RUN === "true" }; + return { ...config, run: { kind: "issue", number: issueNumber } }; } -function describe(config: FixedConfig, verdict: FixedVerdict): string { - if (verdict.kind === "skip") { - return `#${config.issueNumber}: skipped, ${verdict.reason}`; +const CLOSE_DRY_RUN_HINT = "set the ISSUE_FIXED_CLOSE_PRS_ENABLED repo variable to true to close pull requests"; + +function describeClose(config: FixedConfig, outcome: CloseOutcome): string { + if (outcome.kind === "skip") { + return `#${outcome.number}: left open, ${outcome.reason}`; } - if (config.dryRun) { - return `#${config.issueNumber}: DRY RUN, set the ISSUE_FIXED_COMMENT_ENABLED repo variable to true to post this:\n\n${verdict.body}`; - } - return `#${config.issueNumber}: commented, fixed by #${verdict.pullRequest} in ${verdict.tag}`; + const text = outcome.body.replace(`${SUPERSEDED_MARKER}\n`, ""); + return config.closeDryRun ? `#${outcome.number}: DRY RUN, would close with: ${text}` : `#${outcome.number}: closed with: ${text}`; +} + +export function describeIssue(config: FixedConfig, issueNumber: number, outcome: IssueOutcome): string { + const comment = + outcome.comment.kind === "skip" + ? `#${issueNumber}: skipped, ${outcome.comment.reason}` + : config.commentDryRun + ? `#${issueNumber}: DRY RUN, set the ISSUE_FIXED_COMMENT_ENABLED repo variable to true to post this:\n\n${outcome.comment.body}` + : `#${issueNumber}: commented, fixed by #${outcome.comment.pullRequest} in ${outcome.comment.tag}`; + const hint = config.closeDryRun && outcome.pullRequests.some((pullRequest) => pullRequest.kind === "closed") ? [`Closing is a DRY RUN, ${CLOSE_DRY_RUN_HINT}`] : []; + return [comment, ...hint, ...outcome.pullRequests.map((pullRequest) => describeClose(config, pullRequest))].join("\n"); +} + +export function describeSweep(config: FixedConfig, outcome: SweepOutcome): string { + const closed = outcome.pullRequests.filter((pullRequest) => pullRequest.kind === "closed"); + const verb = config.closeDryRun ? `would be closed, DRY RUN, ${CLOSE_DRY_RUN_HINT}` : "closed"; + const header = `Swept ${outcome.considered} open pull requests, ${outcome.pullRequests.length} linked to an issue, ${closed.length} ${verb}`; + return [header, ...closed.map((pullRequest) => describeClose(config, pullRequest))].join("\n"); } if (import.meta.main) { - const { token, ...config } = readConfig(process.env); - console.log(describe(config, await commentFixedIssue(githubApi(token), config))); + const { token, run, ...config } = readConfig(process.env); + const api = githubApi(token); + const pause = (): Promise => new Promise((resolve) => setTimeout(resolve, CLOSE_PAUSE_MS)); + console.log( + run.kind === "sweep" + ? describeSweep(config, await sweep(api, config, pause)) + : describeIssue(config, run.number, await handleFixedIssue(api, config, run.number, pause)), + ); } diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index b77348b5d73..dc1f8592612 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -69,6 +69,9 @@ IGNORE_FUNCTIONS = [ "_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible). "_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible). "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. + "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). ] diff --git a/tests/e2e/claude_code/cron_vm/Dockerfile b/tests/e2e/claude_code/cron_vm/Dockerfile new file mode 100644 index 00000000000..623d6b1840a --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/Dockerfile @@ -0,0 +1,41 @@ +FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 + +ARG GH_VERSION=2.101.0 +ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8 +ARG UV_VERSION=0.10.9 +ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 +ARG CLAUDE_CODE_VERSION=2.1.228 +ARG CLAUDE_CODE_SHA256=d535985e6941a3eb00179ccd7f52ceb0c6623a0305a518ebc4e6514f84a94c99 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \ + && echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \ + && rm /tmp/gh.tar.gz + +RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ + && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \ + && rm /tmp/uv.tar.gz + +RUN curl -fsSLo /tmp/claude "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/linux-x64/claude" \ + && echo "${CLAUDE_CODE_SHA256} /tmp/claude" | sha256sum -c - \ + && install -m 0755 /tmp/claude /usr/local/bin/claude \ + && rm /tmp/claude + +RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator + +ENV HOME=/home/populator \ + LITELLM_REPO=/opt/litellm \ + DISABLE_AUTOUPDATER=1 + +COPY --chown=populator:populator . /opt/litellm/tests/e2e/ + +USER populator +WORKDIR /home/populator +CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"] diff --git a/tests/e2e/claude_code/cron_vm/README.md b/tests/e2e/claude_code/cron_vm/README.md index f120c30605b..ed2bf4ab436 100644 --- a/tests/e2e/claude_code/cron_vm/README.md +++ b/tests/e2e/claude_code/cron_vm/README.md @@ -1,66 +1,59 @@ -# Cron VM setup for the Claude Code compatibility-matrix populator +# Render cron job for the Claude Code compatibility-matrix populator -The populator runs daily on a dedicated GCP VM -(`litellm-compatibility-matrix-populator`) rather than as a GitHub -Action. Trade-offs: +The populator runs daily as the Render cron job `litellm-compat-matrix` +(Docker runtime, built from the `Dockerfile` in this directory) rather +than as a GitHub Action or on a dedicated VM. Trade-offs: -- ✅ Real VM means we can `gh auth login` against an account that's - already a collaborator on `BerriAI/litellm-docs`, instead of - provisioning a GitHub App with `pull-requests: write`. -- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`) - is reused across runs, so each daily run does a fast `git checkout` + - incremental `uv sync` rather than a fresh clone + cold sync. -- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`. -- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers - from short outages, but a multi-day outage means the matrix goes - stale until the VM is back. -- ⚠️ Provider credentials live on the VM filesystem - (`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat - the VM as an environment with comparable blast radius to a CI runner. - -This directory used to live at `tests/claude_code/cron_vm/` (paired with -the standalone `tests/claude_code/` suite); it now runs the maintained -`tests/e2e/claude_code/` suite instead. The pytest env interface changed -accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY` -(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure -column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously -`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and -`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`. +- ✅ No machine to keep on or patch. Render builds the image from this + directory on every push to `main` that touches `tests/e2e/**` and + runs it on the schedule. +- ✅ Credentials live in Render env vars and secret files, scoped to + this one service, instead of on a VM filesystem. +- ✅ The publish token still uses the `mateo-berri` account, which is a + collaborator on `BerriAI/litellm-docs`, so no GitHub App with + `pull-requests: write` has to be provisioned. +- ⚠️ The disk is ephemeral, so every run starts from a fresh (blobless) + clone of litellm plus a cold `uv sync`. That adds a few minutes on + top of the ~10 minute test run; the job's 12 hour ceiling is nowhere + near. +- ⚠️ The Claude Code CLI version under test is pinned in the + `Dockerfile` (`CLAUDE_CODE_VERSION` + its checksum). Bumping it is a + PR, see the gotchas below. ## Layout | File | Purpose | | --- | --- | -| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | +| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh`, `uv`, and the Claude Code CLI, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). | +| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | | `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | | `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. | -| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. | -| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. | -| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. | +| `litellm-compat-matrix.env.example` | The service's env vars, one per line, with what each is for. | ## What `run_daily.sh` does 1. **Resolves the latest LiteLLM final release tag** (newest bare `vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the GitHub Releases API (`curl | jq`). -2. **Reads the local Claude Code CLI version** via `claude --version`. - The cron does not auto-upgrade the CLI — operators do that - out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`. -3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`: - `git fetch --tags --force`, `git reset --hard`, - `git clean -fdx -e .venv -e .uv-bin`, `git checkout --force `. - The `.venv` is preserved across runs so `uv sync --frozen` is - incremental. Then **shims the test suite**: `tests/e2e/` in the - worktree is rebuilt from the dev checkout — the `claude_code/` suite - plus the five shared transport helpers it imports (`proxy_client.py`, - `e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the - cron always runs *today's* tests against the latest stable proxy. The - tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`, - whose imports the stable venv doesn't install) is deliberately not - used. +2. **Reads the Claude Code CLI version** via `claude --version`. That + is whatever the `Dockerfile` pins; the job never upgrades it on its + own. +3. **Clones the worktree** at `~/litellm-cron-worktree/` (a + `--filter=blob:none` clone, so only the checked-out tag's blobs are + fetched), `git checkout --force `, then `uv sync --frozen + --no-install-project` against a uv-managed CPython 3.12 followed by + `uv pip install --no-build litellm==`, so the proxy under + test is the published PyPI wheel (what users install) rather than a + source build: the tag builds a Rust extension through maturin, and + the image ships no C or Rust toolchain. Then **shims the test suite**: + `tests/e2e/` in the worktree is replaced by the image's copy of this + whole tree, so the cron always runs *today's* tests against the + latest stable proxy, and pytest runs with `--confcutdir` pointed at + `claude_code/` so the tree's EKS-harness `conftest.py` (whose imports + the stable venv doesn't install) is never loaded. The tag's own + `tests/e2e/` is deliberately not used. 4. **Boots the proxy** as a `setsid` background process on port `4100` - (so it can't collide with a developer's `:4000`), then polls - `/health/liveliness` until it's up. + bound to loopback, then polls `/health/liveliness` until it's up. 5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL` pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest hook writes the per-test results artifact. Test failures become @@ -74,8 +67,9 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously `mateo-berri` token has write access, so this is a same-repo branch, not a fork), `gh pr create`. A re-run on the same day fast-forwards the existing branch and `gh pr create` no-ops ("a pull request for - branch ... already exists" is treated as success). These PRs are no - longer gated on a second human review. + branch ... already exists" is treated as success). If the JSON is + byte-identical to what `main` already publishes, the push is skipped + entirely. These PRs are not gated on a second human review. 8. **Gates auto-merge on a regression check**: before enabling auto-merge, `check_regressions.py` diffs the new matrix against the one currently on `main`. Auto-merge (`gh pr merge --auto --squash`) @@ -89,107 +83,155 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously human reviews before it lands on the public table. The check fails *closed*: if it errors, auto-merge is withheld. 9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every - other open `compat-matrix/*` PR on the docs repo is closed (and its - bot-owned branch deleted), so at most one compat-matrix PR is ever - open — the newest. + other open `compat-matrix/*` PR that the publishing account opened + from a branch on the docs repo itself is closed (and its bot-owned + branch deleted), so at most one compat-matrix PR is ever open — the + newest. A contributor's PR under that prefix is never touched. -## One-time VM setup +## The Render service -Run as `mateo` on the cron VM: +Everything below is what the live service is set to; recreate it with +the same values if it ever has to be rebuilt. + +| Setting | Value | +| --- | --- | +| Workspace | Litellm (the one that already builds the other litellm services) | +| Type | Cron job, Docker runtime | +| Repo / branch | `BerriAI/litellm` @ `main` | +| Dockerfile path | `tests/e2e/claude_code/cron_vm/Dockerfile` | +| Docker build context | `tests/e2e` (the repo root `.dockerignore` excludes `tests`, so the context has to start below it) | +| Build filter | included paths `tests/e2e/**` | +| Schedule | `0 6 * * *` (06:00 UTC daily) | +| Plan / region | `4c-16g` (4 CPU, 16 GB, what the dashboard calls Pro Max; the suite fans out to ~75 concurrent CLI calls) / Oregon | +| Env vars | every key in `litellm-compat-matrix.env.example` | +| Secret files | `github-token` (the publish PAT, one line) and `vertex-service-account.json` (the Vertex service-account key) | + +Render mounts secret files at `/etc/secrets/`, which is where +`CREDENTIALS_DIRECTORY` and `GOOGLE_APPLICATION_CREDENTIALS` in the env +example point. Render also passes env vars to `docker build` as build +args, which is why the `Dockerfile` declares no `ARG` that could ever +be given a secret's name. + +Creating it through the API looks like this (fill `envVars` and +`secretFiles` from the env example and the two secrets; `ownerId` is +the workspace id from `GET /v1/owners`): ```bash -# 1. Toolchain -sudo apt-get update -sudo apt-get install -y git nodejs npm jq curl -curl -LsSf https://astral.sh/uv/install.sh | sh -sudo apt-get install -y gh # or follow https://cli.github.com/ - -# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this -# line out-of-band when you want a fresh CLI to be tested) -sudo npm install -g @anthropic-ai/claude-code@latest - -# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the -# source of the .service / .timer files. The cron itself runs out -# of the separate worktree at ~/litellm-cron-worktree/. -mkdir -p ~/litellm -git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm -git -C ~/litellm/litellm checkout litellm_internal_staging - -# 4. gh auth — must be a collaborator on BerriAI/litellm-docs. -gh auth login # follow prompts; pick HTTPS + token paste flow - -# 5. Provider credentials + the publish token. -sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \ - /etc/litellm-compat-matrix.env -sudoedit /etc/litellm-compat-matrix.env # fill in real values -sudo chmod 0600 /etc/litellm-compat-matrix.env -# The mateo-berri PAT lives in its own file, mapped into the service via -# systemd LoadCredential so it stays out of the test processes' env -# (see the env.example comment for why). -sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token -sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT - -# 6. systemd units. -sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ -sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now litellm-compat-matrix.timer +curl -fsS https://api.render.com/v1/services \ + -H "Authorization: Bearer ${RENDER_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{ + "type": "cron_job", + "name": "litellm-compat-matrix", + "ownerId": "", + "repo": "https://github.com/BerriAI/litellm", + "branch": "main", + "autoDeploy": "yes", + "buildFilter": {"paths": ["tests/e2e/**"], "ignoredPaths": []}, + "envVars": [{"key": "ANTHROPIC_API_KEY", "value": "..."}], + "secretFiles": [{"name": "github-token", "content": "..."}, + {"name": "vertex-service-account.json", "content": "..."}], + "serviceDetails": { + "runtime": "docker", + "schedule": "0 6 * * *", + "plan": "4c-16g", + "region": "oregon", + "envSpecificDetails": { + "dockerfilePath": "tests/e2e/claude_code/cron_vm/Dockerfile", + "dockerContext": "tests/e2e" + } + } + }' ``` ## Operating it ```bash -# When does it run next? -systemctl list-timers litellm-compat-matrix.timer +# Trigger a real run right now (PRs to litellm-docs). The id is the +# service id (`crn-...`) from the dashboard URL or `GET /v1/services`. +curl -fsS -X POST "https://api.render.com/v1/cron-jobs/${CRON_ID}/runs" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" -# Trigger a real run right now (PRs to litellm-docs). -sudo systemctl start litellm-compat-matrix.service +# Follow a run: the Logs tab on the service, or the API. +curl -fsS "https://api.render.com/v1/logs?ownerId=${OWNER_ID}&resource=${CRON_ID}&limit=100" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" -# Trigger a run that does NOT open a PR (good for first-time validation). -SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh +# Rebuild the image after a merge that touches tests/e2e/** (see the +# auto-deploy gotcha below). The deploy is done once its status is +# `live`; a run triggered before that still uses the previous image. +curl -fsS -X POST "https://api.render.com/v1/services/${CRON_ID}/deploys" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" \ + -H 'Content-Type: application/json' -d '{"clearCache": "do_not_clear"}' +curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" -# Narrow to one cell while debugging. -SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \ - ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh +# A run that does NOT open a PR (first-time validation, CLI bumps): +# set SKIP_PUBLISH=1 on the service, trigger a run, then remove it. +# The matrix JSON is printed at the end of the run's log (nothing on +# the container's disk outlives the run) and saved to +# ~/compatibility-matrix.json for a local docker run. +# PYTEST_K='basic_messaging_non_streaming and anthropic' narrows the +# run to one cell the same way. -# Watch the most recent run. -journalctl -u litellm-compat-matrix.service -f - -# Read older runs. -journalctl -u litellm-compat-matrix.service --since '2 days ago' - -# Disable until further notice (e.g. while debugging). -sudo systemctl disable --now litellm-compat-matrix.timer +# Build and run the image locally (docker on Apple silicon needs the +# platform flag; the context is tests/e2e, see the table above). +docker build --platform linux/amd64 \ + -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix tests/e2e +docker run --rm --platform linux/amd64 \ + --env-file litellm-compat-matrix.env -e SKIP_PUBLISH=1 \ + -v "$PWD/secrets:/etc/secrets:ro" compat-matrix ``` ## Gotchas - **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The - e2e suite uses PEP 695 `type` aliases, which the VM's system Python - (3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython + e2e suite uses PEP 695 `type` aliases, which the image's Debian + Python can't parse; `run_daily.sh` has uv fetch a managed CPython into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against - it. The first run after a version bump is a cold venv rebuild. -- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd - into the same VM with their own `:4000` proxy doesn't collide with a - cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env` - if you need to. + it. +- **The proxy port is `4100`, not `4000`.** Kept from the VM days so a + developer running the script locally next to their own `:4000` proxy + doesn't collide. Override with `PROXY_PORT=...`. - **`uv sync --frozen` requires the resolved tag to be tagged on - GitHub.** If the latest stable release was made but not pushed as a - git tag, the `git checkout` step fails. Push the tag, then rerun. + GitHub, and the wheel install requires it on PyPI.** If the latest + stable release was made but not pushed as a git tag, the `git + checkout` step fails; push the tag, then rerun. PyPI has had every + stable version days before its GitHub release so far (1.102.0 was + uploaded 2026-09-20, released on GitHub 2026-09-22), so the + `--no-build` install failing means the wheel is genuinely missing, + not late. +- **Pushes do not redeploy the service; deploy by hand.** `autoDeploy` + is `yes` on the service, but Render only hears about pushes through + its GitHub app, which is not installed on the `BerriAI` org (an org + admin step), so no push to the branch has ever started a deploy. + After a merge that changes anything under `tests/e2e/**`, run the + deploy command from the operating section (or "Manual Deploy" on the + dashboard) and wait for `live` before triggering a run, otherwise + the next scheduled run still executes the old image. - **Publish-token rotation is your problem.** The cron does not - refresh the token; if `mateo-berri`'s PAT in - `/etc/litellm-compat-matrix-github-token` expires, the run fails at - the `git push`/`gh pr create` step with a 401 ("Bad credentials" / - "Authentication failed"). Mint a fresh PAT and update that file. - The token needs write access to `BerriAI/litellm-docs` (classic - `repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is - delivered via systemd `LoadCredential`, not the env file, so pytest, - the proxy, and the claude CLI never inherit it; manual runs export - `GITHUB_TOKEN` instead. -- **First run after upgrading the Claude Code CLI is the riskiest one.** - If the new CLI changes its wire format the matrix run can produce - systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI - upgrade before letting the next scheduled fire happen. -- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory - is ~1 GB. Plan for at least 5 GB free on the VM, otherwise - `uv sync` will fail mid-run and leave you with a half-installed venv. + refresh the token; if `mateo-berri`'s PAT in the `github-token` + secret file expires, the run fails at the `git push`/`gh pr create` + step with a 401 ("Bad credentials" / "Authentication failed"). Mint + a fresh PAT and replace the secret file on the service. The token + needs write access to `BerriAI/litellm-docs` (classic `repo` scope, + or fine-grained Contents:RW + Pull requests:RW). It is delivered as + a file, not an env var, so pytest, the proxy, and the claude CLI + never inherit it; manual runs export `GITHUB_TOKEN` instead. +- **Bumping the Claude Code CLI is a PR.** Change `CLAUDE_CODE_VERSION` + in the `Dockerfile` and set `CLAUDE_CODE_SHA256` to the `linux-x64` + checksum from + `https://downloads.claude.ai/claude-code-releases//manifest.json`. + The first run on a new CLI is the riskiest one: if the new CLI + changes its wire format the matrix run can produce systematic + failures, so trigger a `SKIP_PUBLISH=1` run before the next scheduled + fire. `gh` and `uv` bump the same way, with the checksum from the + release's `gh__checksums.txt` and the tarball's `.sha256` + sidecar respectively. +- **A local build on Apple silicon only proves the image assembles.** + Under QEMU the Claude Code binary (a Bun executable) dies with + `CPU lacks AVX support` and `gh` panics in the Go runtime, so + `claude --version` and a full run are verified with a + `SKIP_PUBLISH=1` run on Render, not locally. +- **Nothing persists between runs.** A failed run leaves no + half-installed venv behind, but also no cache: don't expect a rerun + to be faster than the first one. diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example index d15561e96cd..579752f1ea8 100644 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -1,9 +1,8 @@ -# Environment file consumed by `litellm-compat-matrix.service`. +# Environment variables of the Render cron job `litellm-compat-matrix`. # -# Install at `/etc/litellm-compat-matrix.env` and chmod 0600. -# `EnvironmentFile=-` in the unit means the service is allowed to start -# even if this file is missing, but the populator will fail at the -# first provider request without these credentials. +# Set every value here on the Render service (Environment tab, or the +# `envVars` list of the create-service call in README.md). A local run +# passes a filled-in copy with `docker run --env-file`. # Anthropic ANTHROPIC_API_KEY= @@ -17,11 +16,12 @@ AWS_BEARER_TOKEN_BEDROCK= AWS_REGION_NAME=us-east-1 # Vertex AI (vertex_ai + vertex_ai_gpt columns). -# On the GCP VM, the default service-account ADC from the metadata server -# is used -- no JSON key file is needed. If you ever need to run outside -# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json. +# The service-account key JSON is the Render secret file +# `vertex-service-account.json`, mounted at /etc/secrets, and +# GOOGLE_APPLICATION_CREDENTIALS points google-auth at it. VERTEXAI_PROJECT= VERTEXAI_LOCATION=global +GOOGLE_APPLICATION_CREDENTIALS=/etc/secrets/vertex-service-account.json # Azure AI Foundry (azure column — Claude models on Foundry) AZURE_AI_API_KEY= @@ -35,19 +35,20 @@ AZURE_API_BASE= AZURE_API_KEY= # The publish PAT (mateo-berri, write access on BerriAI/litellm-docs) -# deliberately does NOT live in this file. Everything here lands in the -# process environment of pytest, the proxy, and the model-driven claude -# CLI, where any same-UID reader can lift it from /proc//environ. -# Instead, install the token at /etc/litellm-compat-matrix-github-token -# (chmod 0600, single line); the service maps it in via systemd -# LoadCredential and run_daily.sh keeps it out of every child process -# env. Used to (a) resolve the latest stable release, (b) push the -# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open -# the same-repo PR, and (d) enable squash auto-merge on it. Scopes: +# deliberately is NOT an env var. Everything here lands in the process +# environment of pytest, the proxy, and the model-driven claude CLI, +# where any same-UID reader can lift it from /proc//environ. +# Instead, the token is the Render secret file `github-token` (single +# line), mounted under CREDENTIALS_DIRECTORY, and run_daily.sh reads it +# from there and keeps it out of every child process env. Used to +# (a) resolve the latest stable release, (b) push the daily +# compat-matrix branch directly to BerriAI/litellm-docs, (c) open the +# same-repo PR, and (d) enable squash auto-merge on it. Scopes: # classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs # with Contents:RW + Pull requests:RW + Workflows:RW. # Manual runs export GITHUB_TOKEN instead, or skip publishing entirely # with SKIP_PUBLISH=1 (only writes the matrix JSON locally). +CREDENTIALS_DIRECTORY=/etc/secrets # Optional: the bedrock_mantle column is opt-in because the AWS account # needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the @@ -59,9 +60,9 @@ AZURE_API_KEY= # usually run them. Skipped cells are recorded as not_tested. # COMPAT_OPENAI_GPT_CELLS=1 -# Optional overrides; defaults are sensible for the cron VM. +# Optional overrides; defaults are sensible for the cron job. # PROXY_PORT=4100 -# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree +# LITELLM_WORKTREE=/home/populator/litellm-cron-worktree # DOCS_REPO=BerriAI/litellm-docs # DOCS_BRANCH=main # DOCS_TARGET_PATH=src/data/compatibility-matrix.json diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service deleted file mode 100644 index 6c74b3b04bb..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service +++ /dev/null @@ -1,113 +0,0 @@ -# systemd service for the Claude Code compatibility-matrix populator. -# -# Triggered by `litellm-compat-matrix.timer`; not started directly. The -# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics -# describe "run once per day" cleanly — there's no long-lived daemon to -# supervise; each invocation runs the populator end-to-end and exits. -# -# Install -# ------- -# -# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ -# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ -# sudo systemctl daemon-reload -# sudo systemctl enable --now litellm-compat-matrix.timer -# -# Paths are hard-coded to /home/mateo rather than using systemd's %h -# specifier. Why: in *system* units (this one), %h is expanded at -# parse time against the *manager's* home -- which is /root for PID 1 -# -- and *not* against the User= directive. That mismatch makes -# ReadWritePaths point at /root/.cache (which doesn't exist), causing -# the namespace setup to fail with status=226/NAMESPACE before the -# script ever runs. The runtime user (`User=mateo`) must: -# -# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the -# publisher module is importable; -# * have a uv venv at `~/litellm/litellm/.venv` (created by -# `uv sync --frozen` inside that checkout once); -# * have `gh` already authenticated against an account with -# `pull-requests: write` on `BerriAI/litellm-docs`; -# * have provider credentials exported in `/etc/litellm-compat-matrix.env` -# (see `litellm-compat-matrix.env.example` in this directory); -# * have the mateo-berri publish PAT at -# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single -# line), delivered via `LoadCredential=` below. - -[Unit] -Description=Claude Code compatibility-matrix populator (oneshot) -Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -User=mateo -Group=mateo - -# Provider credentials + any gh/PROXY_PORT overrides live here. Format -# is the standard `KEY=value` one line per env var. -EnvironmentFile=-/etc/litellm-compat-matrix.env - -# The mateo-berri publish PAT is mapped in via the credential store, NOT -# the EnvironmentFile, so it never lands in the process environment that -# pytest, the proxy, and the model-driven claude CLI inherit (any -# same-UID process can read /proc//environ). run_daily.sh reads -# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call. -# Unlike EnvironmentFile= above, this is deliberately NOT optional: a -# missing token file fails the unit at start instead of 30 minutes in. -LoadCredential=github-token:/etc/litellm-compat-matrix-github-token - -# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin). -# `uv` and `claude` are installed under the runtime user's `~/.local/bin` -# so we have to prepend it explicitly; otherwise run_daily.sh fails at -# the up-front command-presence check. -Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be -# explicit so anything that reads $HOME (e.g. uv's cache lookup, the -# claude CLI's per-session dir) sees the right value even if a future -# refactor flips DynamicUser= or PrivateUsers= on. -Environment=HOME=/home/mateo - -WorkingDirectory=/home/mateo/litellm/litellm - -ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh - -# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new -# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, -# plus the full feature x provider grid of pytest cells hitting several -# cloud providers. -TimeoutStartSec=90min - -# A failed run shouldn't restart automatically — the next timer fire is -# the right retry. Reruns of the same day's matrix are idempotent. -Restart=no - -# Security hardening: the populator only reads the litellm checkout and -# the env-file; everything else it writes lives in either the worktree -# (managed) or `/tmp` (cleaned up by tempfile). -# -# ReadWritePaths whitelist: -# * litellm-cron-worktree - the long-lived stable-tag checkout + -# its `.venv` (`uv sync` rewrites every -# run) + `.uv-bin` (pinned `uv` binary -# cache). -# * .cache - uv's wheel cache (~/.cache/uv) so we -# don't redownload pinned deps each run. -# * .claude - `claude` CLI's per-session state under -# `~/.claude/projects//`; created -# on every `claude --print` invocation. -# * .config/gh - `gh` CLI host config; technically not -# needed when we pass GH_TOKEN inline, -# but cheap to whitelist and prevents -# future regressions if a code path -# ever falls back to the host config. -# * /tmp - mktemp -d workdir + proxy logs. -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp -PrivateTmp=true - -[Install] -WantedBy=multi-user.target diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer deleted file mode 100644 index ee22538c6ed..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer +++ /dev/null @@ -1,25 +0,0 @@ -# Daily timer for the compatibility-matrix populator. -# -# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so -# operators in US/EU timezones see fresh PRs at the start of their work -# day. -# -# `Persistent=true` causes a missed run (VM was off / suspended) to -# fire the next time the timer is started, which is the property we -# want for a once-a-day job: the matrix should refresh as soon as the -# VM is reachable again, not wait another 24h. -# -# `RandomizedDelaySec=10min` smears load if multiple matrix-style -# pipelines are ever colocated on the same VM in the future. - -[Unit] -Description=Run the Claude Code compatibility-matrix populator daily - -[Timer] -OnCalendar=*-*-* 06:00:00 UTC -Persistent=true -RandomizedDelaySec=10min -Unit=litellm-compat-matrix.service - -[Install] -WantedBy=timers.target diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index e878007d8a3..172dd03b614 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Daily Claude Code compatibility-matrix populator. # -# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the -# systemd timer in this directory. The flow is: +# Runs daily as the Render cron job `litellm-compat-matrix`, built from +# the Dockerfile in this directory (see README.md). The flow is: # # 1. Resolve the latest LiteLLM final release tag from the GitHub # Releases API. @@ -33,12 +33,12 @@ # rather than spawning a new one. If the JSON is byte-identical to the # docs branch, we skip the push entirely. # -# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm. +# Required commands on $PATH: git, uv, gh, jq, curl, claude. # Required state: a litellm checkout at $LITELLM_REPO (this file lives in -# it), $WORKTREE is created on first run, gh is already authenticated. +# it); $WORKTREE is created on first run. # -# Override any default by setting the matching env var; see the systemd -# unit for the production wiring. +# Override any default by setting the matching env var; see README.md +# for the production wiring. set -Eeuo pipefail @@ -52,12 +52,12 @@ DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" SKIP_PUBLISH="${SKIP_PUBLISH:-0}" PYTEST_K="${PYTEST_K:-}" # The e2e suite uses PEP 695 `type` aliases, so the venv needs Python -# >= 3.12 (also what repo CI runs) even when the VM's system python is +# >= 3.12 (also what repo CI runs) even when the host's system python is # older. uv fetches a managed CPython of this version on first use -- # checksum-verified against the manifest baked into the pinned uv # binary -- and installs it under ${WORKTREE}/.uv-python (see -# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the -# systemd sandbox lets us write to. +# UV_PYTHON_INSTALL_DIR below) so everything the run writes lives inside +# the worktree. CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}" # Merge method for auto-merge. BerriAI/litellm-docs only allows squash # merges (merge-commit and rebase are disabled at the repo level), so @@ -113,9 +113,9 @@ for cmd in git uv gh jq curl claude; do done # Publishing pushes the branch straight to BerriAI/litellm-docs and opens -# the PR as mateo-berri, who has write access on the docs repo. Under -# systemd the PAT arrives as a file via LoadCredential=, NOT via the -# EnvironmentFile: several suite cells let the model-driven claude CLI +# the PR as mateo-berri, who has write access on the docs repo. On +# Render the PAT arrives as a secret file under ${CREDENTIALS_DIRECTORY}, +# NOT via an env var: several suite cells let the model-driven claude CLI # read arbitrary files as this user, and /proc//environ of the # script, pytest, and the proxy would hand an env-borne token to any # same-UID reader. Kept as an unexported shell variable and passed per @@ -125,13 +125,18 @@ done # quota. if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")" - log "publish token source: systemd credential store" + log "publish token source: ${CREDENTIALS_DIRECTORY}/github-token" elif [[ -n "${GITHUB_TOKEN:-}" ]]; then log "publish token source: process environment" fi if [[ "${SKIP_PUBLISH}" != "1" ]]; then [[ -n "${GITHUB_TOKEN:-}" ]] \ - || die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)" + || die "publish token required: the github-token secret file under CREDENTIALS_DIRECTORY, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)" + # The stale-PR sweep below closes only PRs this account opened, so the + # login is resolved from the token once rather than hardcoded. + PUBLISH_LOGIN="$(GH_TOKEN="${GITHUB_TOKEN}" gh api user --jq .login)" \ + || die "could not resolve the publishing account from the github token" + log "publishing as ${PUBLISH_LOGIN}" fi # --------------------------------------------------------------------------- @@ -205,7 +210,7 @@ log "local claude code: ${CLAUDE_CODE_VERSION}" if [[ ! -d "${WORKTREE}/.git" ]]; then log "first run: cloning litellm into ${WORKTREE}" mkdir -p "$(dirname "${WORKTREE}")" - git clone https://github.com/BerriAI/litellm.git "${WORKTREE}" + git clone --filter=blob:none https://github.com/BerriAI/litellm.git "${WORKTREE}" fi log "updating worktree to ${LITELLM_VERSION}" @@ -221,40 +226,27 @@ git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" # Always rebuild tests/e2e/ in the worktree from the dev checkout, -# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two -# reasons: +# regardless of what the resolved ${LITELLM_VERSION} tag ships: the +# matrix populator's job is to exercise *today's* tests against the +# latest stable proxy, and the dev checkout carries the most recent +# test fixes that haven't yet rolled into a stable release. # -# * The matrix populator's job is to exercise *today's* tests against -# the latest stable proxy. The dev checkout carries the most recent -# test fixes that haven't yet rolled into a stable release, and we -# want every cron run to pick those up the moment they land on -# ${LITELLM_REPO}, not whenever the next stable release happens. -# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose -# top-level conftest.py imports modules (e2e_db, lifecycle, -# otel_client, ...) that the stable venv does not install. Copying -# the whole tree would make pytest collection blow up on those -# imports. -# -# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY -# the claude_code suite plus the shared transport helpers it imports. +# The whole tree is copied rather than an allowlist of the helpers the +# suite imports: the helpers import each other (proxy_client -> +# e2e_config -> fixture_mode -> ...), so a new edge in that graph turned +# an allowlist into a ModuleNotFoundError at conftest load. The tree's +# top-level conftest.py pulls in the full EKS harness (e2e_db, +# lifecycle, ...), which the stable venv does not install, so the pytest +# run below points --confcutdir at claude_code/ and never loads it. # pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while # claude_code/ does), which is what resolves both the `claude_code.*` # and the bare `proxy_client` / `e2e_http` imports inside the suite. -E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py) -if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then - die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" -fi -for helper in "${E2E_HELPER_FILES[@]}"; do - [[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \ - || die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}" -done -log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)" +[[ -d "${LITELLM_REPO}/tests/e2e/claude_code" ]] \ + || die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" +log "shimming tests/e2e/ from ${LITELLM_REPO} (always-overwrite)" rm -rf "${WORKTREE}/tests/e2e" mkdir -p "${WORKTREE}/tests/e2e" -cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/" -for helper in "${E2E_HELPER_FILES[@]}"; do - cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/" -done +cp -r "${LITELLM_REPO}/tests/e2e/." "${WORKTREE}/tests/e2e/" # litellm pins an exact uv version in pyproject.toml's [tool.uv] # `required-version` field, so a system uv that's newer or older @@ -305,10 +297,18 @@ fi # actually serve. `--group proxy-dev` brings in pytest and the rest of # what tests/e2e/claude_code/ needs. `--python` pins the venv to # ${CRON_PYTHON_VERSION}; the first run after a version bump recreates -# the venv from scratch (a one-time cold sync). +# the venv from scratch (a one-time cold sync). `--no-install-project` +# leaves litellm itself out: the tag builds a Rust extension through +# maturin, which needs a C and Rust toolchain the image does not carry, +# so the published PyPI wheel (what users install) goes in right after, +# and every later `uv run` passes `--no-sync` so uv never tries to put +# the source build back. export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python" -log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})" -(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}") +log "uv sync --frozen --group proxy-dev --extra proxy --no-install-project --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})" +(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --no-install-project --python "${CRON_PYTHON_VERSION}") +LITELLM_WHEEL_VERSION="${LITELLM_VERSION#v}" +log "installing the published litellm==${LITELLM_WHEEL_VERSION} wheel from PyPI" +"${WORKTREE_UV}" pip install --python "${WORKTREE}/.venv/bin/python" --no-deps --no-build "litellm==${LITELLM_WHEEL_VERSION}" PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml" [[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)" @@ -321,10 +321,10 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}" # Bind the proxy to loopback only. The populator proxy is talked to # exclusively by the pytest run on the same host (the health check and # the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`), -# so there's no reason to expose it on the VM's external interfaces. +# so there's no reason to expose it on the container's external interfaces. # Without `--host`, `litellm` defaults to 0.0.0.0, which combined with # the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would -# allow anything that can reach :${PROXY_PORT} on the VM to authenticate +# allow anything that can reach :${PROXY_PORT} on the host to authenticate # and burn upstream provider credentials. # # `setsid` puts the proxy in its own session+pgroup so cleanup() can @@ -334,7 +334,7 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}" setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c ' echo "$$" > "$0" cd "$1" - exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4" + exec "$2" run --no-sync litellm --config "$3" --host 127.0.0.1 --port "$4" ' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \ >"${WORKDIR}/proxy.log" 2>&1 & disown @@ -359,6 +359,7 @@ RESULTS_JSON="${WORKDIR}/compat-results.json" # the cron skips them if/when they land in the suite. PYTEST_ARGS=( tests/e2e/claude_code/ + --confcutdir=tests/e2e/claude_code "--ignore-glob=*_unit_tests*" ) if [[ -n "${PYTEST_K}" ]]; then @@ -373,7 +374,7 @@ set +e && LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \ LITELLM_MASTER_KEY="${PROXY_API_KEY}" \ COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ - "${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}" + "${WORKTREE_UV}" run --no-sync pytest "${PYTEST_ARGS[@]}" ) PYTEST_EXIT=$? set -e @@ -392,7 +393,7 @@ MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" log "building ${MATRIX_JSON}" ( cd "${WORKTREE}" \ - && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \ + && "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/build_matrix.py" \ --manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \ --results "${RESULTS_JSON}" \ --output "${MATRIX_JSON}" \ @@ -405,8 +406,9 @@ log "building ${MATRIX_JSON}" # --------------------------------------------------------------------------- if [[ "${SKIP_PUBLISH}" == "1" ]]; then - cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json" - log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json" + cp "${MATRIX_JSON}" "${HOME}/compatibility-matrix.json" + log "SKIP_PUBLISH=1; matrix saved to ${HOME}/compatibility-matrix.json and printed below" + cat "${MATRIX_JSON}" exit 0 fi @@ -415,7 +417,7 @@ BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC} DOCS_CLONE="${WORKDIR}/litellm-docs" log "cloning ${DOCS_REPO}@${DOCS_BRANCH}" -gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" +GH_TOKEN="${GITHUB_TOKEN}" gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" cd "${DOCS_CLONE}" git config user.email "litellm-bot@berri.ai" @@ -452,7 +454,7 @@ log "checking for green->red regressions vs the published matrix" set +e REGRESSION_REPORT="$( cd "${WORKTREE}" \ - && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \ + && "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/check_regressions.py" \ --old "${PUBLISHED_MATRIX}" \ --new "${MATRIX_JSON}" )" @@ -492,7 +494,7 @@ git commit -m "${COMMIT_MSG}" # # Plain --force (not --force-with-lease) is acceptable here: the # compat-matrix/* branch is bot-owned, only this script ever writes to -# it, and runs are serialized by the systemd timer. --force-with-lease +# it, and runs are serialized by the cron schedule. --force-with-lease # would require a fetch to populate the remote-tracking ref before each # push and adds no safety in this single-writer setup. PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git" @@ -553,7 +555,7 @@ Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without mergi EOF )" -log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)" +log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as ${PUBLISH_LOGIN})" # GH_TOKEN is mateo-berri's write-scoped token, the same identity used # for release-listing above. The branch lives on ${DOCS_REPO} itself, so # --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`. @@ -644,15 +646,25 @@ fi # # Non-fatal: a sweep failure (rate limit, transient API error) leaves # stale PRs for the next run to retry; it must not fail the pipeline. +# +# The docs repo carries a few hundred open PRs, so the list has to page +# past gh's default 30 (and the earlier 100, which never reached a +# week-old compat-matrix PR and left it open for good). +# +# `compat-matrix/` is only a naming convention, so the prefix alone does +# not make a PR this job's: a contributor can open a fork PR under that +# name. Only PRs the publishing account itself opened from a branch on +# the docs repo qualify; anything else stays untouched. log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})" set +e STALE_PRS="$( GH_TOKEN="${GITHUB_TOKEN}" gh pr list \ --repo "${DOCS_REPO}" \ --state open \ - --limit 100 \ - --json number,headRefName \ - --jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"' + --author "${PUBLISH_LOGIN}" \ + --limit 1000 \ + --json number,headRefName,isCrossRepository \ + --jq '.[] | select((.headRefName | startswith("compat-matrix/")) and (.isCrossRepository | not)) | "\(.number)\t\(.headRefName)"' )" while IFS=$'\t' read -r stale_pr stale_head; do [[ -z "${stale_pr}" ]] && continue @@ -660,7 +672,7 @@ while IFS=$'\t' read -r stale_pr stale_head; do GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \ --repo "${DOCS_REPO}" \ --delete-branch \ - --comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /' + --comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open" 2>&1 | sed 's/^/ /' if [[ ${PIPESTATUS[0]} -eq 0 ]]; then log "closed stale compat-matrix PR #${stale_pr} (${stale_head})" else diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 24bd76d4429..022caddd42a 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -345,6 +345,50 @@ def request_with_retry[T: RetryableResponse]( return issue() +PROVIDER_RATE_LIMIT_MARKER: Final = "litellm.RateLimitError" +PROVIDER_RATE_LIMIT_ATTEMPTS: Final = 4 +PROVIDER_RATE_LIMIT_BACKOFF_SECONDS: Final = 5.0 + + +def tolerate_provider_rate_limit[R: BaseModel]( + issue: Callable[[], Result[R]], + *, + attempts: int = PROVIDER_RATE_LIMIT_ATTEMPTS, + sleep: Callable[[float], None] = time.sleep, +) -> Result[R]: + """Retry a call up to `attempts` times while the proxy relays the provider's own 429; + any other outcome, the proxy's own 429 included, comes back at once.""" + for attempt in range(1, attempts): + match issue(): + case RateLimitedError(body=body, retry_after_seconds=retry_after) if PROVIDER_RATE_LIMIT_MARKER in body: + delay = retry_after or PROVIDER_RATE_LIMIT_BACKOFF_SECONDS * (1 << (attempt - 1)) + print( + f"e2e-http: provider rate limit relayed by the proxy; retry {attempt}/{attempts - 1} in {delay}s", + flush=True, + ) + sleep(delay) + case result: + return result + return issue() + + +class ProxyErrorDetail(BaseModel): + message: str + type: str + code: str + + +class _ProxyErrorBody(BaseModel): + error: ProxyErrorDetail + + +def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None: + """The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own.""" + if PROVIDER_RATE_LIMIT_MARKER not in outcome.body: + return None + return _ProxyErrorBody.model_validate_json(outcome.body).error + + class ClassifiableResponse(Protocol): """What classifying an outcome reads off a response. requests.Response satisfies it, and so does a fake, so the classification rules are testable on their own.""" @@ -931,7 +975,10 @@ class PreparedForward: def prepare_forward( - method: str, url: str, headers: dict[str, str], body: bytes | None, + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, ) -> PreparedForward | NetworkError: try: with requests.Session() as session: @@ -950,7 +997,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream except requests.RequestException as exc: return NetworkError(message=str(exc)) return StreamHead( - resp.status_code, {name.lower(): value for name, value in resp.headers.items()}, + resp.status_code, + {name.lower(): value for name, value in resp.headers.items()}, primed_steps(_stream_steps(resp)), ) diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 60f875ccb7d..17223dc36fa 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -17,6 +17,7 @@ from models import ( AnthropicMessagesResponse, ChatBody, ChatMessage, + ChatMetadata, ChatResponse, ChatTool, KeyGenerateBody, @@ -133,6 +134,31 @@ class GuardrailCreateResponse(BaseModel): guardrail_id: str +class PolicyConditionBody(BaseModel): + model: str + + +class PolicyCreateBody(BaseModel): + policy_name: str + inherit: str | None = None + guardrails_add: list[str] + condition: PolicyConditionBody | None = None + + +class PolicyCreateResponse(BaseModel): + policy_id: str + policy_name: str + + +class PolicyAttachmentCreateBody(BaseModel): + policy_name: str + tags: list[str] + + +class PolicyAttachmentCreateResponse(BaseModel): + attachment_id: str + + class ApplyGuardrailRequest(BaseModel): guardrail_name: str text: str @@ -243,6 +269,49 @@ class GuardrailsClient: response_type=NoBody, ) + def create_policy(self, body: PolicyCreateBody) -> str: + """Create a policy via POST /policies and return its name once every replica + can be expected to serve it (policies reach the data plane on the periodic + DB sync, same as guardrails).""" + created = unwrap( + self.proxy.transport.post( + "/policies", + headers=self.proxy.transport.master, + json=body, + response_type=PolicyCreateResponse, + ) + ) + settle_propagation(time.monotonic()) + return created.policy_name + + def delete_policy(self, policy_name: str) -> None: + _ = self.proxy.transport.delete( + f"/policies/name/{policy_name}/all-versions", + headers=self.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + + def attach_policy_to_tags(self, policy_name: str, tags: list[str]) -> str: + attachment_id = unwrap( + self.proxy.transport.post( + "/policies/attachments", + headers=self.proxy.transport.master, + json=PolicyAttachmentCreateBody(policy_name=policy_name, tags=tags), + response_type=PolicyAttachmentCreateResponse, + ) + ).attachment_id + settle_propagation(time.monotonic()) + return attachment_id + + def delete_policy_attachment(self, attachment_id: str) -> None: + _ = self.proxy.transport.delete( + f"/policies/attachments/{attachment_id}", + headers=self.proxy.transport.master, + json=NoBody(), + response_type=NoBody, + ) + def create_team_opted_out_of_global_guardrails(self, alias: str) -> str: team_id = unwrap( self.proxy.transport.post( @@ -322,11 +391,13 @@ class GuardrailsClient: max_tokens: int = 16, tools: list[ChatTool] | None = None, tool_choice: str | None = None, + tags: list[str] | None = None, ) -> StreamingResponse: """Drive /chat/completions returning the raw HTTP outcome, for the assertions a typed body cannot carry: the `x-litellm-applied-guardrails` response header, which is how an ALLOW scenario proves the guardrail ran - rather than being absent.""" + rather than being absent. `tags` land in `metadata.tags`, which is what a + tag-scoped policy attachment matches on.""" return self.proxy.transport.send( "/chat/completions", headers=self.proxy.transport.bearer(key), @@ -337,6 +408,7 @@ class GuardrailsClient: guardrails=guardrails, tools=tools, tool_choice=tool_choice, + metadata=ChatMetadata(tags=tags) if tags is not None else None, ), ) diff --git a/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py b/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py new file mode 100644 index 00000000000..6298a1de038 --- /dev/null +++ b/tests/e2e/guardrails/test_policy_inherited_guardrail_e2e.py @@ -0,0 +1,127 @@ +"""Live e2e: a policy attached to a request keeps its inherited parent guardrails +when only the child's own `condition` fails to match the request model. + +The parent policy has no condition and adds a content filter. The child inherits +it, adds a second content filter, and carries a model condition. The attachment +points at the child only, so the parent is reachable through inheritance alone. +A request the child condition does not match must still be blocked by the +parent's filter; a request it does match must be blocked by both. + +Uses litellm_content_filter (keyword match, no external service) so the block is +deterministic and free, with the request model routed to a real provider. +""" + +from __future__ import annotations + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_http import StreamingResponse +from guardrails_client import ( + GuardrailsClient, + PolicyConditionBody, + PolicyCreateBody, +) +from lifecycle import ResourceManager + +pytestmark = pytest.mark.e2e + +MODEL = CHEAP_OPENAI_MODEL + + +def _applied_guardrails(outcome: StreamingResponse) -> frozenset[str]: + return frozenset( + name.strip() for name in outcome.headers.get("x-litellm-applied-guardrails", "").split(",") if name.strip() + ) + + +def _setup_child_policy_attached_to_tag( + client: GuardrailsClient, + resources: ResourceManager, + *, + child_condition_model: str, + parent_banned: str, + child_banned: str, + tag: str, +) -> tuple[str, str]: + """Register parent and child content filters, a parent policy adding the parent + filter, a child policy inheriting it with `child_condition_model`, and attach + only the child to `tag`. Returns (parent_guardrail_name, child_guardrail_name).""" + parent_guardrail = f"e2e-parent-guard-{parent_banned}" + child_guardrail = f"e2e-child-guard-{child_banned}" + parent_guardrail_id = client.create_content_filter_guardrail(parent_guardrail, parent_banned, default_on=False) + resources.defer(lambda: client.delete_guardrail(parent_guardrail_id)) + child_guardrail_id = client.create_content_filter_guardrail(child_guardrail, child_banned, default_on=False) + resources.defer(lambda: client.delete_guardrail(child_guardrail_id)) + + parent_policy = client.create_policy( + PolicyCreateBody(policy_name=f"e2e-parent-policy-{parent_banned}", guardrails_add=[parent_guardrail]) + ) + resources.defer(lambda: client.delete_policy(parent_policy)) + child_policy = client.create_policy( + PolicyCreateBody( + policy_name=f"e2e-child-policy-{child_banned}", + inherit=parent_policy, + guardrails_add=[child_guardrail], + condition=PolicyConditionBody(model=child_condition_model), + ) + ) + resources.defer(lambda: client.delete_policy(child_policy)) + + attachment_id = client.attach_policy_to_tags(child_policy, [tag]) + resources.defer(lambda: client.delete_policy_attachment(attachment_id)) + return parent_guardrail, child_guardrail + + +class TestPolicyInheritedGuardrail: + def test_child_condition_miss_still_applies_inherited_parent_guardrail( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + parent_banned = unique_marker() + child_banned = unique_marker() + tag = f"e2e-policy-tag-{unique_marker()}" + parent_guardrail, child_guardrail = _setup_child_policy_attached_to_tag( + client, + resources, + child_condition_model=f"never-matches-{unique_marker()}", + parent_banned=parent_banned, + child_banned=child_banned, + tag=tag, + ) + + outcome = client.chat_raw(scoped_key, MODEL, f"Reply with the single word OK. {parent_banned}", tags=[tag]) + + assert outcome.status_code == 400, ( + f"the inherited parent content filter must block the banned keyword even though the child " + f"policy's own model condition does not match {MODEL}; got {outcome.status_code}: {outcome.body[:300]}" + ) + assert parent_guardrail in _applied_guardrails(outcome), ( + f"x-litellm-applied-guardrails must name the inherited parent guardrail; got {outcome.headers}" + ) + assert child_guardrail not in _applied_guardrails(outcome), ( + f"the child's own guardrail must not run when its condition fails; got {outcome.headers}" + ) + + def test_child_condition_match_applies_child_and_inherited_parent_guardrails( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + parent_banned = unique_marker() + child_banned = unique_marker() + tag = f"e2e-policy-tag-{unique_marker()}" + parent_guardrail, child_guardrail = _setup_child_policy_attached_to_tag( + client, + resources, + child_condition_model=MODEL, + parent_banned=parent_banned, + child_banned=child_banned, + tag=tag, + ) + + outcome = client.chat_raw(scoped_key, MODEL, f"Reply with the single word OK. {child_banned}", tags=[tag]) + + assert outcome.status_code == 400, ( + f"the child's own content filter must block its banned keyword when the condition matches {MODEL}; " + f"got {outcome.status_code}: {outcome.body[:300]}" + ) + assert {parent_guardrail, child_guardrail} <= _applied_guardrails(outcome), ( + f"both the child and inherited parent guardrails must run; got {outcome.headers}" + ) diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index ceb3620183a..bec65144c9f 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -13,7 +13,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix call, so a never-seen prefix must come back cached on its very first call (Gemini's implicit caching cannot hit a cold prefix), the cached count must cover the marked block, and the spend row must be billed below the uncached - price of the prompt. + price of the cached tokens. Vertex's cache create is nondeterministic: + identical bodies come back 200 or with the minimum-token 400 ("The cached + content is of 1 tokens"), in failure bursts of 45 seconds and more, so up to + eight never-seen prefixes are tried with a pause after each rejection. The + billing check prices the cached tokens rather than prompt_tokens, which + Vertex reports inclusive of the cached prefix on some calls and exclusive + of it on others. - Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over the OpenAI-compatible route; the second call must report cache-read tokens > 0. - OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the @@ -50,7 +56,8 @@ VERTEX_MODEL = "vertex_ai/gemini-2.5-flash" ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" OPENAI_MODEL = "openai/gpt-5.6" VERTEX_CACHE_TTL: Final = "300s" -VERTEX_COLD_CALL_ATTEMPTS: Final = 3 +VERTEX_COLD_CALL_ATTEMPTS: Final = 8 +VERTEX_COLD_CALL_PAUSE_SECONDS: Final = 15.0 VERTEX_MINIMUM_CACHED_TOKENS: Final = 1024 CACHED_SHARE_OF_PROMPT: Final = 0.9 VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit caching" @@ -156,26 +163,36 @@ def _cold_cache_call(send: Callable[[str], Result[ChatResponse]]) -> ChatRespons return unwrap(result) +def _first_engaged_cold_call(send: Callable[[str], Result[ChatResponse]]) -> ChatResponse | None: + for attempt in range(1, VERTEX_COLD_CALL_ATTEMPTS + 1): + candidate = _cold_cache_call(send) + if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS: + return candidate + if attempt < VERTEX_COLD_CALL_ATTEMPTS: + print( + f"cache_control: vertex did not engage the cache on cold attempt {attempt}/{VERTEX_COLD_CALL_ATTEMPTS}; " + f"pausing {VERTEX_COLD_CALL_PAUSE_SECONDS}s before the next never-seen prefix", + flush=True, + ) + time.sleep(VERTEX_COLD_CALL_PAUSE_SECONDS) + return None + + def _first_cold_call_reads_cache(model: str, send: Callable[[str], Result[ChatResponse]]) -> ChatResponse: - completion: Final = next( - ( - candidate - for candidate in (_cold_cache_call(send) for _ in range(VERTEX_COLD_CALL_ATTEMPTS)) - if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS - ), - None, - ) + completion: Final = _first_engaged_cold_call(send) assert completion is not None, ( - f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control were each either " - f"rejected by Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} " - "cached tokens on their first call; explicit context caching did not engage" + f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control, spread over " + f"{VERTEX_COLD_CALL_PAUSE_SECONDS * (VERTEX_COLD_CALL_ATTEMPTS - 1):.0f}s, were each either rejected by " + f"Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} cached tokens on " + "their first call; explicit context caching did not engage" ) assert completion.choices, f"{model}: cached call returned no choices: {completion}" usage: Final = completion.usage cached: Final = _cached_read_tokens(usage) - assert usage and usage.prompt_tokens and cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, ( - f"{model}: only {cached} of {usage.prompt_tokens if usage else None} prompt tokens were served from the " - "cache; the cache_control block was not cached whole" + assert usage and usage.prompt_tokens, f"{model}: cached completion carried no prompt_tokens: {usage}" + assert cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, ( + f"{model}: only {cached} of {usage.prompt_tokens} prompt tokens were served from the cache; the " + "cache_control block was not cached whole" ) return completion @@ -196,10 +213,11 @@ def _assert_billed_below_uncached_prompt(client: PassthroughClient, model: str, assert row.prompt_tokens == usage.prompt_tokens, ( f"{model}: spend row prompt_tokens {row.prompt_tokens} != response prompt_tokens {usage.prompt_tokens}" ) - uncached_prompt_cost: Final = usage.prompt_tokens * _input_rate(client, model) - assert row.spend is not None and row.spend < uncached_prompt_cost, ( - f"{model}: spend {row.spend} is not below the uncached price of the prompt alone ({uncached_prompt_cost} for " - f"{usage.prompt_tokens} tokens); cache-read pricing was not applied" + cached: Final = _cached_read_tokens(usage) + uncached_read_cost: Final = cached * _input_rate(client, model) + assert row.spend is not None and row.spend < uncached_read_cost, ( + f"{model}: spend {row.spend} is not below the uncached price of the {cached} tokens read from the cache " + f"({uncached_read_cost}); cache-read pricing was not applied" ) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 363b2a7e02e..235856d7692 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -110,10 +110,11 @@ def _vision_messages() -> list[ChatMessage]: def _assert_describes_cat(response: ChatResponse) -> None: assert response.choices, f"vision returned no choices: {response}" - message = response.choices[0].message - content = (message.content if message else None) or "" + choice = response.choices[0] + content = (choice.message.content if choice.message else None) or "" assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), ( - f"vision response did not describe the image: {content[:200]}" + f"vision response did not describe the image: {content[:200]!r} " + f"(finish_reason={choice.finish_reason!r}, usage={response.usage})" ) @@ -421,7 +422,11 @@ class TestVertexChatCompletions: model = self._register(client, resources, "e2e-vertex-vision") key = resources.key() - response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + response = unwrap( + client.proxy.chat( + key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none") + ) + ) _assert_describes_cat(response) @pytest.mark.covers( diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index c2560b199af..2f7fc74e650 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case fails when no proxy answers and also fails once a request reaches it: the proxy fetches each provider's referenced secrets, so a missing credential surfaces as a live provider error rather than silent green. + +The provider keys are shared with other pipelines, so a provider's rate limit can +hold across the bounded retries. The case then accepts the gateway's faithful relay +of that 429 (throttling_error, code 429) as its second expected outcome; any other +non-success still fails at once. """ from __future__ import annotations from dataclasses import dataclass -from typing import Protocol +from typing import Final, Protocol import pytest from e2e_config import unique_marker -from e2e_http import assert_client_error, unwrap +from e2e_http import ( + PROVIDER_RATE_LIMIT_ATTEMPTS, + RateLimitedError, + Success, + assert_client_error, + relayed_provider_rate_limit, + tolerate_provider_rate_limit, +) from lifecycle import ResourceManager from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse from proxy_client import ProxyClient @@ -146,24 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None: assert response.pages[0].markdown is not None, "first page has no markdown" +def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None: + detail: Final = relayed_provider_rate_limit(outcome) + assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}" + assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}" + print( + f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; " + f"the gateway relayed it as {detail.type} {detail.code}", + flush=True, + ) + + class TestRustOcrGateway: @pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS) - def test_rust_ocr_response( - self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase - ) -> None: + def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None: model = f"rust-ocr-{case.suffix}-{unique_marker()}" model_id = proxy.create_model(model, case.provider.litellm_params()) resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() - response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document))) - _assert_ocr_document(response) + match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))): + case Success(data=response): + _assert_ocr_document(response) + case RateLimitedError() as outcome: + _assert_provider_rate_limit_relayed(model, outcome) + case outcome: + pytest.fail(f"{model}: {outcome!r}") @pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400") @pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works") - def test_missing_document_returns_error( - self, proxy: ProxyClient, resources: ResourceManager - ) -> None: + def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = f"rust-ocr-val-{unique_marker()}" model_id = proxy.create_model(model, MistralOcr().litellm_params()) resources.defer(lambda: proxy.delete_model(model_id)) @@ -174,4 +198,3 @@ class TestRustOcrGateway: json=_OptionalOcrBody(model=model), ) assert_client_error(result, "ocr missing document") - diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 4388e7f11bc..5984d5645d8 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -15,9 +15,10 @@ reliability behavior. from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from typing import Final -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from proxy_client import ProxyClient from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker @@ -367,6 +368,26 @@ def model_id_of(resp: StreamingResponse) -> str | None: return resp.headers.get("x-litellm-model-id") +class _AzurePromptFilterResult(BaseModel): + content_filter_results: Mapping[str, object] | None = None + + +class _AzureAnnotatedChatBody(BaseModel): + prompt_filter_results: Sequence[_AzurePromptFilterResult] | None = None + + +def azure_prompt_filter_skipped(resp: StreamingResponse) -> bool: + """True when Azure's 200 recorded no prompt-filter verdict (every `content_filter_results` + empty), so the prompt has to be sent again.""" + try: + annotated: Final = _AzureAnnotatedChatBody.model_validate_json(resp.body) + except ValidationError: + return False + if not annotated.prompt_filter_results: + return False + return all(not entry.content_filter_results for entry in annotated.prompt_filter_results) + + def _parsed(resp: StreamingResponse) -> ChatResponse | None: try: return ChatResponse.model_validate_json(resp.body) diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py index 8bc60f2829b..54c11b163d0 100644 --- a/tests/e2e/router/test_reliability_fallbacks_e2e.py +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -16,10 +16,16 @@ failure: the provider refuses the prompt itself, on length or on policy, and reroute those, not `fallbacks`. The policy refusal is a real one, from an Azure OpenAI content filter rejecting a jailbreak prompt, and a control call first proves the refusal reaches the customer as a 400 when no reroute is configured. +Azure intermittently answers without running its prompt filter at all (the +body's prompt_filter_results carry no verdict), which is not a pass, so both +calls send the prompt again, a bounded number of times, until the filter ran. """ from __future__ import annotations +from collections.abc import Callable +from typing import Final + import pytest from complexity_router_client import ComplexityRouterClient @@ -29,6 +35,7 @@ from lifecycle import ResourceManager from models import RouterSettingsOverride from reliability_support import ( CONTENT_POLICY_PROMPT, + azure_prompt_filter_skipped, chat_override, completion_tokens_of, content_of, @@ -64,6 +71,28 @@ def _assert_served_by_fallback(resp: StreamingResponse) -> None: assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}" +AZURE_FILTER_ATTEMPTS: Final = 3 + + +def _chat_once_azure_runs_its_filter(send: Callable[[], StreamingResponse]) -> StreamingResponse: + for attempt in range(1, AZURE_FILTER_ATTEMPTS): + resp = send() + if not azure_prompt_filter_skipped(resp): + return resp + print( + "e2e: azure answered without running its prompt filter; sending the jailbreak prompt again " + f"({attempt}/{AZURE_FILTER_ATTEMPTS - 1})", + flush=True, + ) + return send() + + +def _filter_verdict(resp: StreamingResponse) -> str: + if azure_prompt_filter_skipped(resp): + return "azure skipped its prompt filter on every attempt" + return "the filter ran and let the prompt through" + + class TestReliabilityFallbacks: @pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback") def test_5xx_routes_to_fallback( @@ -124,17 +153,21 @@ class TestReliabilityFallbacks: model_id = create_content_filtered_deployment(client.proxy, primary) resources.defer(lambda: client.proxy.delete_model(model_id)) - refused = chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}") + refused = _chat_once_azure_runs_its_filter( + lambda: chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}") + ) assert refused.status_code == 400, ( - f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code}: " - f"{refused.body[:300]}" + f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code} " + f"({_filter_verdict(refused)}): {refused.body[:300]}" ) - resp = chat_override( - client.proxy, - scoped_key, - primary, - f"{CONTENT_POLICY_PROMPT} {unique_marker()}", - override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]), + resp = _chat_once_azure_runs_its_filter( + lambda: chat_override( + client.proxy, + scoped_key, + primary, + f"{CONTENT_POLICY_PROMPT} {unique_marker()}", + override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]), + ) ) _assert_served_by_fallback(resp) diff --git a/tests/integration/README.md b/tests/integration/README.md index f21e04f1ca5..628b6721514 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -12,9 +12,9 @@ The generated lifecycle models use 20 examples, eight steps, generation and shri Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change -The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, skipped tests, failed cleanup or a selected test without a passed call fail qualification. Existing GitHub Actions jobs do not own these tests +The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. Existing GitHub Actions jobs do not own these tests -Define integration contract IDs and their canonical test nodes in `contracts.json`. Every node must declare the same IDs with `covers`. The runner checks exact collected and passed selections against that mapping. These IDs belong to this CircleCI suite and must not be added to the separate E2E coverage registry. A manifest declaration alone does not mean a test passed +Define integration contract IDs and their canonical test nodes in `contracts.json`. Every node must declare the same IDs with `covers`. The runner checks exact collected and passed-or-skipped selections against that mapping. These IDs belong to this CircleCI suite and must not be added to the separate E2E coverage registry. A manifest declaration alone does not mean a test passed Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream @@ -22,7 +22,7 @@ Fixtures must contain synthetic data only. Keep private incident records and sou Database cases own their temporary schemas, roles, constraints and proxy processes. They prove reader-versus-writer execution with PostgreSQL lock observations, exercise real transaction wait limits and verify rollback after a reached database failure -Accounting cases compare persisted input and output cost components against literal rates, including zero and default prices. Cache state models assert actual upstream calls, response identity and every persisted charge. Generated accounting tests have a 180-second test limit to accommodate the asynchronous spend writer; CircleCI keeps the whole shard capped at 11 minutes +Accounting cases compare persisted input and output cost components against literal rates, including zero and default prices. Cache state models assert actual upstream calls, response identity and every persisted charge. Generated accounting tests have a 180-second test limit to accommodate the asynchronous spend writer Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 0798927c6d1..5c44beaa570 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -1,6 +1,6 @@ import os -import socket import signal +import socket import subprocess import sys import time @@ -13,7 +13,6 @@ from typing import Final import httpx import psutil - from integration._support.client import Gateway @@ -61,9 +60,10 @@ def owned_proxy( *, config: Path | None = None, remove_environment: tuple[str, ...] = (), + workers: int = 1, ) -> Iterator[Gateway]: with owned_proxy_process( - gateway, directory, overrides, config=config, remove_environment=remove_environment + gateway, directory, overrides, config=config, remove_environment=remove_environment, workers=workers ) as owned: yield owned.gateway @@ -76,11 +76,12 @@ def owned_proxy_process( *, config: Path | None = None, remove_environment: tuple[str, ...] = (), + workers: int = 1, ) -> Iterator[OwnedProxy]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) port: Final = reserve.getsockname()[1] - root: Final = Path(__file__).resolve().parents[3] + root: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3]) environment: Final = { **{name: value for name, value in os.environ.items() if name not in remove_environment}, "LITELLM_MASTER_KEY": gateway.key, @@ -104,7 +105,7 @@ def owned_proxy_process( "--port", str(port), "--num_workers", - "1", + str(workers), "--use_prisma_db_push", "--enforce_prisma_migration_check", ], diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index e9c50ea7966..eea539643d4 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -152,6 +152,31 @@ class Provider: ) return await chat_completions(request) + async def vector_store_search(self, request: Request) -> Response: + body: Final = JSON_OBJECT.validate_json(await request.body()) + self.observations.put(Observation(request.url.path, request.headers.get("authorization", ""), body)) + query: Final = body.get("query") + if not isinstance(query, str) or not query: + return JSONResponse({"error": {"message": "query is required"}}, status_code=400) + vector_store_id: Final = cast(str, request.path_params["vector_store_id"]) + return JSONResponse( + { + "object": "vector_store.search_results.page", + "search_query": query, + "data": [ + { + "file_id": f"file_{vector_store_id}", + "filename": "scripted.txt", + "score": 0.9, + "attributes": {}, + "content": [{"type": "text", "text": f"scripted context for {query}"}], + } + ], + "has_more": False, + "next_page": None, + } + ) + async def script(self, request: Request) -> Response: name: Final = cast(str, request.path_params["model"]) if request.method in {"DELETE", "GET"} and name not in self.scripts: @@ -338,6 +363,7 @@ class Provider: Route("/v1/completions", completions, methods=["POST"]), Route("/v1/embeddings", embeddings, methods=["POST"]), Route("/v1/moderations", moderations, methods=["POST"]), + Route("/vector_stores/{vector_store_id}/search", self.vector_store_search, methods=["POST"]), Route("/{path:path}", self.scripted, methods=["POST"]), Route("/{path:path}", self.scripted, methods=["GET"]), WebSocketRoute("/v1/realtime", self.realtime), diff --git a/tests/integration/_support/wire.py b/tests/integration/_support/wire.py index 0c6acfde96c..5052c34e021 100644 --- a/tests/integration/_support/wire.py +++ b/tests/integration/_support/wire.py @@ -2,11 +2,13 @@ from __future__ import annotations import ssl import threading +import time from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from queue import SimpleQueue +from types import MappingProxyType from typing import Final @@ -26,6 +28,8 @@ class Reply: chunks: tuple[bytes, ...] | None = None abort_after: int | None = None gate_after_first: threading.Event | None = None + pause_between_chunks: float = 0 + headers: Mapping[str, str] = MappingProxyType({}) @dataclass(frozen=True, slots=True) @@ -64,6 +68,8 @@ def wire_server(respond: Callable[[Request], Reply], tls: ssl.SSLContext | None reply = Reply(status=500) self.send_response(reply.status) self.send_header("content-type", reply.content_type) + for name, value in reply.headers.items(): + self.send_header(name, value) if reply.chunks is None: self.send_header("content-length", str(len(reply.body))) else: @@ -81,6 +87,8 @@ def wire_server(respond: Callable[[Request], Reply], tls: ssl.SSLContext | None self.wfile.flush() if index == 0 and reply.gate_after_first is not None: assert reply.gate_after_first.wait(timeout=5), "Stream barrier was never released" + if reply.pause_between_chunks and index + 1 < len(reply.chunks): + time.sleep(reply.pause_between_chunks) else: self.wfile.write(b"0\r\n\r\n") self.wfile.flush() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c54197c15e6..d5f0726f8b0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -81,17 +81,22 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: collected: Final = session.config.stash.get(COLLECTED, ()) reports: Final = tuple(report for report in session.config.stash[REPORTS] if report.nodeid in collected) passed: Final = tuple(report.nodeid for report in reports if report.when == "call" and report.passed) + skipped: Final = tuple(report.nodeid for report in reports if report.skipped) complete: Final = ( exitstatus == 0 and bool(collected) - and sorted(collected) == sorted(passed) - and all(report.passed for report in reports) + and sorted(collected) == sorted(passed + skipped) + and not any(report.failed for report in reports) ) output: Final = Path(destination) output.mkdir(parents=True, exist_ok=True) (output / "execution.json").write_text( json.dumps({ - "collected": collected, "passed": passed, "complete": complete, "exitstatus": exitstatus, + "collected": collected, + "passed": passed, + "skipped": skipped, + "complete": complete, + "exitstatus": exitstatus, "hypothesis_version": version("hypothesis"), "hypothesis_seed": session.config.getoption("hypothesis_seed"), "order_seed": session.config.getoption("integration_order_seed"), diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 257c4722aae..839178dc59a 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -290,6 +290,12 @@ "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" ], + "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": [ + "quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate" + ], + "tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": [ + "quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate" + ], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ "mgmt.key.update.project_detach_denied_to_restricted_actor" ], @@ -1783,6 +1789,225 @@ "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": [ "other.mcp.permissions.same_url_servers_enforce_discovery_and_execution" ], + "tests/integration/management/test_budget_updates.py::test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule": [ + "mgmt.budget.update.duration_change_recomputes_reset_at" + ], + "tests/integration/management/test_organization_budget_clear.py::test_patch_organization_update_with_null_tpm_limit_clears_it_and_keeps_sibling_limits": [ + "mgmt.organization.update.null_clears_budget_limit" + ], + "tests/integration/management/test_team_budget_duration_defaults.py::test_team_new_explicit_null_budget_duration_is_not_replaced_by_default": [ + "mgmt.team.new.explicit_null_budget_duration_overrides_default" + ], + "tests/integration/management/test_team_member_budget_cache.py::test_team_member_default_budget_lands_in_redis_after_first_member_call": [ + "mgmt.team_member_budget.default_budget_is_cached_in_redis_as_json" + ], + "tests/integration/observability/test_callback_delivery.py::test_streamed_responses_success_callback_carries_provider_apim_request_id": [ + "other.observability.callbacks.streamed_responses_events_carry_provider_response_headers" + ], + "tests/integration/observability/test_guardrail_effects.py::test_bedrock_passthrough_converse_guardrail_ignores_denied_term_in_tool_definition": [ + "other.observability.guardrails.bedrock_passthrough_converse_scans_only_caller_content" + ], + "tests/integration/pricing/test_configured_prices.py::test_cost_estimate_reports_configured_prices_for_model_absent_from_cost_map": [ + "quota_management.cost_estimate.configured_price.reported_for_model_absent_from_cost_map" + ], + "tests/integration/pricing/test_configured_prices.py::test_saving_echoed_model_info_does_not_freeze_cost_map_price_into_deployment": [ + "pricing.model_update.echoed_cost_map_price_is_not_persisted_as_override" + ], + "tests/integration/pricing/test_databricks_cache_pricing.py::test_databricks_cached_prompt_tokens_bill_at_cache_rates_not_input_rate": [ + "pricing.databricks.cached_prompt_tokens_bill_at_cache_rates" + ], + "tests/integration/pricing/test_ocr_page_pricing.py::test_ocr_annotation_pages_are_billed_at_annotation_cost_per_page": [ + "pricing.ocr.annotation_pages_billed_at_annotation_rate" + ], + "tests/integration/pricing/test_service_tier_pricing.py::test_ultrafast_service_tier_bills_ultrafast_rates_and_keeps_pricing_off_the_wire": [ + "quota_management.spend_tracking.service_tier_pricing.ultrafast_bills_ultrafast_rates" + ], + "tests/integration/spend/test_batch_completion_accounting.py::test_completed_batch_spend_row_records_reasoning_tokens_and_error_file_failures": [ + "quota_management.spend_tracking.batch_costs.reasoning_tokens_and_error_file_failures_recorded" + ], + "tests/integration/spend/test_batch_observability.py::test_batch_retrieval_row_sums_reasoning_tokens_and_counts_output_and_error_file_failures": [ + "spend.batches.retrieval_row_aggregates_reasoning_tokens_and_per_request_counts" + ], + "tests/integration/spend/test_batch_poll_starvation.py::test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling": [ + "quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed" + ], + "tests/integration/spend/test_cache_and_quota.py::test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completion": [ + "quota_management.budget.key.in_flight_count_tokens_reserves_nothing_so_completion_reaches_provider" + ], + "tests/integration/spend/test_cache_and_quota.py::test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_later_completion": [ + "quota_management.budget.key.count_tokens_reserves_nothing_so_completion_within_budget_succeeds" + ], + "tests/integration/spend/test_daily_rollup_retry.py::test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_activity_agree": [ + "spend.daily_rollup.failed_user_commit_is_retried_until_report_and_daily_activity_agree" + ], + "tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py::test_client_disconnect_mid_bedrock_messages_stream_still_bills_terminal_usage": [ + "spend.anthropic_messages_stream.client_disconnect_bills_terminal_bedrock_usage" + ], + "tests/integration/spend/test_failed_dispatch_tokens.py::test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_row": [ + "spend.failed_dispatch.failure_row_records_estimated_input_tokens" + ], + "tests/integration/spend/test_model_router_selected_model.py::test_model_router_alias_without_router_in_name_keeps_selected_model_in_response_and_spend_log": [ + "spend.model_router.selected_model_is_returned_and_persisted_for_plain_alias" + ], + "tests/integration/spend/test_org_budget_cli_session_token.py::test_cli_session_token_without_org_id_charges_and_caps_the_team_organization": [ + "quota_management.organization_budget.cli_session_token_without_org_id_charges_team_organization" + ], + "tests/integration/spend/test_passthrough_budget_reservation.py::test_repeated_gemini_passthrough_calls_stay_served_while_key_spend_is_below_max_budget": [ + "spend.budget_reservation.gemini_passthrough_success_releases_reservation_from_spend_counter" + ], + "tests/integration/spend/test_team_daily_activity_aggregated.py::test_aggregated_team_activity_reports_the_whole_range_team_spend_in_one_page": [ + "quota_management.spend_tracking.team_daily_activity_aggregated_reports_whole_range_team_spend" + ], + "tests/integration/spend/test_team_member_spend.py::test_member_added_without_any_budget_is_charged_on_its_membership_row": [ + "spend.team_member.member_without_budget_gets_membership_row_and_spend" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_is_listed_beside_db_store_and_survives_listing": [ + "mgmt.vector_store.list.keeps_config_store_beside_db_stores" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_config_store_refuses_new_update_and_delete": [ + "mgmt.vector_store.write.config_store_is_read_only" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_db_store_lifecycle_is_unchanged_beside_config_store": [ + "mgmt.vector_store.write.db_store_lifecycle_unchanged_beside_config_store" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_chat_with_config_store_searches_upstream_and_injects_context_after_listing": [ + "other.vector_store.chat.config_store_search_reaches_upstream_after_listing" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing": [ + "other.vector_store.search.config_store_passthrough_uses_yaml_credentials_after_listing" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_non_admin_key_access_to_config_store_follows_grants_after_admin_listing": [ + "authz.vector_store.list.non_admin_key_access_to_config_store_follows_grants" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_peer_process_keeps_config_store_and_sees_db_store_created_elsewhere": [ + "mgmt.vector_store.list.peer_process_keeps_config_store_and_sees_db_store" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_concurrent_burst_keeps_config_store_and_refuses_every_config_write": [ + "mgmt.vector_store.chaos.concurrent_burst_keeps_config_store_across_workers" + ], + "tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": [ + "mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers" + ], + "tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": [ + "providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment" + ], + "tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": [ + "providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry" + ], + "tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": [ + "other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body" + ], + "tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": [ + "other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider" + ], + "tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": [ + "providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization" + ], + "tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": [ + "other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages" + ], + "tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": [ + "other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking" + ], + "tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": [ + "other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level" + ], + "tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": [ + "other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format" + ], + "tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": [ + "providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort" + ], + "tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": [ + "providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support" + ], + "tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": [ + "other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types" + ], + "tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": [ + "providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items" + ], + "tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": [ + "other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long" + ], + "tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": [ + "providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text" + ], + "tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": [ + "other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type" + ], + "tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": [ + "providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned" + ], + "tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": [ + "other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text" + ], + "tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": [ + "other.provider_wire.dashscope.reasoning_effort_reaches_provider" + ], + "tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": [ + "other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log" + ], + "tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": [ + "other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base" + ], + "tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": [ + "other.provider_wire.deepseek.vision_image_content_list_reaches_provider" + ], + "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": [ + "other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name" + ], + "tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": [ + "other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name" + ], + "tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": [ + "providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire" + ], + "tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": [ + "other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields" + ], + "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": [ + "other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length" + ], + "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": [ + "other.provider_wire.tencent.thinking_reaches_provider_in_request_body" + ], + "tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": [ + "other.provider_wire.tencent.thinking_reaches_provider_in_request_body" + ], + "tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": [ + "other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use" + ], + "tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": [ + "other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results" + ], + "tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": [ + "other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters" + ], + "tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": [ + "other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers" + ], + "tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": [ + "other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload" + ], + "tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": [ + "other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream" + ], + "tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": [ + "other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost" + ], + "tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": [ + "other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend" + ], + "tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": [ + "other.streaming.responses_bridge.empty_choices_chunks_complete_stream" + ], + "tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": [ + "streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails" + ], + "tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": [ + "streaming.keepalive.sse_pings_fill_silent_time_to_first_token" + ], "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [ "other.observability.callbacks.raising_success_deployment_hook_keeps_response" ], @@ -1797,6 +2022,14 @@ ], "tests/integration/spend/test_passthrough_failure.py::test_config_pass_through_endpoint_failure_writes_a_spend_row": [ "quota_management.spend_tracking.pass_through_failure.failure_row_written" + ], + "tests/integration/management/test_user_updates_wedged_coordination_redis.py::test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged": [ + "mgmt.user.update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.user.bulk_update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.customer.update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.key.reset_spend.returns_promptly_with_wedged_coordination_redis", + "mgmt.auth_cache_invalidation.publish_parked_by_short_redis_wedge_lands_after_recovery", + "mgmt.auth_cache_invalidation.burst_with_worker_kill_keeps_serving_while_redis_wedged" ] }, "browser": { diff --git a/tests/integration/coordination_redis_proxy_config.yaml b/tests/integration/coordination_redis_proxy_config.yaml new file mode 100644 index 00000000000..30294c291bf --- /dev/null +++ b/tests/integration/coordination_redis_proxy_config.yaml @@ -0,0 +1,12 @@ +model_list: [] +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + disable_spend_logs: false + proxy_batch_write_at: 1 + coordination_redis: + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT +router_settings: + disable_cooldowns: true diff --git a/tests/integration/management/test_budget_updates.py b/tests/integration/management/test_budget_updates.py new file mode 100644 index 00000000000..25d525bcaaa --- /dev/null +++ b/tests/integration/management/test_budget_updates.py @@ -0,0 +1,29 @@ +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway, string_value +from tests.integration._support.database import read_rows + + +def _persisted_reset_at(budget_id: str) -> datetime: + rows: Final = read_rows( + 'SELECT budget_reset_at::text AS reset_at FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + assert len(rows) == 1, rows + reset_at: Final = datetime.fromisoformat(string_value(rows[0]["reset_at"])) + return reset_at if reset_at.tzinfo is not None else reset_at.replace(tzinfo=timezone.utc) + + +@pytest.mark.covers("mgmt.budget.update.duration_change_recomputes_reset_at") +def test_shortening_budget_duration_moves_reset_at_onto_the_new_schedule(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + budget_id: Final = scenario.budget(max_budget=10.0, budget_duration="10d") + ten_day_reset_at: Final = _persisted_reset_at(budget_id) + before: Final = datetime.now(timezone.utc) + response: Final = gateway.request("POST", "/budget/update", {"budget_id": budget_id, "budget_duration": "1d"}) + assert response.status_code == 200, response.text + updated: Final = _persisted_reset_at(budget_id) + assert updated < ten_day_reset_at, f"{updated} not before {ten_day_reset_at}" + assert before < updated <= before + timedelta(days=1, minutes=5), f"{updated} not within 1d of {before}" diff --git a/tests/integration/management/test_organization_budget_clear.py b/tests/integration/management/test_organization_budget_clear.py new file mode 100644 index 00000000000..8109dfd5a2d --- /dev/null +++ b/tests/integration/management/test_organization_budget_clear.py @@ -0,0 +1,59 @@ +import uuid +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway, object_value, string_value +from tests.integration._support.database import read_rows + + +def _budget_rows(budget_id: str) -> list[dict[str, object]]: + return read_rows( + 'SELECT tpm_limit, rpm_limit, max_budget FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + + +@pytest.mark.covers("mgmt.organization.update.null_clears_budget_limit") +def test_patch_organization_update_with_null_tpm_limit_clears_it_and_keeps_sibling_limits(gateway: Gateway) -> None: + created: Final = gateway.post( + "/organization/new", + { + "organization_alias": f"integration-{uuid.uuid4().hex}", + "tpm_limit": 4000, + "rpm_limit": 40, + "max_budget": 12.5, + }, + ) + organization_id: Final = string_value(created["organization_id"]) + budget_id: Final = string_value(created["budget_id"]) + try: + assert _budget_rows(budget_id) == [{"tpm_limit": 4000, "rpm_limit": 40, "max_budget": 12.5}] + updated: Final = gateway.request( + "PATCH", "/organization/update", {"organization_id": organization_id, "tpm_limit": None} + ) + assert updated.status_code == 200, updated.text + updated_budget: Final = object_value(object_value(updated.json())["litellm_budget_table"]) + assert (updated_budget["tpm_limit"], updated_budget["rpm_limit"], updated_budget["max_budget"]) == ( + None, + 40, + 12.5, + ), updated.text + assert _budget_rows(budget_id) == [{"tpm_limit": None, "rpm_limit": 40, "max_budget": 12.5}] + info: Final = gateway.request("GET", "/organization/info", params={"organization_id": organization_id}) + assert info.status_code == 200, info.text + info_budget: Final = object_value(object_value(info.json())["litellm_budget_table"]) + assert (info_budget["tpm_limit"], info_budget["rpm_limit"], info_budget["max_budget"]) == ( + None, + 40, + 12.5, + ), info.text + finally: + deleted: Final = gateway.request("DELETE", "/organization/delete", {"organization_ids": [organization_id]}) + assert deleted.status_code == 200, deleted.text + gateway.post("/budget/delete", {"id": budget_id}) + assert ( + read_rows( + 'SELECT organization_id FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,) + ) + == [] + ) diff --git a/tests/integration/management/test_team_budget_duration_defaults.py b/tests/integration/management/test_team_budget_duration_defaults.py new file mode 100644 index 00000000000..fd459f6a7ef --- /dev/null +++ b/tests/integration/management/test_team_budget_duration_defaults.py @@ -0,0 +1,61 @@ +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy + + +def _budget_row(team_id: str) -> dict[str, JsonValue]: + rows: Final = read_rows( + 'SELECT max_budget, budget_duration, budget_reset_at::text FROM "LiteLLM_TeamTable" WHERE team_id = %s', + (team_id,), + ) + assert len(rows) == 1, rows + return rows[0] + + +@pytest.mark.covers("mgmt.team.new.explicit_null_budget_duration_overrides_default") +def test_team_new_explicit_null_budget_duration_is_not_replaced_by_default(gateway: Gateway, tmp_path: Path) -> None: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"]["default_team_params"] = {"budget_duration": "30d"} + path: Final = tmp_path / "team-defaults.yaml" + path.write_text(yaml.safe_dump(config)) + with ( + owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=path) as candidate, + candidate.scenario() as scenario, + ): + never_resetting: Final = candidate.request( + "POST", + "/team/new", + {"team_alias": f"integration-{uuid.uuid4().hex}", "max_budget": 500, "budget_duration": None}, + ) + assert never_resetting.status_code == 200, never_resetting.text + never_resetting_id: Final = string_value(never_resetting.json()["team_id"]) + scenario.cleanups.callback(scenario.delete_team, never_resetting_id) + assert never_resetting.json()["max_budget"] == 500.0, never_resetting.text + assert never_resetting.json()["budget_duration"] is None, never_resetting.text + assert never_resetting.json()["budget_reset_at"] is None, never_resetting.text + assert _budget_row(never_resetting_id) == { + "max_budget": 500.0, + "budget_duration": None, + "budget_reset_at": None, + } + + inheriting: Final = candidate.request( + "POST", "/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}", "max_budget": 500} + ) + assert inheriting.status_code == 200, inheriting.text + inheriting_id: Final = string_value(inheriting.json()["team_id"]) + scenario.cleanups.callback(scenario.delete_team, inheriting_id) + assert inheriting.json()["budget_duration"] == "30d", inheriting.text + assert inheriting.json()["budget_reset_at"] is not None, inheriting.text + inheriting_row: Final = _budget_row(inheriting_id) + assert inheriting_row["max_budget"] == 500.0, inheriting_row + assert inheriting_row["budget_duration"] == "30d", inheriting_row + assert inheriting_row["budget_reset_at"] is not None, inheriting_row diff --git a/tests/integration/management/test_team_member_budget_cache.py b/tests/integration/management/test_team_member_budget_cache.py new file mode 100644 index 00000000000..9c4181915c0 --- /dev/null +++ b/tests/integration/management/test_team_member_budget_cache.py @@ -0,0 +1,40 @@ +import os +from typing import Final + +import pytest +from pydantic import JsonValue, TypeAdapter +from redis import Redis + +from tests.integration._support.client import Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows + +_CACHED_BUDGET: Final = TypeAdapter(dict[str, JsonValue]) + + +@pytest.mark.covers("mgmt.team_member_budget.default_budget_is_cached_in_redis_as_json") +def test_team_member_default_budget_lands_in_redis_after_first_member_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user() + team: Final = scenario.team(team_member_budget=25) + key: Final = scenario.key(team_id=team, user_id=user, models=[model]) + teams: Final = read_rows('SELECT metadata FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) + assert len(teams) == 1, teams + budget_id: Final = string_value(object_value(teams[0]["metadata"])["team_member_budget_id"]) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "member budget cache"}]}, + key=key, + ) + assert response.status_code == 200, response.text + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + cached: Final = eventually( + lambda: cache.get(f"team_member_default_budget:{budget_id}"), + lambda value: value is not None, + seconds=10, + ) + assert isinstance(cached, bytes), cached + budget: Final = _CACHED_BUDGET.validate_json(cached) + assert budget["budget_id"] == budget_id, cached + assert budget["max_budget"] == 25, cached diff --git a/tests/integration/management/test_user_updates_wedged_coordination_redis.py b/tests/integration/management/test_user_updates_wedged_coordination_redis.py new file mode 100644 index 00000000000..d8c84a70778 --- /dev/null +++ b/tests/integration/management/test_user_updates_wedged_coordination_redis.py @@ -0,0 +1,379 @@ +import os +import signal +import time +import uuid +from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import psutil +import psycopg +import pytest +from psycopg import sql +from pydantic import JsonValue +from redis import Redis +from redis.client import PubSub + +from tests.integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from tests.integration._support.process import owned_proxy +from tests.integration._support.redis_process import owned_redis + +_USERS: Final = 60 +_BURST: Final = 30 +_HANDLER_BUDGET_SECONDS: Final = 0.75 +_BULK_BUDGET_SECONDS: Final = 2.0 +_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" + + +def _timed_post(candidate: Gateway, path: str, body: Mapping[str, JsonValue], timeout: float = 15) -> float: + started: Final = time.monotonic() + response: Final = candidate.client.request( + "POST", + path, + json=body, + headers={"Authorization": f"Bearer {candidate.key}"}, + timeout=timeout, + ) + elapsed: Final = time.monotonic() - started + assert response.status_code == 200, f"POST {path}: {response.status_code} {response.text} after {elapsed:.3f}s" + return elapsed + + +def _received(pubsub: PubSub) -> tuple[dict[str, JsonValue], ...]: + messages: list[dict[str, JsonValue]] = [] + while True: + message = pubsub.get_message(ignore_subscribe_messages=True, timeout=0) + if message is None: + return tuple(messages) + data = message.get("data") + if isinstance(data, (bytes, str)): + messages.append(JSON_OBJECT.validate_json(data)) + + +def _worker_pid(port: int) -> int: + for process in psutil.process_iter(): + parent = process.parent() + if parent is None: + continue + try: + cmdline = parent.cmdline() + own_cmdline = process.cmdline() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if ( + "integration._support.proxy" in cmdline + and "--port" in cmdline + and str(port) in cmdline + and not any("prisma" in part for part in own_cmdline) + ): + return process.pid + raise AssertionError(f"no uvicorn worker found under the owned proxy on port {port}") + + +def _burst_call( + index: int, users: tuple[str, ...], key: str, team_id: str, customer_id: str +) -> tuple[str, dict[str, JsonValue]]: + match index % 5: + case 0: + return "/user/update", {"user_id": users[index], "max_budget": 200.0 + index} + case 1: + return "/user/update", {"user_id": users[index], "tpm_limit": 1000 + index} + case 2: + return "/key/update", {"key": key, "max_budget": 7.0 + index} + case 3: + return "/team/update", {"team_id": team_id, "max_budget": 7.0 + index} + case _: + return "/customer/update", {"user_id": customer_id, "max_budget": 7.0 + index} + + +@pytest.mark.timeout(240) +@pytest.mark.covers( + "mgmt.user.update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.user.bulk_update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.customer.update.budget_change_returns_promptly_with_wedged_coordination_redis", + "mgmt.key.reset_spend.returns_promptly_with_wedged_coordination_redis", + "mgmt.auth_cache_invalidation.publish_parked_by_short_redis_wedge_lands_after_recovery", + "mgmt.auth_cache_invalidation.burst_with_worker_kill_keeps_serving_while_redis_wedged", +) +def test_user_budget_updates_return_promptly_while_coordination_redis_is_wedged( + gateway: Gateway, tmp_path: Path, record_property: Callable[[str, object], None] +) -> None: + original: Final = os.environ["DATABASE_URL"] + identity: Final = "integration_wedged_redis_" + uuid.uuid4().hex + parsed: Final = urlsplit(original) + database_url: Final = urlunsplit((parsed.scheme, parsed.netloc, "/" + identity, "", "")) + timings: dict[str, float] = {} + with psycopg.connect(original, autocommit=True) as admin: + admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(identity))) + try: + results_dir: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(tmp_path))) + prior_logs: Final = frozenset(results_dir.glob("owned-proxy-*.log")) + with ( + owned_redis(tmp_path) as coordination, + owned_proxy( + gateway, + tmp_path, + { + "DATABASE_URL": database_url, + "REDIS_HOST": coordination.host, + "REDIS_PORT": str(coordination.port), + }, + config=Path("tests/integration/coordination_redis_proxy_config.yaml"), + workers=2, + ) as candidate, + Redis(host=coordination.host, port=coordination.port, socket_timeout=1) as subscriber_client, + ): + pubsub: Final = subscriber_client.pubsub() + pubsub.subscribe(_CHANNEL) + received: list[dict[str, JsonValue]] = [] + + def drained() -> tuple[dict[str, JsonValue], ...]: + received.extend(_received(pubsub)) + return tuple(received) + + eventually( + lambda: subscriber_client.pubsub_numsub(_CHANNEL)[0][1], + lambda count: count >= 3, + seconds=15, + ) + users: Final = tuple(f"{identity}_u{index}" for index in range(_USERS)) + for user_id in users: + candidate.post("/user/new", {"user_id": user_id, "auto_create_key": False, "max_budget": 10.0}) + key: Final = string_value( + candidate.post("/key/generate", {"user_id": users[0], "max_budget": 5.0})["key"] + ) + team_id: Final = string_value( + candidate.post("/team/new", {"team_alias": identity, "max_budget": 5.0})["team_id"] + ) + customer_id: Final = identity + "_cust" + candidate.post("/customer/new", {"user_id": customer_id, "max_budget": 5.0}) + drained() + timings["h1_healthy"] = _timed_post( + candidate, "/user/update", {"user_id": users[0], "max_budget": 11.0} + ) + assert timings["h1_healthy"] < _HANDLER_BUDGET_SECONDS, ( + f"healthy /user/update took {timings['h1_healthy']:.3f}s" + ) + eventually( + drained, + lambda messages: any(message.get("cache_key") == users[0] for message in messages), + seconds=10, + ) + timings["h2_healthy_control"] = _timed_post( + candidate, "/user/update", {"user_id": users[0], "tpm_limit": 1000} + ) + assert timings["h2_healthy_control"] < _HANDLER_BUDGET_SECONDS, ( + f"healthy control update took {timings['h2_healthy_control']:.3f}s" + ) + coordination.signal(signal.SIGSTOP) + try: + timings["s2_wedged_control"] = _timed_post( + candidate, "/user/update", {"user_id": users[0], "tpm_limit": 1000} + ) + assert timings["s2_wedged_control"] < _HANDLER_BUDGET_SECONDS, ( + f"control update without a cache-relevant field took {timings['s2_wedged_control']:.3f}s" + ) + timings["s1_user_update"] = _timed_post( + candidate, "/user/update", {"user_id": users[0], "max_budget": 98.0} + ) + assert timings["s1_user_update"] < _HANDLER_BUDGET_SECONDS, ( + f"/user/update with max_budget took {timings['s1_user_update']:.3f}s " + "with a wedged coordination Redis" + ) + timings["s3_bulk_update"] = _timed_post( + candidate, "/user/bulk_update", {"all_users": True, "user_updates": {"max_budget": 79.0}} + ) + assert timings["s3_bulk_update"] < _BULK_BUDGET_SECONDS, ( + f"/user/bulk_update over {_USERS} users took {timings['s3_bulk_update']:.3f}s " + "with a wedged coordination Redis" + ) + timings["s4_key_update"] = _timed_post( + candidate, "/key/update", {"key": key, "max_budget": 6.0}, timeout=60 + ) + assert timings["s4_key_update"] < 30, ( + f"/key/update hung for {timings['s4_key_update']:.3f}s with a wedged coordination Redis" + ) + timings["s5_team_update"] = _timed_post( + candidate, "/team/update", {"team_id": team_id, "max_budget": 6.0}, timeout=60 + ) + assert timings["s5_team_update"] < 30, ( + f"/team/update hung for {timings['s5_team_update']:.3f}s with a wedged coordination Redis" + ) + timings["s6_customer_update"] = _timed_post( + candidate, "/customer/update", {"user_id": customer_id, "max_budget": 6.0} + ) + assert timings["s6_customer_update"] < _HANDLER_BUDGET_SECONDS, ( + f"/customer/update took {timings['s6_customer_update']:.3f}s with a wedged coordination Redis" + ) + timings["s7_reset_spend"] = _timed_post(candidate, f"/key/{key}/reset_spend", {"reset_to": 0}) + assert timings["s7_reset_spend"] < _HANDLER_BUDGET_SECONDS, ( + f"/key//reset_spend took {timings['s7_reset_spend']:.3f}s with a wedged coordination Redis" + ) + missing_started: Final = time.monotonic() + missing: Final = candidate.request( + "POST", "/user/update", {"user_id": users[0], "max_budget": "not-a-number"} + ) + timings["s8_invalid_body"] = time.monotonic() - missing_started + assert missing.status_code // 100 == 4, ( + f"/user/update with an invalid body returned {missing.status_code} " + f"in {timings['s8_invalid_body']:.3f}s" + ) + assert timings["s8_invalid_body"] < _HANDLER_BUDGET_SECONDS, ( + f"/user/update with an invalid body took {timings['s8_invalid_body']:.3f}s" + ) + + def burst_request(path: str, body: Mapping[str, JsonValue]) -> tuple[object, float]: + started: Final = time.monotonic() + try: + response: Final = candidate.client.request( + "POST", + path, + json=body, + headers={"Authorization": f"Bearer {candidate.key}"}, + timeout=60, + ) + return response.status_code, time.monotonic() - started + except Exception as error: # noqa: BLE001 # the killed worker drops in-flight requests + return error, time.monotonic() - started + + port: Final = candidate.client.base_url.port + assert port is not None, f"owned proxy client has no port: {candidate.client.base_url}" + with ThreadPoolExecutor(_BURST) as pool: + futures: Final = [ + pool.submit( + burst_request, + *_burst_call(i, users, key, team_id, customer_id), + ) + for i in range(_BURST) + ] + os.kill(_worker_pid(port), signal.SIGKILL) + results: Final = [future.result() for future in futures] + responses: Final = [(status, elapsed) for status, elapsed in results if isinstance(status, int)] + failures: Final = [status for status, _elapsed in responses if status != 200] + assert not failures, f"burst responses that were not 200: {failures}" + transport_errors: Final = [status for status, _elapsed in results if not isinstance(status, int)] + assert len(transport_errors) <= 3, ( + f"{len(transport_errors)} requests raised transport errors: {transport_errors!r}" + ) + elapsed_sorted: Final = sorted( + elapsed for i, (status, elapsed) in enumerate(results) if i % 5 in (0, 1, 4) and status == 200 + ) + timings["c1_burst_p95"] = elapsed_sorted[int(len(elapsed_sorted) * 0.95) - 1] + assert timings["c1_burst_p95"] < _HANDLER_BUDGET_SECONDS, ( + f"burst p95 {timings['c1_burst_p95']:.3f}s" + ) + eventually( + lambda: candidate.request("GET", "/health/liveliness").status_code, + lambda status: status == 200, + seconds=15, + ) + timings["c1_survivor"] = _timed_post( + candidate, "/user/update", {"user_id": users[0], "tpm_limit": 2000} + ) + assert timings["c1_survivor"] < _HANDLER_BUDGET_SECONDS, ( + f"control update on the surviving worker took {timings['c1_survivor']:.3f}s" + ) + finally: + coordination.signal(signal.SIGCONT) + wedged_keys: Final = {users[i] for i in range(_BURST) if i % 5 == 0 and i != 0} | {f"team_id:{team_id}"} + + def proxy_log() -> str: + return "".join( + path.read_text() for path in results_dir.glob("owned-proxy-*.log") if path not in prior_logs + ) + + team_wedged_key: Final = f"team_id:{team_id}" + eventually( + proxy_log, + lambda text: ( + all( + f"publish for {wedged_key} failed" in text + for wedged_key in wedged_keys + if wedged_key != team_wedged_key + ) + and ( + f"publish for {team_wedged_key} failed" in text + or f"internal usage cache entry {team_wedged_key}" in text + ) + ), + seconds=45, + ) + marker: Final = len(received) + drained() + recovered_keys: Final = {str(message.get("cache_key")) for message in received[marker:]} + assert recovered_keys.isdisjoint(wedged_keys), ( + f"wedged publishes unexpectedly landed after recovery: {sorted(recovered_keys & wedged_keys)}" + ) + coordination.signal(signal.SIGSTOP) + try: + timings["r1b_short_wedge_a"] = _timed_post( + candidate, "/user/update", {"user_id": users[4], "max_budget": 15.0} + ) + assert timings["r1b_short_wedge_a"] < _HANDLER_BUDGET_SECONDS, ( + f"/user/update inside a short wedge took {timings['r1b_short_wedge_a']:.3f}s" + ) + timings["r1b_short_wedge_b"] = _timed_post( + candidate, "/user/update", {"user_id": users[5], "max_budget": 16.0} + ) + assert timings["r1b_short_wedge_b"] < _HANDLER_BUDGET_SECONDS, ( + f"/user/update inside a short wedge took {timings['r1b_short_wedge_b']:.3f}s" + ) + finally: + coordination.signal(signal.SIGCONT) + eventually( + drained, + lambda messages: {str(message.get("cache_key")) for message in messages} >= {users[4], users[5]}, + seconds=10, + ) + timings["r2_resumed"] = _timed_post( + candidate, "/user/update", {"user_id": users[1], "max_budget": 12.0} + ) + assert timings["r2_resumed"] < _HANDLER_BUDGET_SECONDS, ( + f"post-recovery /user/update took {timings['r2_resumed']:.3f}s" + ) + eventually( + drained, + lambda messages: any(message.get("cache_key") == users[1] for message in messages), + seconds=10, + ) + coordination.stop() + timings["f1_refused"] = _timed_post( + candidate, "/user/update", {"user_id": users[2], "max_budget": 13.0} + ) + assert timings["f1_refused"] < _HANDLER_BUDGET_SECONDS, ( + f"/user/update with refused coordination Redis took {timings['f1_refused']:.3f}s" + ) + coordination.start() + restarted_pubsub: Final = subscriber_client.pubsub() + restarted_pubsub.subscribe(_CHANNEL) + restarted_received: list[dict[str, JsonValue]] = [] + + def drained_after_restart() -> tuple[dict[str, JsonValue], ...]: + restarted_received.extend(_received(restarted_pubsub)) + return tuple(restarted_received) + + eventually( + lambda: subscriber_client.pubsub_numsub(_CHANNEL)[0][1], + lambda count: count >= 3, + seconds=30, + ) + timings["f2_restarted"] = _timed_post( + candidate, "/user/update", {"user_id": users[3], "max_budget": 14.0} + ) + assert timings["f2_restarted"] < _HANDLER_BUDGET_SECONDS, ( + f"/user/update after Redis restart took {timings['f2_restarted']:.3f}s" + ) + eventually( + drained_after_restart, + lambda messages: any(message.get("cache_key") == users[3] for message in messages), + seconds=10, + ) + info_last: Final = object_value(candidate.get("/user/info", {"user_id": users[-1]})["user_info"]) + assert info_last["max_budget"] == 79.0, info_last + info_user3: Final = object_value(candidate.get("/user/info", {"user_id": users[3]})["user_info"]) + assert info_user3["max_budget"] == 14.0, info_user3 + finally: + admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(identity))) + record_property("cell_elapsed_seconds", timings) diff --git a/tests/integration/management/test_vector_store_config_ownership.py b/tests/integration/management/test_vector_store_config_ownership.py new file mode 100644 index 00000000000..e1e7ac42472 --- /dev/null +++ b/tests/integration/management/test_vector_store_config_ownership.py @@ -0,0 +1,351 @@ +import os +import uuid +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import httpx +import psycopg +import pytest +from psycopg import sql +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, eventually, object_value +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy +from tests.integration._support.redis_process import owned_redis + +CONFIG_STORE_ID: Final = "vs_integration_config_store" +CONFIG_STORE_NAME: Final = "integration-config-store" +SEARCH_PATH: Final = f"/vector_stores/{CONFIG_STORE_ID}/search" +PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" + + +def listed_rows(response: httpx.Response) -> tuple[dict[str, JsonValue], ...]: + rows: Final = object_value(response.json()).get("data") + assert isinstance(rows, list), response.text + return tuple(object_value(row) for row in rows) + + +def listed_store(gateway: Gateway, vector_store_id: str, *, key: str | None = None) -> dict[str, JsonValue]: + listed: Final = gateway.request("GET", "/vector_store/list", key=key) + assert listed.status_code == 200, listed.text + matches: Final = tuple(row for row in listed_rows(listed) if row["vector_store_id"] == vector_store_id) + assert len(matches) == 1, f"{vector_store_id} appears {len(matches)} times in {listed.text}" + return matches[0] + + +def listed_ids(gateway: Gateway) -> tuple[str, ...]: + rows: Final = gateway.get("/vector_store/list")["data"] + assert isinstance(rows, list) + return tuple(str(object_value(row)["vector_store_id"]) for row in rows) + + +def config_store_info(gateway: Gateway) -> dict[str, JsonValue]: + return object_value(gateway.post("/vector_store/info", {"vector_store_id": CONFIG_STORE_ID})["vector_store"]) + + +def store_rows(vector_store_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT vector_store_id, vector_store_name FROM "LiteLLM_ManagedVectorStoresTable" WHERE vector_store_id = %s', + (vector_store_id,), + ) + + +def assert_config_write_refused(gateway: Gateway) -> None: + for path, body in ( + ("/vector_store/update", {"vector_store_id": CONFIG_STORE_ID, "vector_store_name": "renamed"}), + ("/vector_store/delete", {"vector_store_id": CONFIG_STORE_ID}), + ("/vector_store/new", {"vector_store_id": CONFIG_STORE_ID, "custom_llm_provider": "openai"}), + ): + refused = gateway.request("POST", path, body) + assert refused.status_code == 400, f"{path}: {refused.status_code} {refused.text}" + error = object_value(object_value(refused.json())["detail"]) + assert error["vector_store_id"] == CONFIG_STORE_ID, refused.text + assert "config file" in str(error["error"]), refused.text + + +def burst_list(gateway: Gateway) -> tuple[int, str]: + response: Final = gateway.request("GET", "/vector_store/list") + if response.status_code != 200: + return response.status_code, response.text + ids: Final = tuple(str(row["vector_store_id"]) for row in listed_rows(response)) + return response.status_code, "config" if CONFIG_STORE_ID in ids else response.text + + +def burst_post(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple[int, str]: + response: Final = gateway.request("POST", path, body) + return response.status_code, response.text + + +def upstream_requests(upstream: httpx.Client, marker: str) -> list[dict[str, JsonValue]]: + observed: Final = upstream.get("/__observations") + observed.raise_for_status() + requests: Final = object_value(observed.json())["requests"] + assert isinstance(requests, list), observed.text + return [object_value(value) for value in requests if marker in str(object_value(value)["body"])] + + +@pytest.mark.covers("mgmt.vector_store.list.keeps_config_store_beside_db_stores") +def test_config_store_is_listed_beside_db_store_and_survives_listing(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + db_store_id: Final = f"vs_db_{uuid.uuid4().hex}" + gateway.post("/vector_store/new", {"vector_store_id": db_store_id, "custom_llm_provider": "openai"}) + scenario.cleanups.callback(gateway.post, "/vector_store/delete", {"vector_store_id": db_store_id}) + before: Final = config_store_info(gateway) + assert before["vector_store_id"] == CONFIG_STORE_ID, before + + config_row: Final = listed_store(gateway, CONFIG_STORE_ID) + assert config_row["is_config"] is True, config_row + assert config_row["vector_store_name"] == CONFIG_STORE_NAME, config_row + assert object_value(config_row["litellm_params"])["api_key"] != "integration-provider-key", config_row + db_row: Final = listed_store(gateway, db_store_id) + assert db_row["is_config"] is False, db_row + + after: Final = config_store_info(gateway) + assert after["vector_store_id"] == CONFIG_STORE_ID, after + assert after["is_config"] is True, after + assert after["vector_store_description"] == "declared in tests/integration/proxy_config.yaml", after + assert store_rows(CONFIG_STORE_ID) == [], "config store must not need a database row" + assert listed_store(gateway, CONFIG_STORE_ID)["is_config"] is True + + +@pytest.mark.covers("mgmt.vector_store.write.config_store_is_read_only") +def test_config_store_refuses_new_update_and_delete(gateway: Gateway) -> None: + assert_config_write_refused(gateway) + row: Final = listed_store(gateway, CONFIG_STORE_ID) + assert row["vector_store_name"] == CONFIG_STORE_NAME, row + assert row["is_config"] is True, row + assert config_store_info(gateway)["vector_store_name"] == CONFIG_STORE_NAME + + +@pytest.mark.covers("mgmt.vector_store.write.db_store_lifecycle_unchanged_beside_config_store") +def test_db_store_lifecycle_is_unchanged_beside_config_store(gateway: Gateway) -> None: + incomplete: Final = gateway.request("POST", "/vector_store/new", {"custom_llm_provider": "openai"}) + assert incomplete.status_code == 400, incomplete.text + db_store_id: Final = f"vs_db_{uuid.uuid4().hex}" + created: Final = gateway.request( + "POST", + "/vector_store/new", + {"vector_store_id": db_store_id, "custom_llm_provider": "openai", "vector_store_name": "first"}, + ) + assert created.status_code == 200, created.text + assert store_rows(db_store_id) == [{"vector_store_id": db_store_id, "vector_store_name": "first"}] + updated: Final = gateway.post( + "/vector_store/update", {"vector_store_id": db_store_id, "vector_store_name": "second"} + ) + assert object_value(updated["vector_store"])["vector_store_name"] == "second", updated + assert store_rows(db_store_id) == [{"vector_store_id": db_store_id, "vector_store_name": "second"}] + row: Final = listed_store(gateway, db_store_id) + assert row["vector_store_name"] == "second" and row["is_config"] is False, row + info: Final = object_value(gateway.post("/vector_store/info", {"vector_store_id": db_store_id})["vector_store"]) + assert info["vector_store_name"] == "second" and info["is_config"] is False, info + gateway.post("/vector_store/delete", {"vector_store_id": db_store_id}) + assert store_rows(db_store_id) == [] + assert db_store_id not in listed_ids(gateway) + assert CONFIG_STORE_ID in listed_ids(gateway) + missing: Final = gateway.request("POST", "/vector_store/info", {"vector_store_id": db_store_id}) + assert missing.status_code == 404, missing.text + + +@pytest.mark.covers("other.vector_store.chat.config_store_search_reaches_upstream_after_listing") +def test_chat_with_config_store_searches_upstream_and_injects_context_after_listing(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + marker: Final = f"lit6337 {uuid.uuid4().hex}" + assert CONFIG_STORE_ID in listed_ids(gateway) + upstream.get("/__observations").raise_for_status() + completion: Final = gateway.post( + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker}], "vector_store_ids": [CONFIG_STORE_ID]}, + ) + assert object_value(completion["usage"])["total_tokens"] == 40, completion + requests: Final = upstream_requests(upstream, marker) + searches: Final = [value for value in requests if value["path"] == SEARCH_PATH] + assert len(searches) == 1, requests + assert object_value(searches[0]["body"])["query"] == marker, searches + assert searches[0]["authorization"] == "Bearer integration-provider-key", searches + chats: Final = [value for value in requests if value["path"] == "/v1/chat/completions"] + assert len(chats) == 1, requests + messages: Final = object_value(chats[0]["body"])["messages"] + assert isinstance(messages, list), chats + contents: Final = tuple(str(object_value(message)["content"]) for message in messages) + assert contents == (f"Context:\n\nscripted context for {marker}\n\n", marker), contents + + +@pytest.mark.covers("other.vector_store.search.config_store_passthrough_uses_yaml_credentials_after_listing") +def test_passthrough_search_on_config_store_uses_yaml_credentials_after_listing(gateway: Gateway) -> None: + with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: + marker: Final = f"lit6337 passthrough {uuid.uuid4().hex}" + assert CONFIG_STORE_ID in listed_ids(gateway) + upstream.get("/__observations").raise_for_status() + searched: Final = gateway.request("POST", f"/v1/vector_stores/{CONFIG_STORE_ID}/search", {"query": marker}) + assert searched.status_code == 200, searched.text + data: Final = listed_rows(searched) + assert len(data) == 1, searched.text + content: Final = data[0]["content"] + assert isinstance(content, list), searched.text + assert object_value(content[0])["text"] == f"scripted context for {marker}", searched.text + requests: Final = upstream_requests(upstream, marker) + assert [value["path"] for value in requests] == [SEARCH_PATH], requests + assert requests[0]["authorization"] == "Bearer integration-provider-key", requests + + +@pytest.mark.covers("authz.vector_store.list.non_admin_key_access_to_config_store_follows_grants") +def test_non_admin_key_access_to_config_store_follows_grants_after_admin_listing(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + granted: Final = scenario.key(object_permission={"vector_stores": [CONFIG_STORE_ID]}) + plain: Final = scenario.key() + assert CONFIG_STORE_ID in listed_ids(gateway) + row: Final = listed_store(gateway, CONFIG_STORE_ID, key=granted) + assert row["is_config"] is True and row["vector_store_name"] == CONFIG_STORE_NAME, row + unlisted: Final = gateway.request("GET", "/vector_store/list", key=plain) + assert unlisted.status_code == 200, unlisted.text + assert CONFIG_STORE_ID not in {value["vector_store_id"] for value in listed_rows(unlisted)}, unlisted.text + for key in (granted, plain): + info = gateway.request("POST", "/vector_store/info", {"vector_store_id": CONFIG_STORE_ID}, key=key) + assert info.status_code == 200, info.text + assert object_value(object_value(info.json())["vector_store"])["is_config"] is True, info.text + forbidden: Final = gateway.request( + "POST", "/vector_store/delete", {"vector_store_id": CONFIG_STORE_ID}, key=granted + ) + assert forbidden.status_code in {400, 401, 403}, forbidden.text + assert CONFIG_STORE_ID in listed_ids(gateway) + + +@pytest.mark.covers("mgmt.vector_store.list.peer_process_keeps_config_store_and_sees_db_store") +def test_peer_process_keeps_config_store_and_sees_db_store_created_elsewhere(gateway: Gateway, peer: Gateway) -> None: + with gateway.scenario() as scenario: + db_store_id: Final = f"vs_db_{uuid.uuid4().hex}" + gateway.post("/vector_store/new", {"vector_store_id": db_store_id, "custom_llm_provider": "openai"}) + scenario.cleanups.callback(gateway.request, "POST", "/vector_store/delete", {"vector_store_id": db_store_id}) + for side in (gateway, peer, gateway, peer): + assert listed_store(side, CONFIG_STORE_ID)["is_config"] is True + assert listed_store(side, db_store_id)["is_config"] is False + assert config_store_info(side)["is_config"] is True + assert_config_write_refused(side) + gateway.post("/vector_store/delete", {"vector_store_id": db_store_id}) + assert db_store_id not in listed_ids(peer) + assert CONFIG_STORE_ID in listed_ids(peer) + + +@pytest.mark.covers("mgmt.vector_store.chaos.concurrent_burst_keeps_config_store_across_workers") +def test_concurrent_burst_keeps_config_store_and_refuses_every_config_write(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + db_store_ids: Final = tuple(f"vs_db_{uuid.uuid4().hex}" for _ in range(6)) + for db_store_id in db_store_ids: + scenario.cleanups.callback( + gateway.request, "POST", "/vector_store/delete", {"vector_store_id": db_store_id} + ) + + def act(index: int) -> tuple[str, int, str]: + match index % 5: + case 0: + return ("list", *burst_list(gateway)) + case 1: + return ("info", *burst_post(gateway, "/vector_store/info", {"vector_store_id": CONFIG_STORE_ID})) + case 2: + return ( + "config-update", + *burst_post( + gateway, + "/vector_store/update", + {"vector_store_id": CONFIG_STORE_ID, "vector_store_name": str(index)}, + ), + ) + case 3: + return ( + "db-new", + *burst_post( + gateway, + "/vector_store/new", + { + "vector_store_id": db_store_ids[index % len(db_store_ids)], + "custom_llm_provider": "openai", + }, + ), + ) + case _: + return ( + "chat", + *burst_post( + gateway, + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"burst {index}"}], + "vector_store_ids": [CONFIG_STORE_ID], + }, + ), + ) + + with ThreadPoolExecutor(max_workers=10) as pool: + outcomes: Final = tuple(pool.map(act, range(30))) + expected: Final = {"list": 200, "info": 200, "config-update": 400, "db-new": 200, "chat": 200} + assert [(kind, status) for kind, status, _ in outcomes] == [ + (kind, expected[kind]) for kind, _, _ in outcomes + ], outcomes + assert all(detail == "config" for kind, _, detail in outcomes if kind == "list"), outcomes + assert listed_store(gateway, CONFIG_STORE_ID)["vector_store_name"] == CONFIG_STORE_NAME + assert config_store_info(gateway)["vector_store_name"] == CONFIG_STORE_NAME + assert store_rows(CONFIG_STORE_ID) == [] + assert all(len(store_rows(db_store_id)) == 1 for db_store_id in db_store_ids), "each DB store exactly once" + + +@pytest.mark.timeout(180) +@pytest.mark.covers("mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers") +def test_redis_outage_keeps_config_store_served_and_recovers( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + original: Final = os.environ["DATABASE_URL"] + identity: Final = "integration_vs_outage_" + uuid.uuid4().hex + parsed: Final = urlsplit(original) + database_url: Final = urlunsplit((parsed.scheme, parsed.netloc, "/" + identity, "", "")) + with psycopg.connect(original, autocommit=True) as admin: + admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(identity))) + try: + with owned_redis(tmp_path) as cache, monkeypatch.context() as environment: + environment.setenv("DATABASE_URL", database_url) + overrides: Final = { + "DATABASE_URL": database_url, + "REDIS_HOST": cache.host, + "REDIS_PORT": str(cache.port), + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "1", + } + with owned_proxy(gateway, tmp_path, overrides, config=PROXY_CONFIG, workers=2) as candidate: + db_store_id: Final = f"vs_db_{uuid.uuid4().hex}" + for phase in ("before", "during", "after"): + if phase == "during": + cache.stop() + if phase == "after": + cache.start() + for _ in range(4): + assert listed_store(candidate, CONFIG_STORE_ID)["is_config"] is True, phase + assert config_store_info(candidate)["vector_store_name"] == CONFIG_STORE_NAME, phase + assert_config_write_refused(candidate) + created = candidate.request( + "POST", + "/vector_store/new", + {"vector_store_id": f"{db_store_id}_{phase}", "custom_llm_provider": "openai"}, + ) + assert created.status_code == 200, (phase, created.text) + assert eventually( + lambda phase=phase: store_rows(f"{db_store_id}_{phase}"), lambda rows: len(rows) == 1 + ), phase + assert f"{db_store_id}_{phase}" in listed_ids(candidate), phase + assert store_rows(CONFIG_STORE_ID) == [] + with psycopg.connect(database_url) as fresh: + counted: Final = fresh.execute( + 'SELECT count(*) FROM "LiteLLM_ManagedVectorStoresTable" WHERE vector_store_id LIKE %s', + (f"{db_store_id}%",), + ).fetchone() + assert counted is not None and counted[0] == 3, counted + finally: + admin.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(identity))) + assert admin.execute("SELECT datname FROM pg_database WHERE datname=%s", (identity,)).fetchall() == [] diff --git a/tests/integration/observability/test_callback_delivery.py b/tests/integration/observability/test_callback_delivery.py index 7a5f420d2cc..543e60da4a6 100644 --- a/tests/integration/observability/test_callback_delivery.py +++ b/tests/integration/observability/test_callback_delivery.py @@ -102,7 +102,7 @@ def test_concurrent_success_and_failure_join_callbacks_and_rows_without_credenti responses: Final = tuple(pool.map(request, tags)) assert tuple(response.status_code for response in responses) == (200, 400, 200, 400) assert len(provider.drain()) == 4 - batches = [] + batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls must keep earlier batches def delivered() -> tuple[dict, ...]: batches.extend(endpoint.drain()) @@ -135,7 +135,7 @@ def test_concurrent_success_and_failure_join_callbacks_and_rows_without_credenti assert "synthetic callback failure" in json.dumps(event["error_information"]) rows: Final = eventually( lambda identity=event["id"]: read_rows( - 'SELECT request_id, spend, prompt_tokens, completion_tokens, request_tags ' + "SELECT request_id, spend, prompt_tokens, completion_tokens, request_tags " 'FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,), ), @@ -156,6 +156,118 @@ def test_concurrent_success_and_failure_join_callbacks_and_rows_without_credenti assert event["prompt_tokens"] == event["completion_tokens"] == rows[0]["completion_tokens"] == 0 +def _responses_frames(identity: str, text: str) -> tuple[bytes, ...]: + output: Final = [ + { + "type": "message", + "id": f"msg_{identity}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ] + completed: Final = { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": output, + "usage": { + "input_tokens": 11, + "output_tokens": 4, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + events: Final = ( + {"type": "response.created", "response": {**completed, "status": "in_progress", "output": [], "usage": None}}, + { + "type": "response.output_text.delta", + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": text, + }, + {"type": "response.completed", "response": completed}, + ) + return tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events) + + +@pytest.mark.covers("other.observability.callbacks.streamed_responses_events_carry_provider_response_headers") +def test_streamed_responses_success_callback_carries_provider_apim_request_id(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "resp_" + uuid.uuid4().hex + correlation: Final = "azure-correlation-" + marker + region: Final = "East US 2" + secret: Final = "synthetic-provider-secret-" + marker + sink_secret: Final = "synthetic-sink-secret-" + marker + + def upstream(request: Request) -> Reply: + assert request.target.endswith("/responses"), request.target + assert request.headers["authorization"] == f"Bearer {secret}" + assert json.loads(request.body) == { + "model": "gpt-4o-mini", + "input": "header control " + marker, + "stream": True, + }, request.body + return Reply( + content_type="text/event-stream", + chunks=_responses_frames(marker, "streamed control"), + headers={"apim-request-id": correlation, "x-ms-region": region}, + ) + + def sink(request: Request) -> Reply: + assert request.headers["authorization"] == f"Bearer {sink_secret}" + return Reply() + + with wire_server(upstream) as provider, wire_server(sink) as endpoint: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"].update({"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1}) + path: Final = tmp_path / "callbacks.yaml" + path.write_text(yaml.safe_dump(config)) + with ( + owned_proxy( + gateway, + tmp_path, + { + "GENERIC_LOGGER_ENDPOINT": endpoint.url, + "GENERIC_LOGGER_HEADERS": f"Authorization=Bearer {sink_secret}", + }, + config=path, + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=secret) + response: Final = candidate.request( + "POST", "/v1/responses", {"model": model, "input": "header control " + marker, "stream": True} + ) + assert response.status_code == 200, response.text + assert f'"item_id":"msg_{marker}"' in response.text, response.text + assert '"type":"response.completed"' in response.text, response.text + assert len(provider.drain()) == 1 + batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls must keep earlier batches + + def delivered() -> tuple[dict, ...]: + batches.extend(endpoint.drain()) + return tuple( + event for batch in batches for event in json.loads(batch.body) if event.get("model_group") == model + ) + + events: Final = eventually(delivered, lambda values: len(values) == 1, seconds=10) + assert (events[0]["status"], events[0]["stream"], events[0]["call_type"]) == ("success", True, "aresponses") + additional_headers: Final = events[0]["hidden_params"]["additional_headers"] or {} + provider_headers: Final = { + name: value + for name, value in additional_headers.items() + if name in ("llm_provider-apim-request-id", "llm_provider-x-ms-region") + } + assert provider_headers == { + "llm_provider-apim-request-id": correlation, + "llm_provider-x-ms-region": region, + }, json.dumps(events[0]["hidden_params"]) + + _RAISING_HOOK: Final = """ from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 5a79b619906..f96cae593dc 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -5,9 +5,7 @@ from typing import Final import pytest import yaml - -from integration._support.client import Gateway, eventually -from integration._support.database import read_rows +from integration._support.client import Gateway from integration._support.mcp import mcp_peer, register_mcp, tool_names from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server @@ -146,6 +144,104 @@ def test_guardrail_denial_prevents_provider_and_preserves_allowed_control(gatewa assert len(policy.drain()) == 2 +@pytest.mark.covers("other.observability.guardrails.bedrock_passthrough_converse_scans_only_caller_content") +def test_bedrock_passthrough_converse_guardrail_ignores_denied_term_in_tool_definition( + gateway: Gateway, tmp_path: Path +) -> None: + identity: Final = "guardrail" + uuid.uuid4().hex + denied: Final = "synthetic denied marker" + allowed: Final = "synthetic allowed weather question" + access_key: Final = "AKIASYNTHETICPASSTHROUGH" + tool_config: Final = { + "tools": [ + { + "toolSpec": { + "name": "lookup_weather", + "description": f"Look up the forecast, never answer a {denied}", + "inputSchema": { + "json": { + "type": "object", + "properties": {"city": {"type": "string", "enum": [denied]}}, + "required": ["city"], + } + }, + } + } + ] + } + + def guardrail(request: Request) -> Reply: + assert request.target == "/beta/litellm_basic_guardrail_api" + texts: Final = json.loads(request.body)["texts"] + result: Final = ( + {"action": "BLOCKED", "blocked_reason": "synthetic policy denial"} + if any(denied in text for text in texts) + else {"action": "NONE"} + ) + return Reply(body=json.dumps(result).encode()) + + def runtime(request: Request) -> Reply: + assert request.target == "/model/anthropic.claude-3-haiku-20240307-v1:0/converse" + assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={access_key}/"), ( + request.headers + ) + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "sunny passthrough control"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + } + ).encode() + ) + + with wire_server(guardrail) as policy, wire_server(runtime) as bedrock, gateway.scenario() as scenario: + model: Final = scenario.model( + model="bedrock/anthropic.claude-3-haiku-20240307-v1:0", + api_key=None, + api_base=bedrock.url, + aws_access_key_id=access_key, + aws_secret_access_key="synthetic-secret", + aws_region_name="us-east-1", + ) + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["guardrails"] = [ + { + "guardrail_name": identity, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": True, + "api_base": policy.url, + "api_key": "synthetic-guardrail-key", + }, + } + ] + path: Final = tmp_path / "bedrock-passthrough.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate: + route: Final = f"/bedrock/model/{model}/converse" + passed: Final = candidate.request( + "POST", + route, + {"messages": [{"role": "user", "content": [{"text": allowed}]}], "toolConfig": tool_config}, + ) + assert passed.status_code == 200, passed.text + assert passed.json()["output"]["message"]["content"] == [{"text": "sunny passthrough control"}] + forwarded: Final = bedrock.drain() + assert len(forwarded) == 1, "the runtime peer must see exactly the allowed request" + assert json.loads(forwarded[0].body)["toolConfig"] == tool_config + blocked: Final = candidate.request( + "POST", + route, + {"messages": [{"role": "user", "content": [{"text": denied}]}], "toolConfig": tool_config}, + ) + assert blocked.status_code == 400 and "synthetic policy denial" in blocked.text, blocked.text + assert bedrock.drain() == () + assert [json.loads(request.body)["texts"] for request in policy.drain()] == [[allowed], [denied]] + + @pytest.mark.covers("other.mcp.guardrails.request_selection_blocks_resolved_tool_without_execution") def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway: Gateway, tmp_path: Path) -> None: guardrail = "mcp-policy-" + uuid.uuid4().hex diff --git a/tests/integration/pricing/test_configured_prices.py b/tests/integration/pricing/test_configured_prices.py index 655d74c1402..e39833516c2 100644 --- a/tests/integration/pricing/test_configured_prices.py +++ b/tests/integration/pricing/test_configured_prices.py @@ -1,10 +1,12 @@ -from collections.abc import Iterator, Mapping -from typing import Final -from pathlib import Path +import json import uuid +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Final import pytest import yaml +from pydantic import JsonValue from tests.integration._support.client import Gateway, eventually, object_value, string_value from tests.integration._support.database import read_rows @@ -28,6 +30,31 @@ def test_custom_price_is_reported_and_charged(gateway: Gateway) -> None: assert params["output_cost_per_token"] == 0.002 +@pytest.mark.covers("quota_management.cost_estimate.configured_price.reported_for_model_absent_from_cost_map") +def test_cost_estimate_reports_configured_prices_for_model_absent_from_cost_map(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/integration-on-prem-{uuid.uuid4().hex}", + input_cost_per_token=0.003, + output_cost_per_token=0.007, + ) + response: Final = gateway.request( + "POST", + "/cost/estimate", + {"model": model, "input_tokens": 1000, "output_tokens": 500, "num_requests_per_day": 10}, + ) + assert response.status_code == 200, response.text + body: Final = object_value(response.json()) + assert body["input_cost_per_token"] == pytest.approx(0.003), response.text + assert body["output_cost_per_token"] == pytest.approx(0.007), response.text + assert body["input_cost_per_request"] == pytest.approx(1000 * 0.003), response.text + assert body["output_cost_per_request"] == pytest.approx(500 * 0.007), response.text + margin: Final = body["margin_cost_per_request"] + assert isinstance(margin, float), response.text + assert body["cost_per_request"] == pytest.approx(1000 * 0.003 + 500 * 0.007 + margin), response.text + assert body["daily_cost"] == pytest.approx(10 * (1000 * 0.003 + 500 * 0.007 + margin)), response.text + + @pytest.mark.covers("quota_management.spend_tracking.default_prices.survive_nullable_sibling_reload") def test_default_prices_survive_nullable_sibling_and_reload(gateway: Gateway) -> None: for registration_order in (("custom", "omitted", "nullable"), ("nullable", "omitted", "custom")): @@ -103,6 +130,45 @@ def test_default_prices_survive_nullable_sibling_and_reload(gateway: Gateway) -> assert float(rows[0]["spend"]) == pytest.approx(expected, rel=1e-6) +COST_MAP_DISPLAY_PRICING_KEYS: Final = frozenset( + { + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + } +) + + +def persisted_model_info(identity: str) -> dict[str, JsonValue]: + rows: Final = read_rows('SELECT model_info FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) + assert len(rows) == 1, f"Deployment {identity} has {len(rows)} rows" + stored: Final = rows[0]["model_info"] + return object_value(json.loads(stored) if isinstance(stored, str) else stored) + + +@pytest.mark.covers("pricing.model_update.echoed_cost_map_price_is_not_persisted_as_override") +def test_saving_echoed_model_info_does_not_freeze_cost_map_price_into_deployment(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list) + target: Final = next(object_value(entry) for entry in entries if object_value(entry)["model_name"] == model) + displayed: Final = object_value(target["model_info"]) + identity: Final = string_value(displayed["id"]) + assert isinstance(displayed["input_cost_per_token"], float), displayed + assert isinstance(displayed["output_cost_per_token"], float), displayed + fresh: Final = persisted_model_info(identity) + assert {key: value for key, value in fresh.items() if key in COST_MAP_DISPLAY_PRICING_KEYS} == {}, fresh + saved: Final = gateway.request( + "PATCH", f"/model/{identity}/update", {"model_info": {**displayed, "description": "echoed ui save"}} + ) + assert saved.status_code == 200, saved.text + stored: Final = persisted_model_info(identity) + assert stored["description"] == "echoed ui save", stored + assert {key: value for key, value in stored.items() if key in COST_MAP_DISPLAY_PRICING_KEYS} == {}, stored + + @pytest.mark.covers("quota_management.spend_tracking.default_prices.loaded_router_preserves_cached_defaults") def test_loaded_router_preserves_cached_defaults_during_real_requests(gateway: Gateway, tmp_path: Path) -> None: from litellm import Router diff --git a/tests/integration/pricing/test_databricks_cache_pricing.py b/tests/integration/pricing/test_databricks_cache_pricing.py new file mode 100644 index 00000000000..b32076d1bf9 --- /dev/null +++ b/tests/integration/pricing/test_databricks_cache_pricing.py @@ -0,0 +1,89 @@ +import json +import uuid +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.upstream import delete_scenario, register_scenario +from tests.integration.cost_calculation.cost_tracking_case import JsonResponse + +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +CACHE_CREATION_RATE: Final = 0.004 +CACHE_READ_RATE: Final = 0.0001 +UNCACHED_PROMPT_TOKENS: Final = 1000 +CACHE_CREATION_TOKENS: Final = 2000 +CACHE_READ_TOKENS: Final = 8000 +PROMPT_TOKENS: Final = UNCACHED_PROMPT_TOKENS + CACHE_CREATION_TOKENS + CACHE_READ_TOKENS +COMPLETION_TOKENS: Final = 500 + + +def databricks_cached_response() -> JsonResponse: + return JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1700000000, + "model": "databricks-claude-integration", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "cached reply"}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + "cache_creation_input_tokens": CACHE_CREATION_TOKENS, + "cache_read_input_tokens": CACHE_READ_TOKENS, + }, + }, + ) + + +@pytest.mark.covers("pricing.databricks.cached_prompt_tokens_bill_at_cache_rates") +def test_databricks_cached_prompt_tokens_bill_at_cache_rates_not_input_rate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + scenario_id: Final = f"databricks-cache-{uuid.uuid4().hex[:12]}" + handle: Final = register_scenario(scenario_id, databricks_cached_response()) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="databricks/databricks-claude-integration", + api_base=handle.api_base(), + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + cache_creation_input_token_cost=CACHE_CREATION_RATE, + cache_read_input_token_cost=CACHE_READ_RATE, + ) + response: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "cache control"}]} + ) + assert response.status_code == 200, response.text + expected_prompt_cost: Final = ( + UNCACHED_PROMPT_TOKENS * INPUT_RATE + + CACHE_CREATION_TOKENS * CACHE_CREATION_RATE + + CACHE_READ_TOKENS * CACHE_READ_RATE + ) + expected_completion_cost: Final = COMPLETION_TOKENS * OUTPUT_RATE + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx( + expected_prompt_cost + expected_completion_cost, rel=1e-6 + ), response.text + request_id: Final = string_value(object_value(response.json())["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, metadata, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" ' + "WHERE request_id = %s", + (request_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["prompt_tokens"] == PROMPT_TOKENS + assert rows[0]["completion_tokens"] == COMPLETION_TOKENS + assert float(rows[0]["spend"]) == pytest.approx(expected_prompt_cost + expected_completion_cost, rel=1e-6) + metadata: Final = rows[0]["metadata"] + parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata) + breakdown: Final = object_value(parsed["cost_breakdown"]) + assert float(breakdown["input_cost"]) == pytest.approx(expected_prompt_cost, rel=1e-6) + assert float(breakdown["output_cost"]) == pytest.approx(expected_completion_cost, rel=1e-6) diff --git a/tests/integration/pricing/test_ocr_page_pricing.py b/tests/integration/pricing/test_ocr_page_pricing.py new file mode 100644 index 00000000000..65f94ea673e --- /dev/null +++ b/tests/integration/pricing/test_ocr_page_pricing.py @@ -0,0 +1,59 @@ +import uuid +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway, eventually, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.upstream import delete_scenario, register_scenario +from tests.integration.cost_calculation.cost_tracking_case import JsonResponse + + +@pytest.mark.covers("pricing.ocr.annotation_pages_billed_at_annotation_rate") +def test_ocr_annotation_pages_are_billed_at_annotation_cost_per_page(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + scenario_id: Final = f"ocr-annotation-{uuid.uuid4().hex[:12]}" + handle: Final = register_scenario( + scenario_id, + JsonResponse( + content_type="application/json", + body={ + "pages": [{"index": index, "markdown": f"page {index}"} for index in range(3)], + "model": "integration-ocr", + "document_annotation": '{"title": "annotated"}', + "usage_info": {"pages_processed": 3, "pages_processed_annotation": 2, "doc_size_bytes": 4096}, + }, + ), + ) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model=f"mistral/integration-ocr-{scenario_id}", + api_base=f"{handle.api_base()}/v1", + ocr_cost_per_page=0.002, + annotation_cost_per_page=0.01, + ) + response: Final = gateway.request( + "POST", + "/v1/ocr", + { + "model": model, + "document": {"type": "document_url", "document_url": "https://example.com/annotated.pdf"}, + "document_annotation_format": {"type": "json_schema", "json_schema": {"name": "title"}}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["usage_info"] == { + "pages_processed": 3, + "pages_processed_annotation": 2, + "credits": None, + "doc_size_bytes": 4096, + }, response.text + expected: Final = 3 * 0.002 + 2 * 0.01 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected), response.text + request_id: Final = string_value(response.headers["x-litellm-call-id"]) + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(rows[0]["spend"]) == pytest.approx(expected) diff --git a/tests/integration/pricing/test_service_tier_pricing.py b/tests/integration/pricing/test_service_tier_pricing.py new file mode 100644 index 00000000000..e0d26392f7f --- /dev/null +++ b/tests/integration/pricing/test_service_tier_pricing.py @@ -0,0 +1,71 @@ +import json +from typing import Final + +import httpx +import pytest + +from tests.integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows + +STANDARD_INPUT_RATE: Final = 0.001 +STANDARD_OUTPUT_RATE: Final = 0.002 +ULTRAFAST_INPUT_RATE: Final = 0.01 +ULTRAFAST_OUTPUT_RATE: Final = 0.02 + + +def assert_chat_bills_rates( + gateway: Gateway, model: str, service_tier: str | None, input_rate: float, output_rate: float +) -> None: + with httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: + upstream.get("/__observations").raise_for_status() + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"service tier {service_tier} control"}], + **({} if service_tier is None else {"service_tier": service_tier}), + }, + ) + assert response.status_code == 200, response.text + expected: Final = 20 * input_rate + 20 * output_rate + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected, rel=1e-6), response.text + observations: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] + assert isinstance(observations, list) + assert len(observations) == 1 + body: Final = object_value(object_value(observations[0])["body"]) + assert body == { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": f"service tier {service_tier} control"}], + **({} if service_tier is None else {"service_tier": service_tier}), + }, response.text + request_id: Final = string_value(object_value(response.json())["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, metadata, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id = %s', + (request_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["prompt_tokens"] == 20 + assert rows[0]["completion_tokens"] == 20 + assert float(rows[0]["spend"]) == pytest.approx(expected, rel=1e-6) + metadata: Final = rows[0]["metadata"] + parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata) + breakdown: Final = object_value(parsed["cost_breakdown"]) + assert float(breakdown["input_cost"]) == pytest.approx(20 * input_rate, rel=1e-6) + assert float(breakdown["output_cost"]) == pytest.approx(20 * output_rate, rel=1e-6) + + +@pytest.mark.covers("quota_management.spend_tracking.service_tier_pricing.ultrafast_bills_ultrafast_rates") +def test_ultrafast_service_tier_bills_ultrafast_rates_and_keeps_pricing_off_the_wire(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model( + input_cost_per_token=STANDARD_INPUT_RATE, + output_cost_per_token=STANDARD_OUTPUT_RATE, + input_cost_per_token_ultrafast=ULTRAFAST_INPUT_RATE, + output_cost_per_token_ultrafast=ULTRAFAST_OUTPUT_RATE, + ) + assert_chat_bills_rates(gateway, model, "ultrafast", ULTRAFAST_INPUT_RATE, ULTRAFAST_OUTPUT_RATE) + assert_chat_bills_rates(gateway, model, None, STANDARD_INPUT_RATE, STANDARD_OUTPUT_RATE) diff --git a/tests/integration/providers/test_anthropic_advisor_wire.py b/tests/integration/providers/test_anthropic_advisor_wire.py new file mode 100644 index 00000000000..77fa27cd2a9 --- /dev/null +++ b/tests/integration/providers/test_anthropic_advisor_wire.py @@ -0,0 +1,113 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +_ADVISOR_KEY: Final = "synthetic-advisor-key" +_QUESTION: Final = "which index should this query use" +_ADVICE: Final = "use the composite index on (tenant_id, created_at)" +_FINAL_ANSWER: Final = "done, the composite index is the right one" + + +_ADVISOR_CALL_MESSAGE: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "advisor-call", + "type": "function", + "function": {"name": "advisor", "arguments": json.dumps({"question": _QUESTION})}, + } + ], +} +_FINAL_MESSAGE: Final = {"role": "assistant", "content": _FINAL_ANSWER} + + +def _chat_completion(identity: str, message: dict[str, object], finish_reason: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{identity}", + "object": "chat.completion", + "created": 1, + "model": "llama-3.3-70b-versatile", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14}, + } + ).encode() + ) + + +def _executor_reply(body: dict[str, object], identity: str) -> Reply: + messages: Final = body["messages"] + assert isinstance(messages, list) + if any(message.get("role") == "tool" for message in messages): + assert messages[-1]["content"] == _ADVICE + return _chat_completion(identity, _FINAL_MESSAGE, "stop") + tools: Final = body["tools"] + assert isinstance(tools, list) + assert tools[0]["function"]["name"] == "advisor" + return _chat_completion(identity, _ADVISOR_CALL_MESSAGE, "tool_calls") + + +@pytest.mark.covers("providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment") +def test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated( + gateway: Gateway, +) -> None: + identity: Final = "advisor-wire-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + body: Final = json.loads(request.body) + if request.target == "/v1/chat/completions": + assert request.headers["authorization"] == "Bearer integration-provider-key" + return _executor_reply(body, identity) + assert request.target == "/v1/messages" + assert request.headers["x-api-key"] == _ADVISOR_KEY + assert body["model"] == "claude-opus-4-1-20250805" + assert body["messages"] == [ + {"role": "user", "content": "please plan the migration"}, + {"role": "user", "content": _QUESTION}, + ] + assert "tools" not in body + return Reply( + body=json.dumps( + { + "id": f"msg-{identity}", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-1-20250805", + "content": [{"type": "text", "text": _ADVICE}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 6}, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + executor: Final = scenario.model(model="hosted_vllm/llama-3.3-70b", api_base=wire.url + "/v1") + advisor: Final = scenario.model( + model="anthropic/claude-opus-4-1-20250805", api_base=wire.url, api_key=_ADVISOR_KEY + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": executor, + "max_tokens": 64, + "messages": [{"role": "user", "content": "please plan the migration"}], + "tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor}], + }, + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["content"] == [{"type": "text", "text": _FINAL_ANSWER}], response.text + assert body["stop_reason"] == "end_turn", response.text + assert [request.target for request in wire.drain()] == [ + "/v1/chat/completions", + "/v1/messages", + "/v1/chat/completions", + ] diff --git a/tests/integration/providers/test_azure_ai_chat_wire.py b/tests/integration/providers/test_azure_ai_chat_wire.py new file mode 100644 index 00000000000..57acb9773b9 --- /dev/null +++ b/tests/integration/providers/test_azure_ai_chat_wire.py @@ -0,0 +1,85 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "kimi-k2-thinking" +_API_KEY: Final = "synthetic-azure-ai-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_THINKING_BLOCK: Final[JsonValue] = { + "type": "thinking", + "thinking": "The user wants the sum of 17 and 26.", + "signature": "synthetic-signature", +} +_HISTORY_WITH_ANTHROPIC_FIELDS: Final[JsonValue] = [ + { + "role": "system", + "content": "You are a calculator.", + "cache_control": {"type": "ephemeral"}, + }, + {"role": "user", "content": "What is 17 + 26?"}, + { + "role": "assistant", + "content": "43", + "thinking_blocks": [_THINKING_BLOCK], + "provider_specific_fields": {"citations": None}, + }, + {"role": "user", "content": "And doubled?"}, +] +_HISTORY_AS_OPENAI_SPEC: Final[JsonValue] = [ + {"role": "system", "content": "You are a calculator."}, + {"role": "user", "content": "What is 17 + 26?"}, + {"role": "assistant", "content": "43"}, + {"role": "user", "content": "And doubled?"}, +] + + +def _completion(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "86"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 31, "completion_tokens": 2, "total_tokens": 33}, + } + ).encode() + + +@pytest.mark.covers("providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry") +def test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages(gateway: Gateway) -> None: + identity: Final = f"azure-ai-strip-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["messages"] == _HISTORY_AS_OPENAI_SPEC + return Reply(body=_completion(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"azure_ai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _HISTORY_WITH_ANTHROPIC_FIELDS}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": {"role": "assistant", "content": "86"}, + "provider_specific_fields": {}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] diff --git a/tests/integration/providers/test_azure_ai_flux2_image_wire.py b/tests/integration/providers/test_azure_ai_flux2_image_wire.py new file mode 100644 index 00000000000..59f125463a6 --- /dev/null +++ b/tests/integration/providers/test_azure_ai_flux2_image_wire.py @@ -0,0 +1,48 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_FLEX_MODEL: Final = "azure_ai/FLUX.2-flex" +_PROMPT: Final = "a red fox in the snow" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +@pytest.mark.covers("other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body") +def test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/providers/blackforestlabs/v1/flux-2-flex?api-version=preview" + assert request.headers["api-key"] == "synthetic-azure-key" + assert _JSON_OBJECT.validate_json(request.body) == { + "model": "FLUX.2-flex", + "prompt": _PROMPT, + "num_images": 2, + "width": 1536, + "height": 1024, + "guidance": 4.5, + "steps": 32, + } + return Reply(body=json.dumps({"data": [{"b64_json": "aW1n"}, {"b64_json": "aW1n"}]}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=_FLEX_MODEL, api_base=wire.url, api_key="synthetic-azure-key", api_version="preview" + ) + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT, "n": 2, "size": "1536x1024", "guidance": 4.5, "steps": 32}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["data"] == [ + {"url": None, "b64_json": "aW1n", "revised_prompt": None, "provider_specific_fields": None}, + {"url": None, "b64_json": "aW1n", "revised_prompt": None, "provider_specific_fields": None}, + ] + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/providers/blackforestlabs/v1/flux-2-flex?api-version=preview") + ] diff --git a/tests/integration/providers/test_azure_ai_rerank_auth_wire.py b/tests/integration/providers/test_azure_ai_rerank_auth_wire.py new file mode 100644 index 00000000000..bc700b9a628 --- /dev/null +++ b/tests/integration/providers/test_azure_ai_rerank_auth_wire.py @@ -0,0 +1,46 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL: Final = "azure_ai/Cohere-rerank-v4.0-fast" +ENTRA_TOKEN: Final = "synthetic-entra-access-token" +QUERY: Final = "which document mentions the gateway" +DOCUMENTS: Final = ("the gateway proxies rerank calls", "unrelated synthetic text") +RESPONSE: Final = json.dumps( + { + "id": "synthetic-rerank-id", + "results": [{"index": 0, "relevance_score": 0.91}, {"index": 1, "relevance_score": 0.03}], + "meta": {"api_version": {"version": "2"}, "billed_units": {"search_units": 1}}, + } +).encode() + + +def entra_rerank_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/providers/cohere/v2/rerank" + assert request.headers["authorization"] == f"Bearer {ENTRA_TOKEN}" + assert "api-key" not in request.headers + body: Final = json.loads(request.body) + assert body == {"model": "Cohere-rerank-v4.0-fast", "query": QUERY, "documents": list(DOCUMENTS), "top_n": 2} + return Reply(body=RESPONSE) + + +@pytest.mark.covers("other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider") +def test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider(gateway: Gateway) -> None: + with wire_server(entra_rerank_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, + api_key=None, + api_base=f"{wire.url}/providers/cohere/v2", + azure_ad_token=ENTRA_TOKEN, + model_info={"mode": "rerank"}, + ) + response: Final = gateway.request( + "POST", "/v1/rerank", {"model": model, "query": QUERY, "documents": list(DOCUMENTS), "top_n": 2} + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert [(result["index"], result["relevance_score"]) for result in body["results"]] == [(0, 0.91), (1, 0.03)] + assert len(wire.drain()) == 1, "Expected exactly one provider rerank call" diff --git a/tests/integration/providers/test_bedrock_auth_wire.py b/tests/integration/providers/test_bedrock_auth_wire.py index bd24dc171ba..0dc2dfbf581 100644 --- a/tests/integration/providers/test_bedrock_auth_wire.py +++ b/tests/integration/providers/test_bedrock_auth_wire.py @@ -7,18 +7,22 @@ from typing import Final import pytest import yaml - from integration._support.client import Gateway from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server MODEL: Final = "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0" TOKEN: Final = "synthetic-bedrock-bearer" -RESPONSE: Final = json.dumps({ - "output": {"message": {"role": "assistant", "content": [{"text": "bedrock wire control"}]}}, - "stopReason": "end_turn", "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}, - "metrics": {"latencyMs": 1}, -}).encode() +ACCESS_KEY: Final = "AKIAINTEGRATION000002" +CLIENT_OAUTH_TOKEN: Final = "Bearer sk-ant-oat01-synthetic-client-subscription-token" +RESPONSE: Final = json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "bedrock wire control"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + } +).encode() def bearer_peer(request: Request) -> Reply: @@ -34,31 +38,58 @@ def bearer_peer(request: Request) -> Reply: @pytest.mark.covers("other.provider_wire.bedrock.bearer_sdk_skips_credential_chain") -async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: import litellm empty: Final = tmp_path / "empty-aws-config" empty.write_text("") for name in tuple(name for name in os.environ if name.startswith("AWS_")): monkeypatch.delenv(name, raising=False) - for name, value in {"AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_RUST": "false"}.items(): + for name, value in { + "AWS_CONFIG_FILE": str(empty), + "AWS_SHARED_CREDENTIALS_FILE": str(empty), + "AWS_EC2_METADATA_DISABLED": "true", + "LITELLM_RUST": "false", + }.items(): monkeypatch.setenv(name, value) with wire_server(bearer_peer) as wire: with pytest.raises(litellm.APIConnectionError, match=r"config profile .* could not be found"): - await asyncio.to_thread(litellm.completion, model=MODEL, aws_profile_name="integration-profile-must-not-be-read", aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url, messages=[{"role": "user", "content": "synthetic credential control"}], timeout=5, num_retries=0) + await asyncio.to_thread( + litellm.completion, + model=MODEL, + aws_profile_name="integration-profile-must-not-be-read", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + messages=[{"role": "user", "content": "synthetic credential control"}], + timeout=5, + num_retries=0, + ) assert wire.drain() == () for source in ("argument", "environment"): if source == "environment": monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", TOKEN) parameters: Final = { - "model": MODEL, "api_key": TOKEN if source == "argument" else None, - "aws_region_name": "us-east-1", "aws_profile_name": "integration-profile-must-not-be-read", - "aws_bedrock_runtime_endpoint": wire.url, "timeout": 5, "num_retries": 0, - "messages": [{"role": "system", "content": "synthetic system"}, {"role": "user", "content": "synthetic bearer request"}], + "model": MODEL, + "api_key": TOKEN if source == "argument" else None, + "aws_region_name": "us-east-1", + "aws_profile_name": "integration-profile-must-not-be-read", + "aws_bedrock_runtime_endpoint": wire.url, + "timeout": 5, + "num_retries": 0, + "messages": [ + {"role": "system", "content": "synthetic system"}, + {"role": "user", "content": "synthetic bearer request"}, + ], "max_tokens": 16, } for asynchronous in (False, True): - result: Final = await litellm.acompletion(**parameters) if asynchronous else await asyncio.to_thread(litellm.completion, **parameters) + result: Final = ( + await litellm.acompletion(**parameters) + if asynchronous + else await asyncio.to_thread(litellm.completion, **parameters) + ) assert result.choices[0].message.content == "bedrock wire control" assert result.choices[0].finish_reason == "stop" assert result.usage.prompt_tokens == 11 and result.usage.completion_tokens == 4 @@ -66,28 +97,57 @@ async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credential @pytest.mark.covers("other.provider_wire.bedrock.bearer_db_yaml_survives_reload") -def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload(gateway: Gateway, tmp_path: Path) -> None: +def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload( + gateway: Gateway, tmp_path: Path +) -> None: empty: Final = tmp_path / "empty-aws-config" empty.write_text("") with wire_server(bearer_peer) as wire: parameters: Final = { - "model": MODEL, "api_key": "os.environ/INTEGRATION_BEARER_TOKEN", "aws_region_name": "us-east-1", - "aws_profile_name": "integration-profile-must-not-be-read", "aws_bedrock_runtime_endpoint": wire.url, + "model": MODEL, + "api_key": "os.environ/INTEGRATION_BEARER_TOKEN", + "aws_region_name": "us-east-1", + "aws_profile_name": "integration-profile-must-not-be-read", + "aws_bedrock_runtime_endpoint": wire.url, } alias: Final = f"integration-yaml-{uuid.uuid4().hex}" configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) configuration["model_list"] = [{"model_name": alias, "litellm_params": parameters, "model_info": {"id": alias}}] path: Final = tmp_path / "bedrock.yaml" path.write_text(yaml.safe_dump(configuration)) - overrides: Final = {"INTEGRATION_BEARER_TOKEN": TOKEN, "AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_RUST": "false"} - with owned_proxy(gateway, tmp_path, overrides, config=path, remove_environment=tuple(name for name in os.environ if name.startswith("AWS_"))) as candidate, candidate.scenario() as scenario: + overrides: Final = { + "INTEGRATION_BEARER_TOKEN": TOKEN, + "AWS_CONFIG_FILE": str(empty), + "AWS_SHARED_CREDENTIALS_FILE": str(empty), + "AWS_EC2_METADATA_DISABLED": "true", + "LITELLM_RUST": "false", + } + with ( + owned_proxy( + gateway, + tmp_path, + overrides, + config=path, + remove_environment=tuple(name for name in os.environ if name.startswith("AWS_")), + ) as candidate, + candidate.scenario() as scenario, + ): database_model: Final = scenario.model(**parameters) for generation in range(2): for model in (alias, database_model): - response: Final = candidate.request("POST", "/v1/chat/completions", { - "model": model, "messages": [{"role": "system", "content": "synthetic system"}, {"role": "user", "content": "synthetic bearer request"}], - "max_tokens": 16, "cache": {"no-cache": True}, - }) + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + {"role": "system", "content": "synthetic system"}, + {"role": "user", "content": "synthetic bearer request"}, + ], + "max_tokens": 16, + "cache": {"no-cache": True}, + }, + ) assert response.status_code == 200, response.text assert response.json()["choices"][0]["message"]["content"] == "bedrock wire control" assert response.json()["usage"]["total_tokens"] == 15 @@ -95,5 +155,61 @@ def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload if generation == 0: entries: Final = candidate.get("/model/info")["data"] target: Final = next(entry for entry in entries if entry["model_name"] == database_model) - response: Final = candidate.request("PATCH", f"/model/{target['model_info']['id']}/update", {"model_info": {"description": "bearer reload"}}) + response: Final = candidate.request( + "PATCH", + f"/model/{target['model_info']['id']}/update", + {"model_info": {"description": "bearer reload"}}, + ) assert response.status_code == 200, response.text + + +INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-3-haiku-20240307-v1:0" +INVOKE_RESPONSE: Final = json.dumps( + { + "id": "msg_synthetic", + "type": "message", + "role": "assistant", + "model": "anthropic.claude-3-haiku-20240307-v1:0", + "content": [{"type": "text", "text": "bedrock invoke wire control"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 4}, + } +).encode() + + +def sigv4_invoke_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/model/anthropic.claude-3-haiku-20240307-v1:0/invoke" + assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/"), dict( + request.headers + ) + assert CLIENT_OAUTH_TOKEN not in request.headers.values(), dict(request.headers) + assert json.loads(request.body)["messages"] == [{"role": "user", "content": "synthetic oauth isolation request"}] + return Reply(body=INVOKE_RESPONSE) + + +@pytest.mark.covers("providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization") +def test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature(gateway: Gateway) -> None: + with wire_server(sigv4_invoke_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=INVOKE_MODEL, + api_key=None, + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key="synthetic-secret-key-for-testing", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + api_base=wire.url, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "messages": [{"role": "user", "content": "synthetic oauth isolation request"}], + "max_tokens": 16, + }, + headers={"Authorization": CLIENT_OAUTH_TOKEN, "x-litellm-api-key": f"Bearer {gateway.key}"}, + ) + assert response.status_code == 200, response.text + assert response.json()["content"] == [{"type": "text", "text": "bedrock invoke wire control"}], response.text + assert len(wire.drain()) == 1, response.text diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py new file mode 100644 index 00000000000..1a834fc2f8c --- /dev/null +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -0,0 +1,78 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL: Final = "bedrock/anthropic.claude-3-haiku-20240307-v1:0" +BUCKET: Final = "integration-batch-bucket" +PROMPT: Final = "synthetic completions prompt" +RESPONSES_INPUT: Final = "synthetic responses input" +INPUT_LINES: Final = ( + { + "custom_id": "completions-record", + "method": "POST", + "url": "/v1/completions", + "body": {"model": MODEL, "prompt": PROMPT, "max_tokens": 64}, + }, + { + "custom_id": "responses-record", + "method": "POST", + "url": "/v1/responses", + "body": {"model": MODEL, "input": RESPONSES_INPUT, "max_output_tokens": 16}, + }, +) +EXPECTED_S3_OBJECT: Final = ( + { + "recordId": "completions-record", + "modelInput": { + "messages": [{"role": "user", "content": [{"type": "text", "text": PROMPT}]}], + "max_tokens": 64, + "anthropic_version": "bedrock-2023-05-31", + }, + }, + { + "recordId": "responses-record", + "modelInput": { + "messages": [{"role": "user", "content": [{"type": "text", "text": RESPONSES_INPUT}]}], + "max_tokens": 16, + "anthropic_version": "bedrock-2023-05-31", + }, + }, +) + + +def s3_peer(request: Request) -> Reply: + assert request.method == "PUT" and request.target.startswith(f"/{BUCKET}/"), request.target + assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 ") + return Reply(body=b"") + + +@pytest.mark.covers( + "other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages" +) +def test_completions_and_responses_batch_records_upload_as_anthropic_user_messages(gateway: Gateway) -> None: + with wire_server(s3_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, + api_key=None, + api_base=None, + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_region_name="us-east-1", + s3_bucket_name=BUCKET, + s3_endpoint_url=wire.url, + ) + jsonl: Final = "\n".join(json.dumps(line, separators=(",", ":")) for line in INPUT_LINES) + "\n" + response: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model}, + {"file": ("in.jsonl", jsonl.encode(), "application/jsonl")}, + ) + assert response.status_code == 200, response.text + assert response.json()["object"] == "file" and response.json()["purpose"] == "batch", response.text + uploads: Final = wire.drain() + assert len(uploads) == 1, f"Expected exactly one S3 PUT, saw {[upload.target for upload in uploads]}" + stored: Final = tuple(json.loads(line) for line in uploads[0].body.decode().splitlines() if line.strip()) + assert stored == EXPECTED_S3_OBJECT diff --git a/tests/integration/providers/test_bedrock_claude_thinking_wire.py b/tests/integration/providers/test_bedrock_claude_thinking_wire.py new file mode 100644 index 00000000000..d3673a55660 --- /dev/null +++ b/tests/integration/providers/test_bedrock_claude_thinking_wire.py @@ -0,0 +1,60 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL: Final = "bedrock/invoke/us.anthropic.claude-opus-4-8" +TOKEN: Final = "synthetic-bedrock-bearer" +RESPONSE: Final = json.dumps( + { + "id": "msg_adaptive_control", + "type": "message", + "role": "assistant", + "model": "us.anthropic.claude-opus-4-8", + "content": [{"type": "text", "text": "adaptive thinking control"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 5}, + } +).encode() + + +def adaptive_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/model/us.anthropic.claude-opus-4-8/invoke" + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = json.loads(request.body) + assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "synthetic effort request"}]}] + assert body["thinking"]["type"] == "adaptive", body + assert body["output_config"] == {"effort": "high"}, body + assert "budget_tokens" not in json.dumps(body), body + return Reply(body=RESPONSE) + + +@pytest.mark.covers("other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking") +def test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens( + gateway: Gateway, +) -> None: + with wire_server(adaptive_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, + api_key=TOKEN, + aws_region_name="us-east-1", + api_base=wire.url, + aws_bedrock_runtime_endpoint=wire.url, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "synthetic effort request"}], + "max_tokens": 4096, + "reasoning_effort": "high", + }, + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == "adaptive thinking control" + assert response.json()["usage"]["prompt_tokens"] == 12 and response.json()["usage"]["completion_tokens"] == 5 + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_bedrock_converse_config_blocks_wire.py b/tests/integration/providers/test_bedrock_converse_config_blocks_wire.py new file mode 100644 index 00000000000..239236bb29d --- /dev/null +++ b/tests/integration/providers/test_bedrock_converse_config_blocks_wire.py @@ -0,0 +1,46 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from integration.providers.test_bedrock_auth_wire import MODEL, RESPONSE, TOKEN + +GUARDRAIL: Final = {"guardrailIdentifier": "integration-guardrail", "guardrailVersion": "DRAFT", "trace": "enabled"} +PERFORMANCE: Final = {"latency": "optimized"} + + +def converse_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse" + body: Final = json.loads(request.body) + assert body["inferenceConfig"] == {"maxTokens": 16, "temperature": 0.2}, body + assert body["guardrailConfig"] == GUARDRAIL, body + assert body["performanceConfig"] == PERFORMANCE, body + return Reply(body=RESPONSE) + + +@pytest.mark.covers("other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level") +def test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config(gateway: Gateway) -> None: + with wire_server(converse_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, + api_key=TOKEN, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + guardrailConfig=GUARDRAIL, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "synthetic guardrail request"}], + "max_tokens": 16, + "temperature": 0.2, + "performanceConfig": PERFORMANCE, + "cache": {"no-cache": True}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == "bedrock wire control" + assert len(wire.drain()) == 1, response.text diff --git a/tests/integration/providers/test_bedrock_embedding_wire.py b/tests/integration/providers/test_bedrock_embedding_wire.py new file mode 100644 index 00000000000..40eb0fe2a70 --- /dev/null +++ b/tests/integration/providers/test_bedrock_embedding_wire.py @@ -0,0 +1,58 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL: Final = "bedrock/cohere.embed-english-v3" +TOKEN: Final = "synthetic-bedrock-bearer" +INPUT: Final = "hello world" +VECTOR: Final = [0.1, 0.2, 0.3] +RESPONSE: Final = json.dumps( + { + "embeddings": {"float": [VECTOR]}, + "id": "synthetic-cohere-embed", + "response_type": "embeddings_by_type", + "texts": [INPUT], + } +).encode() + + +def cohere_english_v3_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/model/cohere.embed-english-v3/invoke" + assert request.headers["authorization"] == f"Bearer {TOKEN}" + assert json.loads(request.body) == { + "texts": [INPUT], + "input_type": "search_document", + "embedding_types": ["float"], + "output_dimension": 512, + } + return Reply(body=RESPONSE) + + +@pytest.mark.covers("other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format") +def test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions(gateway: Gateway) -> None: + with wire_server(cohere_english_v3_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, + api_key=TOKEN, + api_base=wire.url, + aws_region_name="us-east-1", + ) + for encoding_format in ("float", "base64"): + response: Final = gateway.request( + "POST", + "/v1/embeddings", + { + "model": model, + "input": INPUT, + "encoding_format": encoding_format, + "dimensions": 512, + }, + ) + assert response.status_code == 200, f"encoding_format={encoding_format}: {response.text}" + assert response.json()["data"] == [ + {"object": "embedding", "index": 0, "embedding": VECTOR, "type": "float"}, + ], response.text + assert len(wire.drain()) == 1, f"encoding_format={encoding_format} never reached Bedrock" diff --git a/tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py b/tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py new file mode 100644 index 00000000000..d69c05ad1b1 --- /dev/null +++ b/tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py @@ -0,0 +1,50 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL: Final = "bedrock/converse/us.openai.gpt-5.6-sol" +TOKEN: Final = "synthetic-bedrock-bearer" +RESPONSE: Final = json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "gpt-5 reasoning wire control"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14}, + "metrics": {"latencyMs": 1}, + } +).encode() + + +def gpt5_converse_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/model/us.openai.gpt-5.6-sol/converse" + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = json.loads(request.body) + assert body["messages"] == [{"role": "user", "content": [{"text": "synthetic reasoning request"}]}] + assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body + assert body["inferenceConfig"] == {"maxTokens": 16}, body + return Reply(body=RESPONSE) + + +@pytest.mark.covers("providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort") +def test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort(gateway: Gateway) -> None: + with wire_server(gpt5_converse_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, api_key=TOKEN, aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "synthetic reasoning request"}], + "reasoning_effort": "high", + "max_tokens": 16, + "cache": {"no-cache": True}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == "gpt-5 reasoning wire control", response.text + assert response.json()["usage"]["total_tokens"] == 14, response.text + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_bedrock_invoke_tool_search_wire.py b/tests/integration/providers/test_bedrock_invoke_tool_search_wire.py new file mode 100644 index 00000000000..3806787ed95 --- /dev/null +++ b/tests/integration/providers/test_bedrock_invoke_tool_search_wire.py @@ -0,0 +1,88 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +MODEL_ID: Final = "us.anthropic.claude-sonnet-5" +TOKEN: Final = "synthetic-bedrock-bearer" +TOOL_SEARCH_TOOL: Final = {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} +DEFERRED_TOOL: Final = { + "name": "get_weather", + "description": "Weather lookup", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + "defer_loading": True, +} +RESPONSE: Final = json.dumps( + { + "id": "msg_tool_search_control", + "type": "message", + "role": "assistant", + "model": MODEL_ID, + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_control", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather"}, + }, + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_control", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + }, + }, + {"type": "text", "text": "tool search wire control"}, + ], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 6}, + } +).encode() + + +def tool_search_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == f"/model/{MODEL_ID}/invoke", request.target + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = json.loads(request.body) + assert body["anthropic_beta"] == ["tool-search-tool-2025-10-19"], body + assert body["messages"] == [{"role": "user", "content": "find the weather tool"}] + assert body["tools"] == [TOOL_SEARCH_TOOL, DEFERRED_TOOL], body["tools"] + assert body["max_tokens"] == 64 + assert "model" not in body + return Reply(body=RESPONSE) + + +@pytest.mark.covers("providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support") +def test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field(gateway: Gateway) -> None: + with wire_server(tool_search_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/invoke/{MODEL_ID}", + api_key=TOKEN, + aws_region_name="us-east-1", + api_base=wire.url, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "messages": [{"role": "user", "content": "find the weather tool"}], + "tools": [TOOL_SEARCH_TOOL, DEFERRED_TOOL], + }, + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["content"][2] == {"type": "text", "text": "tool search wire control"}, response.text + assert body["stop_reason"] == "end_turn" + assert body["usage"]["input_tokens"] == 12 and body["usage"]["output_tokens"] == 6 + assert len(wire.drain()) == 1 + entries: Final = gateway.get("/v1/model/info")["data"] + assert isinstance(entries, list) + info: Final = next(entry for entry in entries if isinstance(entry, dict) and entry["model_name"] == model) + assert isinstance(info["model_info"], dict) + assert info["model_info"]["supports_tool_search"] is True, info["model_info"] diff --git a/tests/integration/providers/test_bedrock_mantle_codex_input_wire.py b/tests/integration/providers/test_bedrock_mantle_codex_input_wire.py new file mode 100644 index 00000000000..bb7961160dc --- /dev/null +++ b/tests/integration/providers/test_bedrock_mantle_codex_input_wire.py @@ -0,0 +1,88 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol" +TOKEN: Final = "synthetic-mantle-bearer" +CIPHERTEXT: Final = "synthetic-compaction-ciphertext" +CALL_ID: Final = "call_synthetic_shell" +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +ACTION: Final[dict[str, JsonValue]] = {"type": "exec", "command": ["ls", "-la"], "timeout_ms": 1000} +RESPONSE: Final = json.dumps( + { + "id": "resp_synthetic_mantle", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "openai.gpt-5.6-sol", + "output": [ + { + "type": "message", + "id": "msg_synthetic_mantle", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}], + } + ], + "usage": {"input_tokens": 21, "output_tokens": 4, "total_tokens": 25}, + } +).encode() + + +def user_turn(text: str) -> JsonValue: + return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]} + + +def codex_history(marker: str) -> tuple[JsonValue, ...]: + return ( + user_turn(f"first turn {marker}"), + {"type": "agent_message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent reply"}]}, + {"type": "context_compaction", "encrypted_content": CIPHERTEXT}, + {"type": "local_shell_call", "call_id": CALL_ID, "status": "completed", "action": ACTION}, + {"type": "function_call_output", "call_id": CALL_ID, "output": "synthetic shell output"}, + user_turn(f"next turn {marker}"), + ) + + +def mantle_history(marker: str) -> tuple[JsonValue, ...]: + return ( + user_turn(f"first turn {marker}"), + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent reply"}]}, + {"type": "compaction", "encrypted_content": CIPHERTEXT}, + {"type": "function_call", "call_id": CALL_ID, "name": "local_shell", "arguments": json.dumps(ACTION)}, + {"type": "function_call_output", "call_id": CALL_ID, "output": "synthetic shell output"}, + user_turn(f"next turn {marker}"), + ) + + +@pytest.mark.covers("other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types") +def test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + expected_input: Final = list(mantle_history(marker)) + + def mantle_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/openai/v1/responses", request.target + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = JSON_OBJECT.validate_json(request.body) + assert body["model"] == "openai.gpt-5.6-sol", body + assert body["input"] == expected_input, body["input"] + return Reply(body=RESPONSE) + + with wire_server(mantle_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=MODEL, api_key=TOKEN, api_base=wire.url, aws_region_name="us-east-2") + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": list(codex_history(marker)), "store": False} + ) + assert response.status_code == 200, response.text + assert response.json()["output"][0]["content"][0]["text"] == "mantle wire control", response.text + assert response.json()["usage"]["total_tokens"] == 25, response.text + forwarded: Final = wire.drain() + assert len(forwarded) == 1, forwarded + assert JSON_OBJECT.validate_json(forwarded[0].body)["input"] == expected_input, forwarded[0].body diff --git a/tests/integration/providers/test_bedrock_mantle_responses_wire.py b/tests/integration/providers/test_bedrock_mantle_responses_wire.py new file mode 100644 index 00000000000..9bc6f83f8e4 --- /dev/null +++ b/tests/integration/providers/test_bedrock_mantle_responses_wire.py @@ -0,0 +1,106 @@ +import json +from collections.abc import Callable +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol" +_TOKEN: Final = "synthetic-mantle-bearer" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SHELL_ACTION: Final[dict[str, JsonValue]] = {"type": "exec", "command": ["ls", "-la"], "timeout_ms": 1000} +_OUTPUT_MESSAGE: Final[dict[str, JsonValue]] = { + "type": "message", + "id": "msg_mantle", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}], +} +_RESPONSE: Final = json.dumps( + { + "id": "resp_mantle", + "object": "response", + "status": "completed", + "created_at": 1700000000, + "model": "gpt-5.6-sol", + "output": [_OUTPUT_MESSAGE], + "usage": { + "input_tokens": 11, + "output_tokens": 4, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } +).encode() + + +def _codex_history(marker: str) -> list[JsonValue]: + return [ + {"type": "message", "role": "user", "content": f"delegate to a subagent {marker}"}, + { + "type": "agent_message", + "id": "msg_agent", + "content": [{"type": "text", "text": "sub-agent said "}, {"type": "text", "encrypted_content": "hello"}], + }, + {"type": "context_compaction", "id": "cmp_1", "encrypted_content": "compacted-history"}, + { + "type": "local_shell_call", + "id": "lsc_1", + "call_id": "call_shell", + "status": "completed", + "action": _SHELL_ACTION, + }, + {"type": "function_call_output", "call_id": "call_shell", "output": "total 0"}, + ] + + +def _mantle_history(marker: str) -> list[JsonValue]: + return [ + {"type": "message", "role": "user", "content": f"delegate to a subagent {marker}"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent said hello"}]}, + {"type": "compaction", "encrypted_content": "compacted-history"}, + { + "type": "function_call", + "call_id": "call_shell", + "name": "local_shell", + "arguments": json.dumps(_SHELL_ACTION), + }, + {"type": "function_call_output", "call_id": "call_shell", "output": "total 0"}, + ] + + +def _mantle_peer(marker: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/openai/v1/responses", request.target + assert request.headers["authorization"] == f"Bearer {_TOKEN}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["input"] == _mantle_history(marker), json.dumps(body["input"]) + return Reply(body=_RESPONSE) + + return respond + + +@pytest.mark.covers("providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items") +def test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle(gateway: Gateway) -> None: + marker: Final = uuid4().hex + with wire_server(_mantle_peer(marker)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=wire.url, api_key=_TOKEN, aws_region_name="us-east-1") + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": _codex_history(marker), "stream": False} + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["output"] == [ + { + **_OUTPUT_MESSAGE, + "phase": None, + "content": [ + {"type": "output_text", "text": "mantle wire control", "annotations": [], "logprobs": None} + ], + } + ], response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/openai/v1/responses")] diff --git a/tests/integration/providers/test_bedrock_mantle_wire.py b/tests/integration/providers/test_bedrock_mantle_wire.py new file mode 100644 index 00000000000..48cd0d770aa --- /dev/null +++ b/tests/integration/providers/test_bedrock_mantle_wire.py @@ -0,0 +1,53 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "openai.gpt-5.6-sol" +_API_KEY: Final = "synthetic-mantle-bearer" +_PROMPT: Final = "synthetic long conversation control" +_PROMPT_TOKENS: Final = 1055489 +_MODEL_MAXIMUM: Final = 1050000 +_RESPONSES_PATH: Final = "/openai/v1/responses" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OVERFLOW_BODY: Final = json.dumps( + { + "error": { + "code": "validation_error", + "message": f"prompt tokens ({_PROMPT_TOKENS}) exceed model maximum ({_MODEL_MAXIMUM}) for {_BACKEND}", + "type": "invalid_request_error", + } + } +).encode() + + +def _overflow_peer(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == _RESPONSES_PATH + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert _PROMPT in json.dumps(body["input"]), body + return Reply(status=400, body=_OVERFLOW_BODY) + + +@pytest.mark.covers("other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long") +def test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long(gateway: Gateway) -> None: + with wire_server(_overflow_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _PROMPT}]}, + ) + assert response.status_code == 400, response.text + error: Final = _JSON_OBJECT.validate_json(response.content)["error"] + assert isinstance(error, dict), response.text + assert error["code"] == "400", response.text + message: Final = error["message"] + assert isinstance(message, str), response.text + assert f"prompt is too long: {_PROMPT_TOKENS} tokens > {_MODEL_MAXIMUM} maximum" in message, response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)] diff --git a/tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py b/tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py new file mode 100644 index 00000000000..2b10d6f420c --- /dev/null +++ b/tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py @@ -0,0 +1,88 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + +BEDROCK_MODEL: Final = "us.anthropic.claude-opus-5-v1:0" +TOKEN: Final = "synthetic-bedrock-bearer" +SNIPPET: Final = "synthetic snippet about the integration harness" +INTERCEPTED_TURN: Final = ( + {"type": "server_tool_use", "id": "srvtoolu_synthetic", "name": "web_search", "input": {"query": "harness docs"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_synthetic", + "content": [ + { + "type": "web_search_result", + "url": "https://example.test/harness", + "title": "Harness", + "page_age": None, + "encrypted_content": "", + "snippet": SNIPPET, + }, + ], + }, + {"type": "text", "text": "The harness is documented at example.test"}, +) +FLATTENED_TURN: Final = ( + { + "type": "text", + "text": f"Web search results for 'harness docs':\n\nTitle: Harness\nURL: https://example.test/harness\nSnippet: {SNIPPET}", + }, + {"type": "text", "text": "The harness is documented at example.test"}, +) +REPLY: Final = json.dumps( + { + "id": "msg_synthetic_replay", + "type": "message", + "role": "assistant", + "model": BEDROCK_MODEL, + "content": [{"type": "text", "text": "replay accepted"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 30, "output_tokens": 3}, + } +).encode() + + +def bedrock_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == f"/model/{BEDROCK_MODEL}/invoke" + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = json.loads(request.body) + assert body["messages"] == [ + {"role": "user", "content": "where is the harness documented"}, + {"role": "assistant", "content": list(FLATTENED_TURN)}, + {"role": "user", "content": "and what does it say"}, + ], request.body.decode() + assert "tools" not in body, request.body.decode() + return Reply(body=REPLY) + + +@pytest.mark.covers("providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text") +def test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers(gateway: Gateway) -> None: + with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{BEDROCK_MODEL}", + api_key=TOKEN, + api_base=wire.url, + aws_region_name="us-east-1", + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "messages": [ + {"role": "user", "content": "where is the harness documented"}, + {"role": "assistant", "content": list(INTERCEPTED_TURN)}, + {"role": "user", "content": "and what does it say"}, + ], + }, + headers={"x-api-key": gateway.key, "anthropic-version": "2023-06-01"}, + ) + assert response.status_code == 200, response.text + assert response.json()["content"] == [{"type": "text", "text": "replay accepted"}], response.text + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_bedrock_passthrough_stream_wire.py b/tests/integration/providers/test_bedrock_passthrough_stream_wire.py new file mode 100644 index 00000000000..bb9bbc65f30 --- /dev/null +++ b/tests/integration/providers/test_bedrock_passthrough_stream_wire.py @@ -0,0 +1,42 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.upstream import _aws_event_frame +from integration._support.wire import Reply, Request, wire_server + +_MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0" +_EVENT_STREAM: Final = "application/vnd.amazon.eventstream" +_REQUEST_BODY: Final = {"messages": [{"role": "user", "content": [{"text": "synthetic passthrough stream"}]}]} +_EVENTS: Final = ( + ("messageStart", {"role": "assistant"}), + ("contentBlockDelta", {"delta": {"text": "bedrock stream control"}, "contentBlockIndex": 0}), + ("messageStop", {"stopReason": "end_turn"}), + ("metadata", {"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}}), +) +_STREAM_BYTES: Final = b"".join(_aws_event_frame(kind, payload, "sc", "u") for kind, payload in _EVENTS) + + +def event_stream_peer(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == f"/model/{_MODEL_ID}/converse-stream" + assert json.loads(request.body)["messages"] == _REQUEST_BODY["messages"] + return Reply(body=_STREAM_BYTES, content_type=_EVENT_STREAM) + + +@pytest.mark.covers("other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type") +def test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type(gateway: Gateway) -> None: + with wire_server(event_stream_peer) as wire, gateway.scenario() as scenario: + deployment: Final = scenario.model( + model=f"bedrock/{_MODEL_ID}", + api_base=wire.url, + aws_access_key_id="AKIASCRIPTEDPROVIDER", + aws_secret_access_key="scripted-secret", + aws_region_name="us-east-1", + ) + response: Final = gateway.request("POST", f"/bedrock/model/{deployment}/converse-stream", _REQUEST_BODY) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1, response.text + assert response.headers.get("content-type") == _EVENT_STREAM, dict(response.headers) + assert response.content == _STREAM_BYTES, response.text diff --git a/tests/integration/providers/test_bedrock_rerank_wire.py b/tests/integration/providers/test_bedrock_rerank_wire.py new file mode 100644 index 00000000000..86a3bbbd292 --- /dev/null +++ b/tests/integration/providers/test_bedrock_rerank_wire.py @@ -0,0 +1,92 @@ +import json +import os +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +MODEL: Final = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" +ACCESS_KEY: Final = "AKIAINTEGRATION000002" +FORWARDED_FOR: Final = "203.0.113.5" +RESPONSE: Final = json.dumps( + {"results": [{"index": 1, "relevanceScore": 0.9}, {"index": 0, "relevanceScore": 0.1}]} +).encode() + + +def signed_headers(authorization: str) -> tuple[str, ...]: + return tuple(authorization.split("SignedHeaders=")[1].split(",")[0].split(";")) + + +def rerank_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/rerank" + assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/") + assert signed_headers(request.headers["authorization"]) == ("content-type", "host", "x-amz-date"), request.headers[ + "authorization" + ] + assert request.headers["x-forwarded-for"] == FORWARDED_FOR + body: Final = json.loads(request.body) + assert body["queries"] == [{"textQuery": {"text": "synthetic rerank query"}, "type": "TEXT"}] + assert body["rerankingConfiguration"]["bedrockRerankingConfiguration"]["modelConfiguration"] == { + "modelArn": "arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" + } + assert body["rerankingConfiguration"]["bedrockRerankingConfiguration"]["numberOfResults"] == 2 + return Reply(body=RESPONSE) + + +@pytest.mark.covers("providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned") +def test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature( + gateway: Gateway, tmp_path: Path +) -> None: + empty: Final = tmp_path / "empty-aws-config" + empty.write_text("") + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["general_settings"]["forward_client_headers_to_llm_api"] = True + path: Final = tmp_path / "forwarding.yaml" + path.write_text(yaml.safe_dump(configuration)) + overrides: Final = { + "AWS_CONFIG_FILE": str(empty), + "AWS_SHARED_CREDENTIALS_FILE": str(empty), + "AWS_EC2_METADATA_DISABLED": "true", + "LITELLM_RUST": "false", + } + with wire_server(rerank_peer) as wire: + with ( + owned_proxy( + gateway, + tmp_path, + overrides, + config=path, + remove_environment=tuple(name for name in os.environ if name.startswith("AWS_")), + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model( + model=MODEL, + api_key=None, + api_base=None, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key="synthetic-rerank-secret-key-for-testing", + ) + response: Final = candidate.request( + "POST", + "/v1/rerank", + { + "model": model, + "query": "synthetic rerank query", + "documents": ["first synthetic document", "second synthetic document"], + "top_n": 2, + }, + headers={"x-forwarded-for": FORWARDED_FOR}, + ) + assert response.status_code == 200, response.text + assert response.json()["results"] == [ + {"index": 1, "relevance_score": 0.9}, + {"index": 0, "relevance_score": 0.1}, + ], response.text + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_bedrock_thinking_tokens_wire.py b/tests/integration/providers/test_bedrock_thinking_tokens_wire.py new file mode 100644 index 00000000000..074adeb41f6 --- /dev/null +++ b/tests/integration/providers/test_bedrock_thinking_tokens_wire.py @@ -0,0 +1,94 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +MODEL: Final = "bedrock/converse/global.anthropic.claude-opus-4-8" +TOKEN: Final = "synthetic-bedrock-bearer" +PROMPT: Final = "How many prime numbers are less than 30? Think it through, then answer with just the number." +RESPONSES_PROMPT: Final = "How many prime numbers are less than 30? Answer with just the number." +REDACTED_DATA: Final = "RWRhY3RlZC1ieS1CZWRyb2Nr" +INPUT_TOKENS: Final = 31 +OUTPUT_TOKENS: Final = 257 +RESPONSE: Final = json.dumps( + { + "output": { + "message": { + "role": "assistant", + "content": [{"reasoningContent": {"redactedContent": REDACTED_DATA}}, {"text": "10"}], + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": INPUT_TOKENS, + "outputTokens": OUTPUT_TOKENS, + "totalTokens": INPUT_TOKENS + OUTPUT_TOKENS, + }, + "metrics": {"latencyMs": 1}, + } +).encode() +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_LIST: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def redacted_thinking_peer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/model/global.anthropic.claude-opus-4-8/converse" + assert request.headers["authorization"] == f"Bearer {TOKEN}" + body: Final = json.loads(request.body) + assert body["messages"] in ( + [{"role": "user", "content": [{"text": PROMPT}]}], + [{"role": "user", "content": [{"text": RESPONSES_PROMPT}]}], + ), body + assert body["additionalModelRequestFields"]["thinking"]["type"] == "adaptive", body + return Reply(body=RESPONSE) + + +@pytest.mark.covers("other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text") +def test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens(gateway: Gateway) -> None: + with wire_server(redacted_thinking_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=MODEL, api_key=TOKEN, aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url + ) + chat: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": PROMPT}], + "max_tokens": 4000, + "reasoning_effort": "max", + }, + ) + assert chat.status_code == 200, chat.text + chat_body: Final = _JSON_OBJECT.validate_json(chat.content) + message: Final = _JSON_OBJECT.validate_python(_JSON_LIST.validate_python(chat_body["choices"])[0]["message"]) + assert message["content"] == "10", chat.text + assert message["thinking_blocks"] == [{"type": "redacted_thinking", "data": REDACTED_DATA}], chat.text + usage: Final = _JSON_OBJECT.validate_python(chat_body["usage"]) + assert usage["completion_tokens"] == OUTPUT_TOKENS, chat.text + details: Final = _JSON_OBJECT.validate_python(usage["completion_tokens_details"]) + assert details == {}, chat.text + assert len(wire.drain()) == 1 + + responses: Final = gateway.request( + "POST", + "/v1/responses", + {"model": model, "input": RESPONSES_PROMPT, "max_output_tokens": 4000, "reasoning": {"effort": "max"}}, + ) + assert responses.status_code == 200, responses.text + responses_body: Final = _JSON_OBJECT.validate_json(responses.content) + output: Final = _JSON_LIST.validate_python(responses_body["output"]) + reasoning_items: Final = tuple(item for item in output if item["type"] == "reasoning") + assert len(reasoning_items) == 1, responses.text + assert reasoning_items[0]["encrypted_content"] == json.dumps( + [{"type": "redacted_thinking", "data": REDACTED_DATA}], separators=(",", ":") + ), responses.text + responses_usage: Final = _JSON_OBJECT.validate_python(responses_body["usage"]) + assert responses_usage["output_tokens"] == OUTPUT_TOKENS, responses.text + assert _JSON_OBJECT.validate_python(responses_usage["output_tokens_details"])["reasoning_tokens"] == 0, ( + responses.text + ) + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_dashscope_chat_wire.py b/tests/integration/providers/test_dashscope_chat_wire.py new file mode 100644 index 00000000000..a2b3a36d6e3 --- /dev/null +++ b/tests/integration/providers/test_dashscope_chat_wire.py @@ -0,0 +1,62 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3.7-plus" +_API_KEY: Final = "synthetic-dashscope-key" +_PROMPT: Final = "What is 3^3?" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _completion(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "27"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 17, "completion_tokens": 5, "total_tokens": 22}, + } + ).encode() + + +@pytest.mark.covers("other.provider_wire.dashscope.reasoning_effort_reaches_provider") +def test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider(gateway: Gateway) -> None: + identity: Final = f"dashscope-reasoning-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _BACKEND, + "messages": [{"role": "user", "content": _PROMPT}], + "reasoning_effort": "none", + } + return Reply(body=_completion(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"dashscope/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _PROMPT}], "reasoning_effort": "none"}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": {"role": "assistant", "content": "27", "provider_specific_fields": {"refusal": None}}, + "provider_specific_fields": {}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] diff --git a/tests/integration/providers/test_databricks_chat_wire.py b/tests/integration/providers/test_databricks_chat_wire.py new file mode 100644 index 00000000000..614382a77f0 --- /dev/null +++ b/tests/integration/providers/test_databricks_chat_wire.py @@ -0,0 +1,129 @@ +import json +import uuid +from collections.abc import Mapping +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_BACKEND: Final = "databricks-glm-5-2" +_API_KEY: Final = "synthetic-databricks-key" +_PROMPT: Final = "Summarise the cached briefing in one sentence." +_PROVIDER_USAGE: Final[Mapping[str, JsonValue]] = { + "prompt_tokens": 12011, + "completion_tokens": 8, + "total_tokens": 12019, + "cache_read_input_tokens": 12002, + "cache_creation_input_tokens": 0, +} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +class _PromptTokensDetails(BaseModel): + model_config = ConfigDict(extra="ignore") + cached_tokens: int | None = None + + +class _Usage(BaseModel): + model_config = ConfigDict(extra="ignore") + prompt_tokens: int + completion_tokens: int + total_tokens: int + prompt_tokens_details: _PromptTokensDetails | None = None + + +class _Delta(BaseModel): + model_config = ConfigDict(extra="ignore") + content: str | None = None + + +class _Choice(BaseModel): + model_config = ConfigDict(extra="ignore") + delta: _Delta + + +class _Chunk(BaseModel): + model_config = ConfigDict(extra="ignore") + id: str + choices: tuple[_Choice, ...] + usage: _Usage | None = None + + +def _frame(identity: str, choices: list[Mapping[str, object]], usage: Mapping[str, JsonValue] | None = None) -> bytes: + value: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _BACKEND, + "choices": choices, + **({} if usage is None else {"usage": usage}), + } + return b"data: " + json.dumps(value).encode() + b"\n\n" + + +@pytest.mark.covers("other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log") +def test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log(gateway: Gateway) -> None: + identity: Final = f"databricks-stream-{uuid.uuid4().hex}" + frames: Final = ( + _frame( + identity, [{"index": 0, "delta": {"role": "assistant", "content": "The briefing "}, "finish_reason": None}] + ), + _frame(identity, [{"index": 0, "delta": {"content": "is short."}, "finish_reason": None}]), + _frame(identity, [{"index": 0, "delta": {}, "finish_reason": "stop"}]), + _frame(identity, [], usage=_PROVIDER_USAGE), + b"data: [DONE]\n\n", + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["messages"] == [{"role": "user", "content": _PROMPT}] + assert body["stream"] is True + return Reply(content_type="text/event-stream", chunks=frames) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": _PROMPT}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: ")) + assert lines[-1] == "data: [DONE]", lines + chunks: Final = tuple(_Chunk.model_validate_json(line.removeprefix("data: ")) for line in lines[:-1]) + assert {chunk.id for chunk in chunks} == {identity} + assert ( + "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + == "The briefing is short." + ) + usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None) + assert len(usages) == 1, lines + assert ( + usages[0].prompt_tokens, + usages[0].completion_tokens, + usages[0].total_tokens, + usages[0].prompt_tokens_details.cached_tokens if usages[0].prompt_tokens_details is not None else None, + ) == (12011, 8, 12019, 12002), lines + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + rows: Final = eventually( + lambda: read_rows( + 'SELECT prompt_tokens, completion_tokens, total_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"], rows[0]["total_tokens"]) == (12011, 8, 12019) diff --git a/tests/integration/providers/test_databricks_oauth_wire.py b/tests/integration/providers/test_databricks_oauth_wire.py new file mode 100644 index 00000000000..7dbc5f17838 --- /dev/null +++ b/tests/integration/providers/test_databricks_oauth_wire.py @@ -0,0 +1,92 @@ +import base64 +import json +import uuid +from pathlib import Path +from typing import Final +from urllib.parse import parse_qs + +import pytest +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "databricks/synthetic-vendor.chat-model.v1" +_CLIENT_ID: Final = "synthetic-databricks-client-id" +_CLIENT_SECRET: Final = "synthetic-databricks-client-secret" +_ACCESS_TOKEN: Final = "synthetic-databricks-oauth-token" +_PROMPT: Final = "Which workspace issued this token?" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _basic_credentials(client_id: str, client_secret: str) -> str: + return "Basic " + base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() + + +def _completion(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _MODEL.removeprefix("databricks/"), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "the workspace origin"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}, + } + ).encode() + + +@pytest.mark.covers("other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base") +def test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin( + gateway: Gateway, tmp_path: Path +) -> None: + identity: Final = f"databricks-oauth-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.target == "/oidc/v1/token": + assert request.method == "POST" + assert request.headers["authorization"] == _basic_credentials(_CLIENT_ID, _CLIENT_SECRET) + assert request.headers["content-type"] == "application/x-www-form-urlencoded" + assert parse_qs(request.body.decode()) == {"grant_type": ["client_credentials"], "scope": ["all-apis"]} + return Reply( + body=json.dumps({"access_token": _ACCESS_TOKEN, "token_type": "Bearer", "expires_in": 3600}).encode() + ) + if request.target == "/ai-gateway/mlflow/v1/chat/completions": + assert request.method == "POST" + assert request.headers["authorization"] == f"Bearer {_ACCESS_TOKEN}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _MODEL.removeprefix("databricks/") + assert body["messages"] == [{"role": "user", "content": _PROMPT}] + return Reply(body=_completion(identity)) + return Reply(status=401, body=json.dumps({"error": f"unauthenticated path {request.target}"}).encode()) + + overrides: Final = {"DATABRICKS_CLIENT_ID": _CLIENT_ID, "DATABRICKS_CLIENT_SECRET": _CLIENT_SECRET} + with wire_server(respond) as wire, owned_proxy(gateway, tmp_path, overrides) as candidate: + with candidate.scenario() as scenario: + model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/ai-gateway/mlflow/v1", api_key=None) + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _PROMPT}]}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": "the workspace origin", "role": "assistant"}, + } + ] + assert payload["usage"] == {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13} + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/oidc/v1/token"), + ("POST", "/ai-gateway/mlflow/v1/chat/completions"), + ] diff --git a/tests/integration/providers/test_deepseek_vision_wire.py b/tests/integration/providers/test_deepseek_vision_wire.py new file mode 100644 index 00000000000..25ddaf1aa02 --- /dev/null +++ b/tests/integration/providers/test_deepseek_vision_wire.py @@ -0,0 +1,40 @@ +from typing import Final + +import httpx +import pytest +from pydantic import JsonValue + +from tests.integration._support.client import JSON_OBJECT, Gateway, object_value + +_VISION_MODEL: Final = "deepseek-v4-flash-vision-exp" +_API_KEY: Final = "synthetic-deepseek-key" +_VISION_CONTENT: Final[JsonValue] = [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}}, +] + + +@pytest.mark.covers("other.provider_wire.deepseek.vision_image_content_list_reaches_provider") +def test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text(gateway: Gateway) -> None: + with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream: + upstream.get("/__observations").raise_for_status() + model: Final = scenario.model( + model=f"deepseek/{_VISION_MODEL}", + api_key=_API_KEY, + model_info={"mode": "chat", "supports_vision": True}, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _VISION_CONTENT}]}, + ) + assert response.status_code == 200, response.text + observations: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"] + assert isinstance(observations, list) + assert len(observations) == 1, response.text + observed: Final = object_value(observations[0]) + assert observed["path"] == "/v1/chat/completions", response.text + assert observed["authorization"] == f"Bearer {_API_KEY}", response.text + body: Final = object_value(observed["body"]) + assert body["model"] == _VISION_MODEL, response.text + assert body["messages"] == [{"role": "user", "content": _VISION_CONTENT}], response.text diff --git a/tests/integration/providers/test_fireworks_ai_router_slug_wire.py b/tests/integration/providers/test_fireworks_ai_router_slug_wire.py new file mode 100644 index 00000000000..4b4ad0b1243 --- /dev/null +++ b/tests/integration/providers/test_fireworks_ai_router_slug_wire.py @@ -0,0 +1,84 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_ROUTER_SLUG: Final = "routers/glm-latest" +_ROUTER_RESOURCE: Final = "accounts/fireworks/routers/glm-latest" +_API_KEY: Final = "synthetic-fireworks-key" +_PROMPT: Final = "route me through the router" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _provider_body(request: Request, target: str) -> dict[str, JsonValue]: + assert request.method == "POST" + assert request.target == target + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + return _JSON_OBJECT.validate_json(request.body) + + +@pytest.mark.covers("other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name") +def test_fireworks_router_slug_chat_sends_router_resource_not_models_path(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + body: Final = _provider_body(request, "/chat/completions") + assert body["model"] == _ROUTER_RESOURCE, body + assert body["messages"] == [{"role": "user", "content": _PROMPT}] + return Reply( + body=json.dumps( + { + "id": "fw-router-chat", + "object": "chat.completion", + "created": 1, + "model": _ROUTER_RESOURCE, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fireworks_ai/{_ROUTER_SLUG}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _PROMPT}]}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [ + {"finish_reason": "stop", "index": 0, "message": {"role": "assistant", "content": "routed"}} + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + + +@pytest.mark.covers("other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name") +def test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + body: Final = _provider_body(request, "/completions") + assert body["model"] == _ROUTER_RESOURCE, body + assert body["prompt"] == _PROMPT + return Reply( + body=json.dumps( + { + "id": "fw-router-text", + "object": "text_completion", + "created": 1, + "model": _ROUTER_RESOURCE, + "choices": [{"index": 0, "text": "routed", "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fireworks_ai/{_ROUTER_SLUG}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/completions", {"model": model, "prompt": _PROMPT}) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [{"index": 0, "text": "routed", "finish_reason": "stop", "logprobs": None}] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/completions")] diff --git a/tests/integration/providers/test_openai_chat_wire.py b/tests/integration/providers/test_openai_chat_wire.py new file mode 100644 index 00000000000..24d7d83e519 --- /dev/null +++ b/tests/integration/providers/test_openai_chat_wire.py @@ -0,0 +1,66 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "gpt-5.4-mini" +_API_KEY: Final = "synthetic-openai-key" +_PROMPT: Final = "Summarize this conversation in one sentence." +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _completion(identity: str, content: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 19, "completion_tokens": 7, "total_tokens": 26}, + } + ).encode() + + +@pytest.mark.covers("providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire") +def test_openai_chat_tool_choice_without_tools_is_not_forwarded(gateway: Gateway) -> None: + identity: Final = f"openai-toolless-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["messages"] == [{"role": "user", "content": _PROMPT}] + assert "tool_choice" not in body, body + assert "tools" not in body, body + return Reply(body=_completion(identity, "One sentence.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _PROMPT}], "tool_choice": "none"}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "One sentence.", + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] diff --git a/tests/integration/providers/test_openai_image_edit_wire.py b/tests/integration/providers/test_openai_image_edit_wire.py new file mode 100644 index 00000000000..501cfaa2a25 --- /dev/null +++ b/tests/integration/providers/test_openai_image_edit_wire.py @@ -0,0 +1,75 @@ +import json +from email.message import Message +from email.parser import BytesParser +from email.policy import HTTP +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import BaseModel + +_PNG_BYTES: Final = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00" + b"\x1f\x15\xc4\x89\x00\x00\x00\rIDAT\x08\xd7c\xf8\xcf\xc0\xf0\x1f\x00\x05\x00\x01\xff" + b"\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82" +) +_PROMPT: Final = "turn the red circle green" +_EDITED_IMAGE_B64: Final = "aW50ZWdyYXRpb24tZWRpdGVkLWltYWdl" + + +class _Image(BaseModel): + b64_json: str + + +class _ImageResponse(BaseModel): + data: tuple[_Image, ...] + + +def _multipart_parts(request: Request) -> tuple[Message, ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope) + assert parsed.is_multipart(), request.headers["content-type"] + return tuple(parsed.iter_parts()) + + +def _text_fields(parts: tuple[Message, ...]) -> dict[str, str]: + return { + part.get_param("name", header="content-disposition"): part.get_payload(decode=True).decode() + for part in parts + if part.get_filename() is None + } + + +def _file_fields(parts: tuple[Message, ...]) -> dict[str, bytes]: + return { + part.get_param("name", header="content-disposition"): part.get_payload(decode=True) + for part in parts + if part.get_filename() is not None + } + + +@pytest.mark.covers("other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields") +def test_openai_compatible_image_edit_forwards_seed_form_field_to_backend(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/images/edits" + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + parts: Final = _multipart_parts(request) + assert _text_fields(parts) == {"model": "gpt-image-1", "prompt": _PROMPT, "seed": "42"} + assert _file_fields(parts) == {"image[]": _PNG_BYTES} + return Reply(body=json.dumps({"created": 1700000000, "data": [{"b64_json": _EDITED_IMAGE_B64}]}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-image-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + response: Final = gateway.request_multipart( + "/v1/images/edits", + {"model": model, "prompt": _PROMPT, "seed": "42"}, + {"image": ("red_circle.png", _PNG_BYTES, "image/png")}, + ) + assert response.status_code == 200, response.text + payload: Final = _ImageResponse.model_validate_json(response.content) + assert [image.b64_json for image in payload.data] == [_EDITED_IMAGE_B64], response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/images/edits")] diff --git a/tests/integration/providers/test_responses_bridge_incomplete.py b/tests/integration/providers/test_responses_bridge_incomplete.py new file mode 100644 index 00000000000..3352ca5775e --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_incomplete.py @@ -0,0 +1,64 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + + +@pytest.mark.covers("other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length") +def test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out(gateway: Gateway) -> None: + identity: Final = "responses-incomplete-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses", request.target + assert request.headers["authorization"] == "Bearer synthetic-openai-key" + body: Final = json.loads(request.body) + assert body["model"] == "gpt-5.3-codex" + assert body["max_output_tokens"] == 16 + assert body["reasoning"] == {"effort": "high"} + assert body["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": f"explain the plan in detail {identity}"}], + } + ] + return Reply( + body=json.dumps( + { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + "model": "gpt-5.3-codex", + "output": [{"type": "reasoning", "id": f"rs_{identity}", "summary": []}], + "usage": {"input_tokens": 12, "output_tokens": 16, "total_tokens": 28}, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/responses/gpt-5.3-codex", api_base=wire.url, api_key="synthetic-openai-key" + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"explain the plan in detail {identity}"}], + "reasoning_effort": "high", + "max_completion_tokens": 16, + }, + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert len(wire.drain()) == 1 + assert [choice["finish_reason"] for choice in body["choices"]] == ["length"], response.text + assert body["choices"][0]["message"]["content"] == "", response.text + assert body["choices"][0]["message"]["role"] == "assistant", response.text + assert body["usage"]["prompt_tokens"] == 12 and body["usage"]["completion_tokens"] == 16, response.text + assert body["usage"]["total_tokens"] == 28, response.text diff --git a/tests/integration/providers/test_tencent_chat_wire.py b/tests/integration/providers/test_tencent_chat_wire.py new file mode 100644 index 00000000000..84e9eb8ffea --- /dev/null +++ b/tests/integration/providers/test_tencent_chat_wire.py @@ -0,0 +1,86 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "deepseek-v4-pro" +_API_KEY: Final = "synthetic-tencent-key" +_PROMPT: Final = "What is 17 + 26? Answer with just the number." +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_REASONING_REQUESTS: Final[tuple[tuple[str, dict[str, JsonValue], dict[str, JsonValue]], ...]] = ( + ("thinking_enabled", {"thinking": {"type": "enabled"}}, {"type": "enabled"}), + ("reasoning_effort_none", {"reasoning_effort": "none"}, {"type": "disabled"}), +) + + +def _completion(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43."}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}, + } + ).encode() + + +@pytest.mark.covers("other.provider_wire.tencent.thinking_reaches_provider_in_request_body") +@pytest.mark.parametrize( + ("reasoning_params", "expected_thinking"), + tuple(case[1:] for case in _REASONING_REQUESTS), + ids=tuple(case[0] for case in _REASONING_REQUESTS), +) +def test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request( + gateway: Gateway, reasoning_params: dict[str, JsonValue], expected_thinking: dict[str, JsonValue] +) -> None: + identity: Final = f"tencent-thinking-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert request.headers["content-type"] == "application/json" + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _BACKEND, + "messages": [{"role": "user", "content": _PROMPT}], + "thinking": expected_thinking, + } + return Reply(body=_completion(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"tencent/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": _PROMPT}], **reasoning_params}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "43", + "reasoning_content": "17 plus 26 is 43.", + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert payload["usage"] == {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64} + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] diff --git a/tests/integration/providers/test_websearch_interception_wire.py b/tests/integration/providers/test_websearch_interception_wire.py new file mode 100644 index 00000000000..cba4f174233 --- /dev/null +++ b/tests/integration/providers/test_websearch_interception_wire.py @@ -0,0 +1,299 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +BEDROCK_MODEL: Final = "us.anthropic.claude-haiku-4-5-20251001-v1:0" +INVOKE_TARGET: Final = f"/model/{BEDROCK_MODEL}/invoke" +SEARCH_TARGET: Final = "/tavily/search" +SEARCH_RESULT: Final = { + "title": "Synthetic result", + "url": "https://example.test/result", + "content": "the snippet text", +} + + +def sse_events(text: str) -> tuple[tuple[str, dict[str, object]], ...]: + frames: Final = tuple(frame for frame in text.split("\n\n") if frame.strip()) + return tuple( + ( + next(line.removeprefix("event: ") for line in frame.splitlines() if line.startswith("event: ")), + json.loads(next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: "))), + ) + for frame in frames + ) + + +@pytest.mark.covers("other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results") +def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks( + gateway: Gateway, tmp_path: Path +) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST", request.target + body: Final = json.loads(request.body) + if request.target == SEARCH_TARGET: + assert request.headers["authorization"] == "Bearer synthetic-tavily-key" + assert body["query"] == "query-0", body + return Reply(body=json.dumps({"query": "query-0", "results": [SEARCH_RESULT]}).encode()) + assert request.target == INVOKE_TARGET + assert request.headers["authorization"] == "Bearer synthetic-bedrock-token" + assert [tool["name"] for tool in body["tools"]] == ["litellm_web_search"], body["tools"] + assert "stream" not in body, body + depth: Final = sum( + 1 + for message in body["messages"] + if isinstance(message["content"], list) + for block in message["content"] + if block["type"] == "tool_result" + ) + if depth == 1: + assert body["messages"][2]["content"] == [ + { + "type": "tool_result", + "tool_use_id": "toolu_0", + "content": "Title: Synthetic result\nURL: https://example.test/result\nSnippet: the snippet text", + } + ], body["messages"] + return Reply( + body=json.dumps( + { + "id": f"msg_{depth}", + "type": "message", + "role": "assistant", + "model": BEDROCK_MODEL, + "content": [ + {"type": "text", "text": f"turn-{depth}"}, + { + "type": "tool_use", + "id": f"toolu_{depth}", + "name": "litellm_web_search", + "input": {"query": f"query-{depth}"}, + }, + ], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + ).encode() + ) + + with wire_server(respond) as wire: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["search_tools"] = [ + { + "search_tool_name": "integration-search", + "litellm_params": { + "search_provider": "tavily", + "api_key": "synthetic-tavily-key", + "api_base": wire.url + "/tavily", + }, + } + ] + config["litellm_settings"].update( + { + "callbacks": ["websearch_interception"], + "websearch_interception_params": { + "enabled_providers": ["bedrock"], + "search_tool_name": "integration-search", + "max_agentic_loops": 1, + }, + } + ) + path: Final = tmp_path / "websearch.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{BEDROCK_MODEL}", + api_key="synthetic-bedrock-token", + api_base=wire.url, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + ) + response: Final = candidate.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "stream": True, + "messages": [{"role": "user", "content": "search control"}], + "tools": [{"type": "web_search_20250305", "name": "web_search"}], + }, + ) + assert response.status_code == 200, response.text + events: Final = sse_events(response.text) + assert [name for name, _ in events][:1] == ["message_start"], response.text + assert [name for name, _ in events][-2:] == ["message_delta", "message_stop"], response.text + for position, (name, event) in enumerate(events): + if name == "content_block_stop": + assert event["index"] in { + earlier_event["index"] + for earlier, earlier_event in events[:position] + if earlier == "content_block_start" + }, response.text + started: Final = tuple(event["content_block"] for name, event in events if name == "content_block_start") + search_ids: Final = tuple(block["id"] for block in started if block["type"] == "server_tool_use") + assert search_ids and all(search_id.startswith("srvtoolu_") for search_id in search_ids), response.text + assert started[-1] == {"type": "text", "text": ""}, response.text + assert started[:-1] == tuple( + block + for search_id in search_ids + for block in ( + {"type": "server_tool_use", "id": search_id, "name": "web_search", "input": {"query": "query-0"}}, + { + "type": "web_search_tool_result", + "tool_use_id": search_id, + "content": [ + { + "type": "web_search_result", + "url": "https://example.test/result", + "title": "Synthetic result", + "page_age": None, + "encrypted_content": "", + "snippet": "the snippet text", + } + ], + }, + ) + ), response.text + assert ( + "".join(event["delta"]["text"] for name, event in events if name == "content_block_delta") == "turn-1" + ), response.text + assert [event["delta"]["stop_reason"] for name, event in events if name == "message_delta"] == [ + "end_turn" + ], response.text + assert "litellm_web_search" not in response.text, response.text + assert [request.target for request in wire.drain()] == [INVOKE_TARGET, SEARCH_TARGET, INVOKE_TARGET] + + +import threading +import uuid +from typing import Final +from urllib.parse import parse_qs, urlsplit + +import httpx +import pytest +from integration._support.client import Gateway, eventually + +_QUERY: Final = "integration capped search" +_TEXT_BLOCK: Final = {"type": "text", "text": "searching once more"} +_NOT_INTERCEPTED: Final = "native tool reached the provider" +_SEARCH_RESULT_BLOCK: Final = { + "type": "web_search_result", + "url": "https://owned.invalid/a", + "title": "Owned result", + "page_age": None, + "encrypted_content": "", + "snippet": "owned snippet", +} + + +def _search_tool_use(identity: str) -> dict[str, object]: + return {"type": "tool_use", "id": identity, "name": "litellm_web_search", "input": {"query": _QUERY}} + + +def _anthropic_reply(identity: str, content: list[dict[str, object]], stop_reason: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + ).encode() + ) + + +@pytest.mark.covers( + "other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use" +) +def test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use( + gateway: Gateway, tmp_path: Path +) -> None: + identity: Final = "websearch-wire-" + uuid.uuid4().hex + searched: Final = threading.Event() + + def respond(request: Request) -> Reply: + parts: Final = urlsplit(request.target) + if request.method == "GET" and parts.path == "/search": + assert parse_qs(parts.query)["q"] == [_QUERY], request.target + searched.set() + return Reply( + body=json.dumps( + { + "results": [ + {"title": "Owned result", "url": "https://owned.invalid/a", "content": "owned snippet"} + ] + } + ).encode() + ) + assert request.method == "POST" and parts.path == "/v1/messages", request.target + body: Final = json.loads(request.body) + if any(tool.get("type") == "web_search_20250305" for tool in body["tools"]): + return _anthropic_reply(identity, [{"type": "text", "text": _NOT_INTERCEPTED}], "end_turn") + assert [tool["name"] for tool in body["tools"]] == ["litellm_web_search"], body["tools"] + return _anthropic_reply(identity, [_TEXT_BLOCK, _search_tool_use(identity)], "tool_use") + + def send(candidate: Gateway, model: str) -> httpx.Response: + return candidate.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "messages": [{"role": "user", "content": identity + " attempt " + uuid.uuid4().hex}], + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 3}], + }, + ) + + def searched_through_proxy(response: httpx.Response) -> bool: + return searched.is_set() and _NOT_INTERCEPTED not in response.text + + with wire_server(respond) as wire: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["search_tools"] = [ + { + "search_tool_name": "integration-searxng", + "litellm_params": {"search_provider": "searxng", "api_base": wire.url}, + } + ] + config["litellm_settings"].update( + { + "callbacks": ["websearch_interception"], + "websearch_interception_params": { + "enabled": True, + "enabled_providers": ["anthropic"], + "search_tool_name": "integration-searxng", + }, + } + ) + path: Final = tmp_path / "websearch.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-5-20250929", api_base=wire.url, api_key="synthetic-anthropic-key" + ) + response: Final = eventually(lambda: send(candidate, model), searched_through_proxy, seconds=40) + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["stop_reason"] == "end_turn", response.text + content: Final = body["content"] + assert [block["type"] for block in content] == ["server_tool_use", "web_search_tool_result", "text"], ( + response.text + ) + assert content[0]["name"] == "web_search" and content[0]["input"] == {"query": _QUERY}, response.text + assert content[1]["tool_use_id"] == content[0]["id"], response.text + assert content[1]["content"] == [_SEARCH_RESULT_BLOCK], response.text + assert content[2] == _TEXT_BLOCK, response.text + targets: Final = tuple((request.method, urlsplit(request.target).path) for request in wire.drain()) + assert targets[-3:] == (("POST", "/v1/messages"), ("GET", "/search"), ("POST", "/v1/messages")), targets diff --git a/tests/integration/providers/test_xai_web_search_wire.py b/tests/integration/providers/test_xai_web_search_wire.py new file mode 100644 index 00000000000..1f3a7909047 --- /dev/null +++ b/tests/integration/providers/test_xai_web_search_wire.py @@ -0,0 +1,84 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "grok-4.6-web-search-unmapped" +_API_KEY: Final = "synthetic-xai-key" +_SYSTEM_PROMPT: Final = "Answer in one short sentence and cite the source." +_ALLOWED_DOMAINS: Final = ("weather.example.com", "news.example.org") +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _responses_reply(identity: str, text: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _BACKEND, + "output": [ + { + "type": "message", + "id": f"msg-{identity}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [{"type": "web_search"}], + "usage": {"input_tokens": 23, "output_tokens": 41, "total_tokens": 64}, + } + ).encode() + + +@pytest.mark.covers("other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters") +def test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters(gateway: Gateway) -> None: + identity: Final = f"xai-web-search-{uuid.uuid4().hex}" + user_prompt: Final = f"What is the weather in Paris today? Request {identity}." + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/responses", request.target + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["instructions"] == _SYSTEM_PROMPT + assert body["input"] == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": user_prompt}]} + ] + assert body["tools"] == [{"type": "web_search", "filters": {"allowed_domains": list(_ALLOWED_DOMAINS)}}] + assert "web_search_options" not in body + return Reply(body=_responses_reply(identity, "Sunny, 21C.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xai/{_BACKEND}", api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], + "web_search_options": {"filters": {"allowed_domains": list(_ALLOWED_DOMAINS)}}, + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + choices: Final = payload["choices"] + assert isinstance(choices, list) and len(choices) == 1, response.text + choice: Final = choices[0] + assert isinstance(choice, dict), response.text + message: Final = choice["message"] + assert isinstance(message, dict), response.text + assert (message["role"], message["content"]) == ("assistant", "Sunny, 21C."), response.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/responses")] diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index a3b07f76d2f..a3d29e42120 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -5,6 +5,7 @@ general_settings: store_model_in_db: true disable_spend_logs: false proxy_batch_write_at: 1 + proxy_batch_polling_interval: 1 litellm_settings: enable_redis_auth_cache: true cache: true @@ -14,3 +15,11 @@ litellm_settings: port: os.environ/REDIS_PORT router_settings: disable_cooldowns: true +vector_store_registry: + - vector_store_name: integration-config-store + litellm_params: + vector_store_id: vs_integration_config_store + custom_llm_provider: openai + api_base: os.environ/INTEGRATION_UPSTREAM_URL + api_key: integration-provider-key + vector_store_description: declared in tests/integration/proxy_config.yaml diff --git a/tests/integration/routing/test_priority_rate_limit_headers.py b/tests/integration/routing/test_priority_rate_limit_headers.py new file mode 100644 index 00000000000..2d21f3ba8b2 --- /dev/null +++ b/tests/integration/routing/test_priority_rate_limit_headers.py @@ -0,0 +1,88 @@ +import json +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" +MODEL_RPM: Final = 40 +MODEL_TPM: Final = 1000 +PREMIUM_SHARE: Final = 0.5 +UPSTREAM_REPLY: Final = json.dumps( + { + "id": "msg_priority_headers", + "type": "message", + "role": "assistant", + "model": ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "priority header control"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 4}, + } +).encode() + + +@pytest.mark.covers("other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers") +def test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers( + gateway: Gateway, tmp_path: Path +) -> None: + probe: Final = "priority header probe " + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages" + assert request.headers["x-api-key"] == "synthetic-anthropic-key" + assert json.loads(request.body) == { + "model": ANTHROPIC_MODEL, + "messages": [{"role": "user", "content": probe}], + "max_tokens": 16, + "stream": False, + } + return Reply(body=UPSTREAM_REPLY) + + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["litellm_settings"] = { + **configuration["litellm_settings"], + "callbacks": ["dynamic_rate_limiter_v3"], + "priority_reservation": {"premium": PREMIUM_SHARE}, + } + path: Final = tmp_path / "priority.yaml" + path.write_text(yaml.safe_dump(configuration)) + with ( + wire_server(respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=path) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model( + model=f"anthropic/{ANTHROPIC_MODEL}", + api_base=wire.url, + api_key="synthetic-anthropic-key", + rpm=MODEL_RPM, + tpm=MODEL_TPM, + ) + key: Final = scenario.key(metadata={"priority": "premium"}) + response: Final = candidate.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": probe}]}, + key=key, + ) + assert response.status_code == 200, response.text + assert response.json()["content"] == [{"type": "text", "text": "priority header control"}], response.text + assert len(wire.drain()) == 1 + expected: Final = { + "x-litellm-priority": "premium", + "x-litellm-rate-limiter-version": "v3", + "x-ratelimit-model_saturation_check-limit-requests": str(MODEL_RPM), + "x-ratelimit-model_saturation_check-remaining-requests": str(MODEL_RPM - 1), + "x-ratelimit-priority_model-limit-requests": str(int(MODEL_RPM * PREMIUM_SHARE)), + "x-ratelimit-priority_model-remaining-requests": str(int(MODEL_RPM * PREMIUM_SHARE) - 1), + "x-ratelimit-priority_model-limit-tokens": str(int(MODEL_TPM * PREMIUM_SHARE)), + "x-ratelimit-priority_model-remaining-tokens": str(int(MODEL_TPM * PREMIUM_SHARE) - 1), + } + observed: Final = {name: response.headers.get(name) for name in expected} + assert observed == expected, response.headers diff --git a/tests/integration/routing/test_stale_cost_map_boot.py b/tests/integration/routing/test_stale_cost_map_boot.py new file mode 100644 index 00000000000..d2eeb2bdc6c --- /dev/null +++ b/tests/integration/routing/test_stale_cost_map_boot.py @@ -0,0 +1,86 @@ +import json +import threading +import uuid +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + + +def _proxy_config(directory: Path, model: str, upstream_url: str) -> Path: + config: Final = directory / "stale_cost_map_config.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": model, + "litellm_params": {"model": model, "api_base": upstream_url + "/v1", "api_key": "sk-upstream"}, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + }, + "router_settings": {"disable_cooldowns": True}, + } + ) + ) + return config + + +@pytest.mark.covers("other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload") +def test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload( + gateway: Gateway, tmp_path: Path +) -> None: + model: Final = "integration-fresh-" + uuid.uuid4().hex + remote_map: Final = json.dumps( + {model: {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0, "output_cost_per_token": 0}} + ).encode() + fresh_map_published: Final = threading.Event() + + def respond(request: Request) -> Reply: + assert request.target == "/model_prices.json", request + return Reply(body=remote_map) if fresh_map_published.is_set() else Reply(status=503, body=b"{}") + + overrides: Final = {"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"} + with ( + wire_server(respond) as peer, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + config: Final = _proxy_config(tmp_path, model, gateway.upstream_url) + with owned_proxy( + gateway, + tmp_path, + {**overrides, "LITELLM_MODEL_COST_MAP_URL": peer.url + "/model_prices.json"}, + config=config, + remove_environment=("LITELLM_LOCAL_MODEL_COST_MAP",), + ) as candidate: + assert model not in tuple(entry["id"] for entry in candidate.get("/v1/models")["data"]) + fresh_map_published.set() + reload: Final = candidate.request("POST", "/reload/model_cost_map") + assert reload.status_code == 200, reload.text + eventually( + lambda: tuple(str(entry["id"]) for entry in candidate.get("/v1/models")["data"]), + lambda served: model in served, + seconds=30, + ) + upstream.get("/__observations").raise_for_status() + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "stale cost map control"}]}, + ) + assert response.status_code == 200, response.text + assert upstream.get("/__observations").json()["requests"] == [ + { + "path": "/v1/chat/completions", + "authorization": "Bearer sk-upstream", + "body": {"model": model, "messages": [{"role": "user", "content": "stale cost map control"}]}, + } + ] diff --git a/tests/integration/run.py b/tests/integration/run.py index f45164c5ca4..0f9aa75b549 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -47,6 +47,7 @@ def main() -> int: "pytest", *selected, "-vv", + "-rs", "--strict-markers", "-p", "no:pytest-retry", @@ -69,7 +70,8 @@ def main() -> int: if result != 0: return result evidence: Final = json.loads((output / "execution.json").read_text()) - if not evidence["complete"] or sorted(evidence["passed"]) != expected or sorted(evidence["collected"]) != expected: + executed: Final = sorted(evidence["passed"] + evidence["skipped"]) + if not evidence["complete"] or executed != expected or sorted(evidence["collected"]) != expected: print("Executed integration nodes differ from the canonical manifest", file=sys.stderr) return 1 return 0 diff --git a/tests/integration/spend/test_batch_completion_accounting.py b/tests/integration/spend/test_batch_completion_accounting.py new file mode 100644 index 00000000000..0cbeda934f6 --- /dev/null +++ b/tests/integration/spend/test_batch_completion_accounting.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import json +import uuid +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, eventually, string_value +from integration._support.database import read_rows +from integration._support.upstream import delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse, TextResponse +from pydantic import JsonValue + +FIRST_LINE: Final = {"prompt_tokens": 10, "completion_tokens": 7, "reasoning_tokens": 4} +SECOND_LINE: Final = {"prompt_tokens": 5, "completion_tokens": 3, "reasoning_tokens": 2} +ERROR_FILE_LINES: Final = 2 + + +def _succeeded_line(index: int, model: str, prompt_tokens: int, completion_tokens: int, reasoning_tokens: int) -> str: + return json.dumps( + { + "id": f"batch_req_{index}", + "custom_id": f"r{index}", + "response": { + "status_code": 200, + "request_id": f"$REQUEST_ID-{index}", + "body": { + "id": f"chatcmpl-$REQUEST_ID-{index}", + "object": "chat.completion", + "model": model, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + "completion_tokens_details": {"reasoning_tokens": reasoning_tokens}, + }, + }, + }, + "error": None, + }, + separators=(",", ":"), + ) + + +def _failed_line(index: int) -> str: + return json.dumps( + { + "id": f"batch_req_{index}", + "custom_id": f"r{index}", + "response": { + "status_code": 400, + "request_id": f"$REQUEST_ID-{index}", + "body": {"error": {"message": "rejected line", "type": "invalid_request_error", "code": "400"}}, + }, + "error": {"code": "bad_request", "message": "rejected line"}, + }, + separators=(",", ":"), + ) + + +def _batch_routes(model: str) -> RoutedResponse: + output_lines: Final = ( + _succeeded_line(1, model, **FIRST_LINE), + _succeeded_line(2, model, **SECOND_LINE), + _failed_line(3), + ) + error_lines: Final = tuple(_failed_line(index) for index in range(4, 4 + ERROR_FILE_LINES)) + completed: Final = { + "id": "batch-$REQUEST_ID", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": None, + "input_file_id": "file-in-$REQUEST_ID", + "completion_window": "24h", + "status": "completed", + "output_file_id": "file-out-$REQUEST_ID", + "error_file_id": "file-err-$REQUEST_ID", + "created_at": 1, + "in_progress_at": 1, + "completed_at": 1, + "expires_at": 1, + "request_counts": {"total": 5, "completed": 2, "failed": 3}, + "metadata": None, + } + return RoutedResponse( + content_type="application/x-routed", + routes={ + "POST /files": JsonResponse( + content_type="application/json", + body={ + "id": "file-in-$REQUEST_ID", + "object": "file", + "purpose": "batch", + "bytes": 100, + "created_at": 1, + "filename": "in.jsonl", + "status": "processed", + }, + ), + "POST /batches": JsonResponse( + content_type="application/json", + body={**completed, "status": "validating", "output_file_id": None, "error_file_id": None}, + ), + "GET /batches/batch-$REQUEST_ID": JsonResponse(content_type="application/json", body=completed), + "GET /files/file-out-$REQUEST_ID/content": TextResponse( + content_type="application/jsonl", body="\n".join(output_lines) + "\n" + ), + "GET /files/file-err-$REQUEST_ID/content": TextResponse( + content_type="application/jsonl", body="\n".join(error_lines) + "\n" + ), + }, + ) + + +def _input_file(model: str) -> bytes: + return ( + "\n".join( + json.dumps( + { + "custom_id": f"r{index}", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": "batch accounting"}]}, + }, + separators=(",", ":"), + ) + for index in range(1, 6) + ) + + "\n" + ).encode() + + +def _metadata(value: object) -> dict[str, JsonValue]: + return JSON_OBJECT.validate_json(value) if isinstance(value, str) else JSON_OBJECT.validate_python(value) + + +@pytest.mark.covers("quota_management.spend_tracking.batch_costs.reasoning_tokens_and_error_file_failures_recorded") +def test_completed_batch_spend_row_records_reasoning_tokens_and_error_file_failures(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + key: Final = scenario.key() + scenario_id: Final = f"batch-accounting-{uuid.uuid4().hex[:12]}" + handle: Final = register_scenario(scenario_id, _batch_routes("gpt-4o-mini")) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model(api_base=handle.api_base()) + file_response: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model}, + {"file": ("in.jsonl", _input_file(model), "application/jsonl")}, + key=key, + ) + assert file_response.status_code == 200, file_response.text + batch_response: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": string_value(JSON_OBJECT.validate_json(file_response.content)["id"]), + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": model, + }, + key=key, + ) + assert batch_response.status_code == 200, batch_response.text + batch_id: Final = string_value(JSON_OBJECT.validate_json(batch_response.content)["id"]) + retrieval: Final = gateway.request("GET", f"/v1/batches/{batch_id}", key=key) + assert retrieval.status_code == 200, retrieval.text + assert retrieval.json()["status"] == "completed", retrieval.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" ' + "WHERE api_key=%s AND call_type='aretrieve_batch'", + (sha256(key.encode()).hexdigest(),), + ), + lambda values: len(values) == 1, + seconds=70, + ) + row: Final = rows[0] + metadata: Final = _metadata(row["metadata"]) + prompt_tokens: Final = FIRST_LINE["prompt_tokens"] + SECOND_LINE["prompt_tokens"] + completion_tokens: Final = FIRST_LINE["completion_tokens"] + SECOND_LINE["completion_tokens"] + reasoning_tokens: Final = FIRST_LINE["reasoning_tokens"] + SECOND_LINE["reasoning_tokens"] + assert row["status"] == "success", retrieval.text + assert (row["prompt_tokens"], row["completion_tokens"]) == (prompt_tokens, completion_tokens), retrieval.text + assert (metadata["batch_successful_requests"], metadata["batch_failed_requests"]) == ( + 2, + 1 + ERROR_FILE_LINES, + ), json.dumps(metadata) + usage: Final = JSON_OBJECT.validate_python(metadata["usage_object"]) + details: Final = JSON_OBJECT.validate_python(usage["completion_tokens_details"]) + assert (usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) == ( + prompt_tokens, + completion_tokens, + prompt_tokens + completion_tokens, + ), json.dumps(metadata) + assert {name: value for name, value in details.items() if value is not None} == { + "reasoning_tokens": reasoning_tokens, + "text_tokens": completion_tokens - reasoning_tokens, + }, json.dumps(metadata) diff --git a/tests/integration/spend/test_batch_observability.py b/tests/integration/spend/test_batch_observability.py new file mode 100644 index 00000000000..ca9c26ff513 --- /dev/null +++ b/tests/integration/spend/test_batch_observability.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import json +import uuid +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.upstream import delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse, TextResponse +from pydantic import JsonValue + +REASONING_TOKENS: Final = (30, 50) +PROMPT_TOKENS: Final = 10 +COMPLETION_TOKENS: Final = 100 +ERROR_FILE_FAILURES: Final = 2 + + +def _successful_line(index: int, reasoning_tokens: int) -> str: + return json.dumps( + { + "id": f"batch_req_{index}", + "custom_id": f"r{index}", + "response": { + "status_code": 200, + "request_id": f"$REQUEST_ID-{index}", + "body": { + "id": f"chatcmpl-$REQUEST_ID-{index}", + "object": "chat.completion", + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + "completion_tokens_details": {"reasoning_tokens": reasoning_tokens}, + }, + }, + }, + "error": None, + }, + separators=(",", ":"), + ) + + +def _failed_line(index: int) -> str: + return json.dumps( + { + "id": f"batch_req_{index}", + "custom_id": f"r{index}", + "response": {"status_code": 400, "request_id": f"$REQUEST_ID-{index}", "body": {"error": "bad"}}, + "error": {"code": "bad_request", "message": "failed"}, + }, + separators=(",", ":"), + ) + + +def _batch(status: str, *, files_ready: bool) -> dict[str, JsonValue]: + return { + "id": "batch-$REQUEST_ID", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": None, + "input_file_id": "file-in-$REQUEST_ID", + "completion_window": "24h", + "status": status, + "output_file_id": "file-out-$REQUEST_ID" if files_ready else None, + "error_file_id": "file-err-$REQUEST_ID" if files_ready else None, + "created_at": 1, + "in_progress_at": 1, + "completed_at": 1 if files_ready else None, + "expires_at": 1, + "request_counts": {"total": 5, "completed": 2, "failed": 3}, + "metadata": None, + } + + +def _provider_routes() -> RoutedResponse: + output_lines: Final = ( + _successful_line(1, REASONING_TOKENS[0]), + _failed_line(2), + _successful_line(3, REASONING_TOKENS[1]), + ) + error_lines: Final = tuple(_failed_line(index) for index in range(4, 4 + ERROR_FILE_FAILURES)) + return RoutedResponse( + content_type="application/x-routed", + routes={ + "POST /files": JsonResponse( + content_type="application/json", + body={ + "id": "file-in-$REQUEST_ID", + "object": "file", + "purpose": "batch", + "bytes": 100, + "created_at": 1, + "filename": "in.jsonl", + "status": "processed", + }, + ), + "POST /batches": JsonResponse( + content_type="application/json", body=_batch("validating", files_ready=False) + ), + "GET /batches/batch-$REQUEST_ID": JsonResponse( + content_type="application/json", body=_batch("completed", files_ready=True) + ), + "GET /files/file-out-$REQUEST_ID/content": TextResponse( + content_type="application/jsonl", body="\n".join(output_lines) + "\n" + ), + "GET /files/file-err-$REQUEST_ID/content": TextResponse( + content_type="application/jsonl", body="\n".join(error_lines) + "\n" + ), + }, + ) + + +def _input_file(model_name: str) -> bytes: + return ( + "\n".join( + json.dumps( + { + "custom_id": f"r{index}", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model_name, "messages": [{"role": "user", "content": "batch observability"}]}, + }, + separators=(",", ":"), + ) + for index in range(1, 6) + ) + + "\n" + ).encode() + + +def _retrieval_rows(key: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + read_rows( + 'SELECT prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" ' + "WHERE api_key=%s AND call_type='aretrieve_batch'", + (sha256(key.encode()).hexdigest(),), + ) + ) + + +def _metadata(row: dict[str, JsonValue]) -> dict[str, JsonValue]: + value: Final = row["metadata"] + return object_value(JSON_OBJECT.validate_json(value) if isinstance(value, str) else value) + + +@pytest.mark.covers("spend.batches.retrieval_row_aggregates_reasoning_tokens_and_per_request_counts") +def test_batch_retrieval_row_sums_reasoning_tokens_and_counts_output_and_error_file_failures( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + scenario_id: Final = f"batch-observability-{uuid.uuid4().hex[:12]}" + handle: Final = register_scenario(scenario_id, _provider_routes()) + scenario.cleanups.callback(delete_scenario, handle) + model_name: Final = scenario.model(api_base=handle.api_base()) + key: Final = scenario.key(models=[model_name]) + file_response: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model_name}, + {"file": ("in.jsonl", _input_file(model_name), "application/jsonl")}, + key=key, + ) + assert file_response.status_code == 200, file_response.text + input_file_id: Final = string_value(JSON_OBJECT.validate_json(file_response.content)["id"]) + batch_response: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": input_file_id, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": model_name, + }, + key=key, + ) + assert batch_response.status_code == 200, batch_response.text + batch_id: Final = string_value(JSON_OBJECT.validate_json(batch_response.content)["id"]) + retrieval: Final = eventually( + lambda: gateway.request("GET", f"/v1/batches/{batch_id}", key=key), + lambda response: response.status_code == 200 and response.json()["status"] == "completed", + seconds=30, + ) + assert retrieval.status_code == 200, retrieval.text + rows: Final = eventually(lambda: _retrieval_rows(key), lambda values: len(values) == 1, seconds=70) + row: Final = rows[0] + metadata: Final = _metadata(row) + usage: Final = object_value(metadata["usage_object"]) + assert row["prompt_tokens"] == 2 * PROMPT_TOKENS, retrieval.text + assert row["completion_tokens"] == 2 * COMPLETION_TOKENS, retrieval.text + assert object_value(usage["completion_tokens_details"])["reasoning_tokens"] == sum(REASONING_TOKENS), ( + retrieval.text, + usage, + ) + assert metadata["batch_successful_requests"] == 2, (retrieval.text, metadata) + assert metadata["batch_failed_requests"] == 1 + ERROR_FILE_FAILURES, (retrieval.text, metadata) diff --git a/tests/integration/spend/test_batch_poll_starvation.py b/tests/integration/spend/test_batch_poll_starvation.py new file mode 100644 index 00000000000..f68f10e3d8a --- /dev/null +++ b/tests/integration/spend/test_batch_poll_starvation.py @@ -0,0 +1,212 @@ +import json +import os +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.upstream import delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse, TextResponse +from pydantic import JsonValue + +from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE + +INPUT_COST_PER_TOKEN: Final = 0.001 +OUTPUT_COST_PER_TOKEN: Final = 0.002 +PROMPT_TOKENS: Final = 100 +COMPLETION_TOKENS: Final = 50 +BATCH_COST_SHARE: Final = 0.5 + +_INPUT_FILE: Final = JsonResponse( + content_type="application/json", + body={ + "id": "file-in-$REQUEST_ID", + "object": "file", + "purpose": "batch", + "bytes": 100, + "created_at": 1, + "filename": "in.jsonl", + "status": "processed", + }, +) + + +def _batch(status: str, output_file_id: str | None) -> dict[str, JsonValue]: + return { + "id": "batch-$REQUEST_ID", + "object": "batch", + "endpoint": "/v1/chat/completions", + "errors": None, + "input_file_id": "file-in-$REQUEST_ID", + "completion_window": "24h", + "status": status, + "output_file_id": output_file_id, + "error_file_id": None, + "created_at": 1, + "in_progress_at": 1, + "completed_at": 1 if status == "completed" else None, + "expires_at": 1, + "request_counts": {"total": 1, "completed": 1 if status == "completed" else 0, "failed": 0}, + "metadata": None, + } + + +def _accepting_routes() -> dict[str, JsonResponse | TextResponse]: + return { + "POST /files": _INPUT_FILE, + "POST /batches": JsonResponse(content_type="application/json", body=_batch("validating", None)), + } + + +def _gone_at_provider_routes() -> RoutedResponse: + return RoutedResponse( + content_type="application/x-routed", + routes={ + **_accepting_routes(), + "GET /batches/batch-$REQUEST_ID": JsonResponse( + content_type="application/json", + status=404, + body={ + "error": { + "message": "No batch found with id 'batch-$REQUEST_ID'.", + "type": "invalid_request_error", + "param": "id", + "code": "batch_not_found", + } + }, + ), + }, + ) + + +def _completed_routes() -> RoutedResponse: + output_line: Final = { + "id": "batch_req_1", + "custom_id": "r1", + "response": { + "status_code": 200, + "request_id": "$REQUEST_ID-1", + "body": { + "id": "chatcmpl-$REQUEST_ID-1", + "object": "chat.completion", + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + }, + }, + }, + "error": None, + } + return RoutedResponse( + content_type="application/x-routed", + routes={ + **_accepting_routes(), + "GET /batches/batch-$REQUEST_ID": JsonResponse( + content_type="application/json", body=_batch("completed", "file-out-$REQUEST_ID") + ), + "GET /files/file-out-$REQUEST_ID/content": TextResponse( + content_type="application/jsonl", body=json.dumps(output_line, separators=(",", ":")) + "\n" + ), + }, + ) + + +def _scripted_deployment(scenario: Scenario, marker: str, routes: RoutedResponse) -> str: + scenario_id: Final = f"poll-{marker}-{sha256(os.urandom(16)).hexdigest()[:12]}" + handle: Final = register_scenario(scenario_id, routes) + scenario.cleanups.callback(delete_scenario, handle) + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": f"poll-{marker}-{sha256(scenario_id.encode()).hexdigest()[:12]}", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-scripted-provider", + "api_base": handle.api_base(), + "input_cost_per_token": INPUT_COST_PER_TOKEN, + "output_cost_per_token": OUTPUT_COST_PER_TOKEN, + }, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + return string_value(created["model_name"]) + + +def _submitted_batch_id(gateway: Gateway, key: str, model_name: str) -> str: + request_line: Final = { + "custom_id": "r1", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model_name, "messages": [{"role": "user", "content": "poll starvation"}]}, + } + file_response: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": model_name}, + {"file": ("in.jsonl", (json.dumps(request_line) + "\n").encode(), "application/jsonl")}, + key=key, + ) + assert file_response.is_success, file_response.text + batch_response: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": string_value(JSON_OBJECT.validate_json(file_response.content)["id"]), + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": model_name, + }, + key=key, + ) + assert batch_response.is_success, batch_response.text + return string_value(JSON_OBJECT.validate_json(batch_response.content)["id"]) + + +def _managed_rows(batch_ids: tuple[str, ...]) -> list[dict[str, JsonValue]]: + placeholders: Final = ", ".join("%s" for _ in batch_ids) + return read_rows( + f'SELECT batch_processed FROM "LiteLLM_ManagedObjectTable" WHERE unified_object_id IN ({placeholders})', + batch_ids, + ) + + +@pytest.mark.timeout(180) +@pytest.mark.covers("quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed") +def test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + key: Final = scenario.key() + gone_batch_ids: Final = tuple( + _submitted_batch_id( + gateway, key, _scripted_deployment(scenario, f"gone{index}", _gone_at_provider_routes()) + ) + for index in range(MAX_OBJECTS_PER_POLL_CYCLE) + ) + costable_batch_id: Final = _submitted_batch_id( + gateway, key, _scripted_deployment(scenario, "costable", _completed_routes()) + ) + spend_rows: Final = eventually( + lambda: read_rows( + 'SELECT call_type, status, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE api_key = %s AND call_type = %s", + (sha256(key.encode()).hexdigest(), "aretrieve_batch"), + ), + lambda rows: len(rows) == 1, + seconds=120, + ) + assert spend_rows == [ + { + "call_type": "aretrieve_batch", + "status": "success", + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "spend": pytest.approx( + BATCH_COST_SHARE + * (PROMPT_TOKENS * INPUT_COST_PER_TOKEN + COMPLETION_TOKENS * OUTPUT_COST_PER_TOKEN) + ), + } + ] + assert _managed_rows((costable_batch_id,)) == [{"batch_processed": True}] + assert _managed_rows(gone_batch_ids) == [{"batch_processed": True}] * MAX_OBJECTS_PER_POLL_CYCLE diff --git a/tests/integration/spend/test_cache_and_quota.py b/tests/integration/spend/test_cache_and_quota.py index d32297765f6..97ef785aa15 100644 --- a/tests/integration/spend/test_cache_and_quota.py +++ b/tests/integration/spend/test_cache_and_quota.py @@ -1,4 +1,7 @@ +import json +import threading import uuid +from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack from hashlib import sha256 from typing import Final @@ -7,10 +10,10 @@ import httpx import pytest from hypothesis import strategies as st from hypothesis.stateful import RuleBasedStateMachine, rule, run_state_machine_as_test - from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests +from integration._support.wire import Reply, Request, wire_server @pytest.mark.covers("quota_management.response_cache.generated_sequences_preserve_content_and_accounting") @@ -211,6 +214,117 @@ def test_key_budget_at_boundary_blocks_provider_then_explicit_reset_restores(gat assert upstream.get("/__observations").json()["requests"] == [] +@pytest.mark.covers("quota_management.budget.key.count_tokens_reserves_nothing_so_completion_within_budget_succeeds") +def test_repeated_count_tokens_on_budgeted_key_does_not_reserve_budget_or_block_later_completion( + gateway: Gateway, +) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model], max_budget=0.1) + digest: Final = sha256(key.encode()).hexdigest() + upstream.get("/__observations").raise_for_status() + counts: Final = tuple( + gateway.request( + "POST", + "/v1/messages/count_tokens", + {"model": model, "messages": [{"role": "user", "content": "hello!!!"}]}, + key=key, + headers={"anthropic-version": "2023-06-01"}, + ) + for _ in range(3) + ) + for count in counts: + assert count.status_code == 200, count.text + assert count.json() == counts[0].json(), count.text + input_tokens: Final = counts[0].json()["input_tokens"] + assert isinstance(input_tokens, int) and input_tokens > 0, counts[0].text + assert upstream.get("/__observations").json()["requests"] == [] + completion: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"after counting {uuid.uuid4().hex}"}]}, + key=key, + ) + assert completion.status_code == 200, completion.text + assert completion.json()["usage"]["total_tokens"] == 40, completion.text + assert [request["path"] for request in upstream.get("/__observations").json()["requests"]] == [ + "/v1/chat/completions" + ] + spent: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)), + lambda values: len(values) == 1 and float(values[0]["spend"]) > 0, + seconds=70, + ) + assert float(spent[0]["spend"]) == pytest.approx(20 * 0.001 + 20 * 0.002) + rows: Final = eventually( + lambda: read_rows('SELECT call_type, spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (digest,)), + lambda values: len(values) >= 1, + seconds=70, + ) + assert [(row["call_type"], float(row["spend"])) for row in rows] == [("acompletion", pytest.approx(0.06))] + + +@pytest.mark.covers( + "quota_management.budget.key.in_flight_count_tokens_reserves_nothing_so_completion_reaches_provider" +) +def test_in_flight_count_tokens_does_not_reserve_key_budget_away_from_a_completion(gateway: Gateway) -> None: + counting_reached_provider: Final = threading.Event() + completion_answered: Final = threading.Event() + + def respond(request: Request) -> Reply: + counting_reached_provider.set() + assert completion_answered.wait(timeout=30), "completion never ran while count tokens was in flight" + return Reply(body=b'{"totalTokens": 12, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12}]}') + + with ( + wire_server(respond) as wire, + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ThreadPoolExecutor(max_workers=1) as background, + ): + counted: Final = scenario.model( + model="gemini/gemini-3.8-flash", + api_base=wire.url, + api_key="synthetic-gemini-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + completed: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[counted, completed], max_budget=0.06) + contents: Final = [{"role": "user", "parts": [{"text": "hello"}]}] + counting: Final = background.submit( + gateway.request, "POST", f"/v1beta/models/{counted}:countTokens", {"contents": contents}, key=key + ) + assert counting_reached_provider.wait(timeout=30), "count tokens request never reached the provider" + upstream.get("/__observations").raise_for_status() + prompt: Final = f"after count tokens {uuid.uuid4().hex}" + completion: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": completed, "messages": [{"role": "user", "content": prompt}]}, + key=key, + ) + completion_answered.set() + count: Final = counting.result(timeout=30) + assert completion.status_code == 200 and completion.json()["usage"]["total_tokens"] == 40, completion.text + assert [call["body"]["messages"] for call in upstream.get("/__observations").json()["requests"]] == [ + [{"role": "user", "content": prompt}] + ] + assert count.status_code == 200, count.text + assert count.json() == {"totalTokens": 12, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12}]}, ( + count.text + ) + provider_calls: Final = wire.drain() + assert [(call.method, call.target) for call in provider_calls] == [ + ("POST", "/v1beta/models/gemini-3.8-flash:countTokens") + ] + assert provider_calls[0].headers["x-goog-api-key"] == "synthetic-gemini-key" + assert json.loads(provider_calls[0].body) == {"contents": contents} + + @pytest.mark.covers("quota_management.response_cache.system_messages_partition_cache_identity") def test_different_system_messages_do_not_share_a_cached_response(gateway: Gateway) -> None: with ( @@ -219,8 +333,7 @@ def test_different_system_messages_do_not_share_a_cached_response(gateway: Gatew ): model: Final = scenario.model() prompt: Final = uuid.uuid4().hex - identities: dict[str, str] = {} - for system, expected_calls in (("first policy", 1), ("second policy", 1), ("first policy", 0)): + def completion_id(system: str, expected_calls: int) -> str: upstream.get("/__observations").raise_for_status() response: Final = gateway.request( "POST", @@ -232,14 +345,12 @@ def test_different_system_messages_do_not_share_a_cached_response(gateway: Gatew ) assert response.status_code == 200 and response.json()["usage"]["total_tokens"] == 40, response.text calls: Final = upstream.get("/__observations").json()["requests"] - assert len(calls) == expected_calls - if system in identities: - assert response.json()["id"] == identities[system] - else: - assert response.json()["id"] not in identities.values() - identities = {**identities, system: response.json()["id"]} - if calls: - assert calls[0]["body"]["messages"] == [ - {"role": "system", "content": system}, - {"role": "user", "content": prompt}, - ] + assert [call["body"]["messages"] for call in calls] == [ + [{"role": "system", "content": system}, {"role": "user", "content": prompt}] + ] * expected_calls, calls + return response.json()["id"] + + first_policy_id: Final = completion_id("first policy", 1) + second_policy_id: Final = completion_id("second policy", 1) + assert first_policy_id != second_policy_id + assert completion_id("first policy", 0) == first_policy_id diff --git a/tests/integration/spend/test_daily_rollup_retry.py b/tests/integration/spend/test_daily_rollup_retry.py new file mode 100644 index 00000000000..cf1b989639a --- /dev/null +++ b/tests/integration/spend/test_daily_rollup_retry.py @@ -0,0 +1,182 @@ +import json +import os +import uuid +from collections.abc import Iterable +from hashlib import sha256 +from typing import Final + +import psycopg +import pytest +from integration._support.client import Gateway, delete_key_if_present, eventually, string_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from psycopg import sql + + +def _execute(statements: Iterable[sql.Composable]) -> None: + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + for statement in statements: + connection.execute(statement) + + +def _install_daily_user_rollup_fault(user_id: str) -> str: + suffix: Final = f"fault-{uuid.uuid4().hex}" + sequence: Final = sql.Identifier(f"{suffix}_attempts") + function: Final = sql.Identifier(suffix) + _execute( + ( + sql.SQL("CREATE SEQUENCE {}").format(sequence), + sql.SQL( + "CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $fault$ " + "BEGIN PERFORM nextval({}); " + "RAISE EXCEPTION 'synthetic daily rollup outage' USING ERRCODE = '55P03'; " + "END $fault$" + ).format(function, sql.Literal(f"{suffix}_attempts")), + sql.SQL( + 'CREATE TRIGGER {} BEFORE INSERT ON "LiteLLM_DailyUserSpend" ' + "FOR EACH ROW WHEN (NEW.user_id = {}) EXECUTE FUNCTION {}()" + ).format(sql.Identifier(suffix), sql.Literal(user_id), function), + ) + ) + return suffix + + +def _lift_daily_user_rollup_fault(suffix: str) -> None: + _execute( + ( + sql.SQL('DROP TRIGGER IF EXISTS {} ON "LiteLLM_DailyUserSpend"').format(sql.Identifier(suffix)), + sql.SQL("DROP FUNCTION IF EXISTS {}()").format(sql.Identifier(suffix)), + sql.SQL("DROP SEQUENCE IF EXISTS {}").format(sql.Identifier(f"{suffix}_attempts")), + ) + ) + + +def _rollup_attempts(suffix: str) -> int: + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + row: Final = connection.execute( + sql.SQL("SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM {}").format( + sql.Identifier(f"{suffix}_attempts") + ) + ).fetchone() + assert row is not None + return int(row[0]) + + +@pytest.mark.covers("spend.daily_rollup.failed_user_commit_is_retried_until_report_and_daily_activity_agree") +def test_failed_daily_user_rollup_commit_is_retried_so_spend_report_and_daily_activity_agree( + gateway: Gateway, +) -> None: + def provider(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/chat/completions" + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + uuid.uuid4().hex, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "synthetic rollup answer"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40}, + } + ).encode() + ) + + with wire_server(provider) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + api_base=wire.url + "/v1", input_cost_per_token=0.001, output_cost_per_token=0.002, num_retries=0 + ) + user: Final = scenario.user() + key: Final = string_value(gateway.post("/key/generate", {"user_id": user, "models": [model]})["key"]) + scenario.cleanups.callback(delete_key_if_present, gateway, key) + digest: Final = sha256(key.encode()).hexdigest() + suffix: Final = _install_daily_user_rollup_fault(user) + scenario.cleanups.callback(_lift_daily_user_rollup_fault, suffix) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "rollup retry control"}]}, + key=key, + ) + assert response.status_code == 200, response.text + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(20 * 0.001 + 20 * 0.002) + body: Final = response.json() + spend_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, DATE("startTime")::text AS day, model FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (body["id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(spend_rows[0]["spend"]) == pytest.approx(0.06) + day: Final = string_value(spend_rows[0]["day"]) + stored_model: Final = string_value(spend_rows[0]["model"]) + eventually(lambda: _rollup_attempts(suffix), lambda attempts: attempts >= 1, seconds=70) + _lift_daily_user_rollup_fault(suffix) + activity: Final = eventually( + lambda: gateway.request( + "GET", + "/user/daily/activity/aggregated", + params={"start_date": day, "end_date": day, "api_key": digest}, + ), + lambda polled: ( + polled.status_code == 200 + and len(polled.json().get("results", ())) > 0 + and polled.json()["results"][0]["breakdown"]["api_keys"] + .get(digest, {}) + .get("metrics", {}) + .get("spend", 0) + == pytest.approx(0.06) + ), + seconds=90, + ) + assert activity.status_code == 200, activity.text + metrics: Final = activity.json()["results"][0]["breakdown"]["api_keys"][digest]["metrics"] + assert metrics == { + "spend": pytest.approx(0.06), + "flat_cost": pytest.approx(0.0), + "prompt_tokens": 20, + "completion_tokens": 20, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": pytest.approx(0.0), + "prompt_caching_savings_spend": pytest.approx(0.0), + "gateway_injected_caching_savings_spend": pytest.approx(0.0), + "autorouter_savings_spend": pytest.approx(0.0), + "total_tokens": 40, + "successful_requests": 1, + "failed_requests": 0, + "api_requests": 1, + "total_response_time_ms": metrics["total_response_time_ms"], + "timed_requests": metrics["timed_requests"], + } + report: Final = gateway.request( + "GET", + "/global/spend/report", + params={"start_date": day, "end_date": day, "api_key": digest}, + ) + assert report.status_code == 200, report.text + assert report.json() == [ + { + "api_key": digest, + "total_cost": pytest.approx(0.06), + "total_input_tokens": 20, + "total_output_tokens": 20, + "model_details": [ + { + "model": stored_model, + "total_cost": pytest.approx(0.06), + "total_input_tokens": 20, + "total_output_tokens": 20, + } + ], + } + ] + assert metrics["spend"] == pytest.approx(report.json()[0]["total_cost"]) diff --git a/tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py b/tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py new file mode 100644 index 00000000000..91f2c1c0760 --- /dev/null +++ b/tests/integration/spend/test_disconnected_bedrock_messages_stream_billing.py @@ -0,0 +1,123 @@ +import base64 +import json +import uuid +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.upstream import _aws_event_frame +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +BEDROCK_MODEL: Final = "anthropic.claude-haiku-4-5-20251001-v1:0" +INPUT_TOKENS: Final = 30 +FULL_OUTPUT_TOKENS: Final = 412 +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 + + +def _invoke_chunk(payload: dict[str, JsonValue]) -> bytes: + encoded: Final = base64.b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode() + return _aws_event_frame("chunk", {"bytes": encoded}, "", "") + + +def _message_start(message_id: str) -> bytes: + return _invoke_chunk( + { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "model": BEDROCK_MODEL, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": INPUT_TOKENS, "output_tokens": 0}, + }, + } + ) + _invoke_chunk({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}) + + +def _text_delta(text: str) -> bytes: + return _invoke_chunk({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}) + + +def _terminal_usage() -> bytes: + return ( + _invoke_chunk({"type": "content_block_stop", "index": 0}) + + _invoke_chunk( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": FULL_OUTPUT_TOKENS}, + } + ) + + _invoke_chunk({"type": "message_stop"}) + ) + + +@pytest.mark.covers("spend.anthropic_messages_stream.client_disconnect_bills_terminal_bedrock_usage") +@pytest.mark.timeout(120) +def test_client_disconnect_mid_bedrock_messages_stream_still_bills_terminal_usage(gateway: Gateway) -> None: + message_id: Final = f"msg_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.target == f"/model/{BEDROCK_MODEL}/invoke-with-response-stream", request.target + assert json.loads(request.body)["messages"] == [{"role": "user", "content": "disconnect control"}], request.body + return Reply( + content_type="application/vnd.amazon.eventstream", + chunks=( + _message_start(message_id) + _text_delta("first"), + _text_delta("second"), + _text_delta("third"), + _terminal_usage(), + ), + pause_between_chunks=0.5, + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/invoke/{BEDROCK_MODEL}", + api_base=wire.url, + aws_access_key_id="AKIASCRIPTEDPROVIDER", + aws_secret_access_key="scripted-secret", + aws_region_name="us-east-1", + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + ) + key: Final = scenario.key(models=[model]) + with gateway.client.stream( + "POST", + "/v1/messages", + json={ + "model": model, + "messages": [{"role": "user", "content": "disconnect control"}], + "max_tokens": FULL_OUTPUT_TOKENS, + "stream": True, + }, + headers={"Authorization": f"Bearer {key}"}, + ) as response: + assert response.status_code == 200, response.read().decode() + first_event: Final = next(line for line in response.iter_lines() if line.startswith("data:")) + assert json.loads(first_event.removeprefix("data:"))["type"] == "message_start", first_event + + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, status, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE api_key=%s", + (sha256(key.encode()).hexdigest(),), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["request_id"] == message_id, rows + assert rows[0]["status"] == "success", rows + assert rows[0]["prompt_tokens"] == INPUT_TOKENS, rows + assert rows[0]["completion_tokens"] == FULL_OUTPUT_TOKENS, rows + assert float(str(rows[0]["spend"])) == pytest.approx( + INPUT_TOKENS * INPUT_RATE + FULL_OUTPUT_TOKENS * OUTPUT_RATE + ), rows + assert len(wire.drain()) == 1 diff --git a/tests/integration/spend/test_failed_dispatch_tokens.py b/tests/integration/spend/test_failed_dispatch_tokens.py new file mode 100644 index 00000000000..5778ff9654d --- /dev/null +++ b/tests/integration/spend/test_failed_dispatch_tokens.py @@ -0,0 +1,50 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server + + +@pytest.mark.covers("spend.failed_dispatch.failure_row_records_estimated_input_tokens") +def test_provider_500_after_dispatch_records_estimated_prompt_tokens_on_failure_row(gateway: Gateway) -> None: + prompt: Final = "failed dispatch accounting " + uuid.uuid4().hex + system: Final = "You are a terse accounting assistant" + + def provider(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/chat/completions" + body: Final = json.loads(request.body) + assert body["model"] == "gpt-4o-mini" + assert body["messages"] == [{"role": "system", "content": system}, {"role": "user", "content": prompt}] + return Reply( + status=500, + body=b'{"error":{"message":"synthetic provider outage","type":"server_error","code":"500"}}', + ) + + with wire_server(provider) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) + key: Final = scenario.key(models=[model]) + failed: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}]}, + key=key, + ) + assert failed.status_code == 500 and "synthetic provider outage" in failed.text, failed.text + call_id: Final = failed.headers["x-litellm-call-id"] + assert len(wire.drain()) == 1 + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, spend, prompt_tokens, completion_tokens, total_tokens FROM "LiteLLM_SpendLogs" ' + "WHERE request_id=%s", + (call_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + row: Final = rows[0] + assert row["status"] == "failure" and float(row["spend"]) == 0 and row["completion_tokens"] == 0, row + assert row["prompt_tokens"] > 0, f"failure row lost the dispatched input tokens: {row}" + assert row["total_tokens"] == row["prompt_tokens"], row diff --git a/tests/integration/spend/test_model_router_selected_model.py b/tests/integration/spend/test_model_router_selected_model.py new file mode 100644 index 00000000000..a95ee4f9feb --- /dev/null +++ b/tests/integration/spend/test_model_router_selected_model.py @@ -0,0 +1,74 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server + +ROUTER_DEPLOYMENT: Final = "router-deploy" +SELECTED_MODEL: Final = "grok-4-1-fast-reasoning" +SELECTED_MODEL_WITH_PROVIDER: Final = f"azure_ai/{SELECTED_MODEL}" + + +@pytest.mark.covers("spend.model_router.selected_model_is_returned_and_persisted_for_plain_alias") +def test_model_router_alias_without_router_in_name_keeps_selected_model_in_response_and_spend_log( + gateway: Gateway, +) -> None: + prompt: Final = uuid.uuid4().hex + + def provider(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/chat/completions", request.target + assert json.loads(request.body) == { + "model": ROUTER_DEPLOYMENT, + "messages": [{"role": "user", "content": prompt}], + "stream": False, + }, request.body + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + uuid.uuid4().hex, + "object": "chat.completion", + "created": 1, + "model": SELECTED_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "routed answer"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40}, + } + ).encode() + ) + + with wire_server(provider) as wire, gateway.scenario() as scenario: + alias: Final = scenario.model( + model=f"azure_ai/model_router/{ROUTER_DEPLOYMENT}", api_base=wire.url, num_retries=0 + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": alias, "messages": [{"role": "user", "content": prompt}]}, + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["model"] == SELECTED_MODEL_WITH_PROVIDER, response.text + assert body["choices"][0]["message"]["content"] == "routed answer", response.text + assert len(wire.drain()) == 1 + rows: Final = eventually( + lambda: read_rows( + 'SELECT model, model_group, status FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (body["id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows == [{"model": SELECTED_MODEL_WITH_PROVIDER, "model_group": alias, "status": "success"}] + logs: Final = gateway.request("GET", "/spend/logs", params={"request_id": body["id"]}) + assert logs.status_code == 200, logs.text + assert [(row["model"], row["model_group"]) for row in logs.json()] == [(SELECTED_MODEL_WITH_PROVIDER, alias)], ( + logs.text + ) diff --git a/tests/integration/spend/test_org_budget_cli_session_token.py b/tests/integration/spend/test_org_budget_cli_session_token.py new file mode 100644 index 00000000000..f821e337374 --- /dev/null +++ b/tests/integration/spend/test_org_budget_cli_session_token.py @@ -0,0 +1,80 @@ +import os +import uuid +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, string_value +from integration._support.database import read_rows + +from litellm.proxy._types import LiteLLM_UserTable +from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken + + +def _cli_session_token(user_id: str, team_id: str) -> str: + cli_user: Final = LiteLLM_UserTable(user_id=user_id, user_role="internal_user", teams=[team_id], models=[]) + return ExperimentalUIJWTToken.get_cli_jwt_auth_token(user_info=cli_user, team_id=team_id, team_alias="cli-team") + + +@pytest.mark.covers("quota_management.organization_budget.cli_session_token_without_org_id_charges_team_organization") +def test_cli_session_token_without_org_id_charges_and_caps_the_team_organization( + gateway: Gateway, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt")) + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + organization: Final = gateway.post( + "/organization/new", {"organization_alias": f"integration-{uuid.uuid4().hex}", "max_budget": 0.06} + ) + org_id: Final = string_value(organization["organization_id"]) + scenario.cleanups.callback( + lambda: gateway.request("DELETE", "/organization/delete", {"organization_ids": [org_id]}) + ) + user_id: Final = scenario.user() + team_id: Final = scenario.team( + organization_id=org_id, models=[model], members_with_roles=[{"role": "user", "user_id": user_id}] + ) + token: Final = _cli_session_token(user_id, team_id) + prompt: Final = f"org budget {uuid.uuid4().hex}" + upstream.get("/__observations").raise_for_status() + first: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": prompt}]}, + key=token, + ) + assert first.status_code == 200 and first.json()["usage"]["total_tokens"] == 40, first.text + reached_upstream: Final = upstream.get("/__observations").json()["requests"] + assert len(reached_upstream) == 1, reached_upstream + assert reached_upstream[0]["body"]["model"] == "gpt-4o-mini", reached_upstream + assert reached_upstream[0]["body"]["messages"] == [{"role": "user", "content": prompt}], reached_upstream + logged: Final = eventually( + lambda: read_rows( + 'SELECT organization_id, team_id, spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (first.json()["id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert [(row["organization_id"], row["team_id"], float(row["spend"])) for row in logged] == [ + (org_id, team_id, pytest.approx(0.06)) + ] + charged: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_OrganizationTable" WHERE organization_id=%s', (org_id,)), + lambda values: len(values) == 1 and float(values[0]["spend"]) >= 0.06, + seconds=70, + ) + assert float(charged[0]["spend"]) == pytest.approx(0.06) + assert float(gateway.get("/organization/info", {"organization_id": org_id})["spend"]) == pytest.approx(0.06) + denied: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"over org budget {uuid.uuid4().hex}"}]}, + key=token, + ) + assert denied.status_code == 422 and denied.json()["error"]["type"] == "budget_exceeded", denied.text + assert f"Organization={org_id}" in denied.json()["error"]["message"], denied.text + assert upstream.get("/__observations").json()["requests"] == [] diff --git a/tests/integration/spend/test_passthrough_budget_reservation.py b/tests/integration/spend/test_passthrough_budget_reservation.py new file mode 100644 index 00000000000..f072333c616 --- /dev/null +++ b/tests/integration/spend/test_passthrough_budget_reservation.py @@ -0,0 +1,110 @@ +import uuid +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.upstream import delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse +from pydantic import JsonValue + +INPUT_COST_PER_TOKEN: Final = 0.000001 +OUTPUT_COST_PER_TOKEN: Final = 0.001 +PROMPT_TOKENS: Final = 10 +CANDIDATE_TOKENS: Final = 5 +COST_PER_CALL: Final = PROMPT_TOKENS * INPUT_COST_PER_TOKEN + CANDIDATE_TOKENS * OUTPUT_COST_PER_TOKEN +MAX_BUDGET: Final = 0.02 +CALLS_WITHIN_BUDGET: Final = 4 + + +def _key_spend(digest: str) -> float: + rows: Final = read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)) + assert len(rows) == 1, rows + return float(rows[0]["spend"]) + + +def _generate_content_request(model: str) -> dict[str, JsonValue]: + return {"contents": [{"role": "user", "parts": [{"text": f"budget {model}"}]}]} + + +def _generate_content_response(model: str) -> JsonResponse: + return JsonResponse( + content_type="application/json", + body={ + "candidates": [ + { + "content": {"parts": [{"text": f"scripted answer {model}"}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": PROMPT_TOKENS, + "candidatesTokenCount": CANDIDATE_TOKENS, + "totalTokenCount": PROMPT_TOKENS + CANDIDATE_TOKENS, + }, + "modelVersion": model, + }, + ) + + +def _served_call(gateway: Gateway, model: str, key: str, scenario_id: str, call: int) -> None: + digest: Final = sha256(key.encode()).hexdigest() + spend_before: Final = _key_spend(digest) + assert spend_before == pytest.approx((call - 1) * COST_PER_CALL) and spend_before < MAX_BUDGET + response: Final = gateway.request( + "POST", + f"/gemini/v1beta/models/{model}:generateContent", + _generate_content_request(model), + headers={"x-goog-api-key": key, "x-pass-x-scripted-scenario": scenario_id}, + ) + assert response.status_code == 200, f"call {call} with key spend {spend_before}: {response.text}" + assert response.json() == _generate_content_response(model).body, response.text + eventually(lambda: _key_spend(digest), lambda spend: spend >= call * COST_PER_CALL - 1e-9, seconds=70) + + +@pytest.mark.covers("spend.budget_reservation.gemini_passthrough_success_releases_reservation_from_spend_counter") +def test_repeated_gemini_passthrough_calls_stay_served_while_key_spend_is_below_max_budget( + gateway: Gateway, tmp_path: Path +) -> None: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["environment_variables"] = { + "GEMINI_API_BASE": gateway.upstream_url, + "GEMINI_API_KEY": "scripted", + } + path: Final = tmp_path / "gemini-passthrough.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = f"gemini-passthrough-{uuid.uuid4().hex}" + created: Final = candidate.post( + "/model/new", + { + "model_name": model, + "litellm_params": { + "model": "gemini/gemini-2.5-flash", + "api_key": "scripted", + "api_base": gateway.upstream_url, + "input_cost_per_token": INPUT_COST_PER_TOKEN, + "output_cost_per_token": OUTPUT_COST_PER_TOKEN, + }, + "model_info": {"id": model, "max_output_tokens": 10}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + handle: Final = register_scenario(f"sc-{model}", _generate_content_response(model)) + scenario.cleanups.callback(delete_scenario, handle) + key: Final = scenario.key(models=[model], max_budget=MAX_BUDGET) + for call in range(1, CALLS_WITHIN_BUDGET + 1): + _served_call(candidate, model, key, handle.scenario_id, call) + assert _key_spend(sha256(key.encode()).hexdigest()) == pytest.approx(CALLS_WITHIN_BUDGET * COST_PER_CALL) + denied: Final = candidate.request( + "POST", + f"/gemini/v1beta/models/{model}:generateContent", + _generate_content_request(model), + headers={"x-goog-api-key": key, "x-pass-x-scripted-scenario": handle.scenario_id}, + ) + assert denied.status_code == 422 and denied.json()["error"]["type"] == "budget_exceeded", denied.text diff --git a/tests/integration/spend/test_spend_calculate.py b/tests/integration/spend/test_spend_calculate.py index 855dd2b21f3..aa6103bf4de 100644 --- a/tests/integration/spend/test_spend_calculate.py +++ b/tests/integration/spend/test_spend_calculate.py @@ -18,3 +18,51 @@ def test_spend_calculate_rejects_unpriced_model_with_400(gateway: Gateway) -> No assert error["type"] == "invalid_request_error", response.text assert error["param"] == "model", response.text assert model in string_value(error["message"]), response.text + + +GEMINI_LIVE_PREVIEW_MODELS: Final = ( + "gemini-live-2.5-flash-preview-native-audio-09-2025", + "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", +) + + +@pytest.mark.parametrize("model", GEMINI_LIVE_PREVIEW_MODELS) +@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate") +def test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate(gateway: Gateway, model: str) -> None: + def cost_with_cached_tokens(cached_tokens: int) -> float: + response: Final = gateway.request( + "POST", + "/spend/calculate", + { + "completion_response": { + "id": "chatcmpl-live-preview", + "object": "chat.completion", + "created": 1677652288, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "live preview answer"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 101_000, + "completion_tokens": 0, + "total_tokens": 101_000, + "prompt_tokens_details": {"cached_tokens": cached_tokens}, + }, + } + }, + ) + assert response.status_code == 200, response.text + cost: Final = object_value(JSON_OBJECT.validate_json(response.text))["cost"] + assert isinstance(cost, int | float) + return float(cost) + + cached_cost: Final = cost_with_cached_tokens(100_000) + fresh_cost: Final = cost_with_cached_tokens(0) + assert fresh_cost > 0, fresh_cost + assert cached_cost == pytest.approx(fresh_cost), ( + f"the entry publishes no cached rate, so 100k cached tokens must bill like fresh ones: {cached_cost} vs {fresh_cost}" + ) diff --git a/tests/integration/spend/test_team_daily_activity_aggregated.py b/tests/integration/spend/test_team_daily_activity_aggregated.py new file mode 100644 index 00000000000..53c2f2efb1d --- /dev/null +++ b/tests/integration/spend/test_team_daily_activity_aggregated.py @@ -0,0 +1,81 @@ +import uuid +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows + + +@pytest.mark.covers("quota_management.spend_tracking.team_daily_activity_aggregated_reports_whole_range_team_spend") +def test_aggregated_team_activity_reports_the_whole_range_team_spend_in_one_page(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + team: Final = scenario.team(models=[model]) + keys: Final = tuple(scenario.key(team_id=team, models=[model]) for _ in range(2)) + digests: Final = tuple(sha256(key.encode()).hexdigest() for key in keys) + for key in keys: + for _ in range(2): + reply: Final = gateway.chat(model, key=key, text=f"team activity {uuid.uuid4().hex}") + assert reply["usage"]["total_tokens"] == 40, reply + logged: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE team_id=%s', (team,)), + lambda values: len(values) == 4, + seconds=70, + ) + assert sum(float(row["spend"]) for row in logged) == pytest.approx(0.24) + daily: Final = eventually( + lambda: read_rows( + 'SELECT api_key, spend, successful_requests FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,) + ), + lambda values: sum(float(row["spend"]) for row in values) >= 0.24 - 1e-9, + seconds=70, + ) + assert sorted(row["api_key"] for row in daily) == sorted(digests), daily + assert all(float(row["spend"]) == pytest.approx(0.12) and row["successful_requests"] == 2 for row in daily) + today: Final = datetime.now(timezone.utc) + response: Final = gateway.request( + "GET", + "/team/daily/activity/aggregated", + params={ + "team_ids": team, + "start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"), + "end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"), + "timezone": "0", + }, + ) + assert response.status_code == 200, response.text + body: Final = object_value(response.json()) + metadata: Final = object_value(body["metadata"]) + assert ( + metadata["total_spend"], + metadata["total_prompt_tokens"], + metadata["total_completion_tokens"], + metadata["total_tokens"], + metadata["total_api_requests"], + metadata["total_successful_requests"], + metadata["total_failed_requests"], + metadata["page"], + metadata["total_pages"], + metadata["has_more"], + ) == (pytest.approx(0.24), 80, 80, 160, 4, 4, 0, 1, 1, False), response.text + results: Final = body["results"] + assert isinstance(results, list) and len(results) == 1, response.text + day: Final = object_value(results[0]) + assert object_value(day["metrics"])["spend"] == pytest.approx(0.24), response.text + entities: Final = object_value(object_value(day["breakdown"])["entities"]) + assert set(entities) == {team}, response.text + team_bucket: Final = object_value(entities[team]) + team_metrics: Final = object_value(team_bucket["metrics"]) + assert (team_metrics["spend"], team_metrics["api_requests"], team_metrics["successful_requests"]) == ( + pytest.approx(0.24), + 4, + 4, + ), response.text + per_key: Final = object_value(team_bucket["api_key_breakdown"]) + assert set(per_key) == set(digests), response.text + assert tuple(object_value(object_value(per_key[digest])["metrics"])["spend"] for digest in digests) == ( + pytest.approx(0.12), + pytest.approx(0.12), + ), response.text diff --git a/tests/integration/spend/test_team_member_spend.py b/tests/integration/spend/test_team_member_spend.py new file mode 100644 index 00000000000..89eb2a24bcf --- /dev/null +++ b/tests/integration/spend/test_team_member_spend.py @@ -0,0 +1,54 @@ +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows + + +@pytest.mark.covers("spend.team_member.member_without_budget_gets_membership_row_and_spend") +def test_member_added_without_any_budget_is_charged_on_its_membership_row(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + team: Final = scenario.team(models=[model]) + user: Final = scenario.user() + added: Final = gateway.request( + "POST", "/team/member_add", {"team_id": team, "member": {"user_id": user, "role": "user"}} + ) + assert added.status_code == 200, added.text + memberships: Final = added.json()["updated_team_memberships"] + assert [ + {"user_id": row["user_id"], "team_id": row["team_id"], "budget_id": row["budget_id"], "spend": row["spend"]} + for row in memberships + ] == [{"user_id": user, "team_id": team, "budget_id": None, "spend": 0}], added.text + assert read_rows( + 'SELECT budget_id, spend, total_spend FROM "LiteLLM_TeamMembership" WHERE team_id=%s AND user_id=%s', + (team, user), + ) == [{"budget_id": None, "spend": 0.0, "total_spend": 0.0}] + key: Final = scenario.key(team_id=team, user_id=user, models=[model]) + assert gateway.chat(model, key=key, text=f"member spend {uuid.uuid4().hex}")["usage"]["total_tokens"] == 40 + charged: Final = eventually( + lambda: read_rows( + 'SELECT spend, total_spend FROM "LiteLLM_TeamMembership" WHERE team_id=%s AND user_id=%s', + (team, user), + ), + lambda values: len(values) == 1 and float(values[0]["spend"]) >= 0.06, + seconds=70, + ) + assert float(charged[0]["spend"]) == pytest.approx(0.06) + assert float(charged[0]["total_spend"]) == pytest.approx(0.06) + team_rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_TeamTable" WHERE team_id=%s', (team,)), + lambda values: len(values) == 1 and float(values[0]["spend"]) >= 0.06, + seconds=70, + ) + assert float(team_rows[0]["spend"]) == pytest.approx(0.06) + info: Final = gateway.get("/team/info", {"team_id": team}) + listed: Final = info["team_memberships"] + assert isinstance(listed, list) + exposed: Final = [ + (object_value(row)["user_id"], object_value(row)["spend"]) + for row in listed + if object_value(row)["user_id"] == user + ] + assert len(exposed) == 1 and exposed[0][1] == pytest.approx(0.06), info diff --git a/tests/integration/streaming/test_stream_contracts.py b/tests/integration/streaming/test_stream_contracts.py index 0c0fd8bc47c..7466a9454b8 100644 --- a/tests/integration/streaming/test_stream_contracts.py +++ b/tests/integration/streaming/test_stream_contracts.py @@ -2,25 +2,47 @@ import asyncio import json import threading import uuid +from pathlib import Path from typing import Final import pytest -from hypothesis import Phase, example, given, settings, strategies as st -from openai import OpenAI - +import yaml +from hypothesis import Phase, example, given, settings +from hypothesis import strategies as st from integration._support.client import Gateway, eventually from integration._support.database import read_rows +from integration._support.process import owned_proxy from integration._support.wire import Reply, wire_server +from openai import OpenAI def frame(identity: str, delta: dict, *, finish: str | None = None) -> bytes: - value: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini", "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} + value: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } return b"data: " + json.dumps(value, ensure_ascii=False).encode() + b"\n\n" def text_stream(identity: str) -> tuple[bytes, ...]: - usage: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini", "choices": [], "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}} - return (frame(identity, {"role": "assistant", "content": "Hello "}), frame(identity, {"content": "雪 café"}), frame(identity, {}, finish="stop"), b"data: " + json.dumps(usage).encode() + b"\n\n", b"data: [DONE]\n\n") + usage: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + } + return ( + frame(identity, {"role": "assistant", "content": "Hello "}), + frame(identity, {"content": "雪 café"}), + frame(identity, {}, finish="stop"), + b"data: " + json.dumps(usage).encode() + b"\n\n", + b"data: [DONE]\n\n", + ) @pytest.mark.covers("other.streaming.byte_partitions.preserve_text_identity_and_usage") @@ -37,14 +59,27 @@ def test_generated_tcp_partitions_preserve_unicode_text_identity_and_final_usage boundaries: Final = (0, *sorted(cuts), len(body)) pieces: Final = tuple(body[left:right] for left, right in zip(boundaries, boundaries[1:])) with wire_server(lambda request: Reply(content_type="text/event-stream", chunks=pieces)) as wire: - stream: Final = litellm.completion(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="synthetic-stream-key", messages=[{"role": "user", "content": "partition control"}], stream=True, stream_options={"include_usage": True}, timeout=5, num_retries=0) + stream: Final = litellm.completion( + model="openai/gpt-4o-mini", + api_base=wire.url + "/v1", + api_key="synthetic-stream-key", + messages=[{"role": "user", "content": "partition control"}], + stream=True, + stream_options={"include_usage": True}, + timeout=5, + num_retries=0, + ) try: chunks: Final = tuple(stream) finally: asyncio.run(stream.aclose()) - assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == "Hello 雪 café" + assert ( + "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == "Hello 雪 café" + ) assert {chunk.id for chunk in chunks} == {"stream-partition-control"} - assert [choice.finish_reason for chunk in chunks for choice in chunk.choices if choice.finish_reason] == ["stop"] + assert [choice.finish_reason for chunk in chunks for choice in chunk.choices if choice.finish_reason] == [ + "stop" + ] usages: Final = tuple(chunk.usage for chunk in chunks if getattr(chunk, "usage", None) is not None) assert len(usages) == 1 assert usages[0].prompt_tokens == 11 and usages[0].completion_tokens == 4 @@ -59,25 +94,63 @@ def test_fragmented_tool_names_and_arguments_keep_each_call_identity() -> None: identity: Final = "stream-tools-control" deltas: Final = ( - {"role": "assistant", "tool_calls": [{"index": 0, "id": "call-add", "type": "function", "function": {"name": "ad", "arguments": ""}}, {"index": 1, "id": "call-multiply", "type": "function", "function": {"name": "multi", "arguments": ""}}]}, - {"tool_calls": [{"index": 1, "function": {"name": "ply", "arguments": '{"x":3,'}}, {"index": 0, "function": {"arguments": '{"x":1,'}}]}, - {"tool_calls": [{"index": 0, "function": {"name": "d", "arguments": '"y":2}'}}, {"index": 1, "function": {"arguments": '"y":4}'}}]}, + { + "role": "assistant", + "tool_calls": [ + {"index": 0, "id": "call-add", "type": "function", "function": {"name": "ad", "arguments": ""}}, + {"index": 1, "id": "call-multiply", "type": "function", "function": {"name": "multi", "arguments": ""}}, + ], + }, + { + "tool_calls": [ + {"index": 1, "function": {"name": "ply", "arguments": '{"x":3,'}}, + {"index": 0, "function": {"arguments": '{"x":1,'}}, + ] + }, + { + "tool_calls": [ + {"index": 0, "function": {"name": "d", "arguments": '"y":2}'}}, + {"index": 1, "function": {"arguments": '"y":4}'}}, + ] + }, + ) + frames: Final = ( + *tuple(frame(identity, delta) for delta in deltas), + frame(identity, {}, finish="tool_calls"), + b"data: [DONE]\n\n", ) - frames: Final = (*tuple(frame(identity, delta) for delta in deltas), frame(identity, {}, finish="tool_calls"), b"data: [DONE]\n\n") with wire_server(lambda request: Reply(content_type="text/event-stream", chunks=frames)) as wire: - stream: Final = litellm.completion(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="synthetic-stream-key", messages=[{"role": "user", "content": "tool control"}], stream=True, timeout=5, num_retries=0) + stream: Final = litellm.completion( + model="openai/gpt-4o-mini", + api_base=wire.url + "/v1", + api_key="synthetic-stream-key", + messages=[{"role": "user", "content": "tool control"}], + stream=True, + timeout=5, + num_retries=0, + ) try: chunks: Final = tuple(stream) finally: asyncio.run(stream.aclose()) - events: Final = tuple((choice.index, tool) for chunk in chunks for choice in chunk.choices for tool in (choice.delta.tool_calls or ())) - for index, name, call_id, arguments in ((0, "add", "call-add", {"x": 1, "y": 2}), (1, "multiply", "call-multiply", {"x": 3, "y": 4})): + events: Final = tuple( + (choice.index, tool) + for chunk in chunks + for choice in chunk.choices + for tool in (choice.delta.tool_calls or ()) + ) + for index, name, call_id, arguments in ( + (0, "add", "call-add", {"x": 1, "y": 2}), + (1, "multiply", "call-multiply", {"x": 3, "y": 4}), + ): selected: Final = tuple(tool for choice, tool in events if (choice, tool.index) == (0, index)) assert "".join(tool.id or "" for tool in selected) == call_id assert "".join(tool.function.name or "" for tool in selected) == name assert json.loads("".join(tool.function.arguments or "" for tool in selected)) == arguments assert {tool.index for _, tool in events} == {0, 1} - assert [choice.finish_reason for chunk in chunks for choice in chunk.choices if choice.finish_reason] == ["tool_calls"] + assert [choice.finish_reason for chunk in chunks for choice in chunk.choices if choice.finish_reason] == [ + "tool_calls" + ] assert len(wire.drain()) == 1 @@ -86,13 +159,27 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat with gateway.scenario() as scenario: for include in (None, False, True): identity: Final = "stream-usage-" + uuid.uuid4().hex - with wire_server(lambda request, identity=identity: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire: - model: Final = scenario.model(api_base=wire.url + "/v1", input_cost_per_token=0.001, output_cost_per_token=0.002) - with OpenAI(api_key=gateway.key, base_url=str(gateway.client.base_url), timeout=5, max_retries=0) as client: - stream: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": identity}], stream=True, **({} if include is None else {"stream_options": {"include_usage": include}})) + with wire_server( + lambda request, identity=identity: Reply(content_type="text/event-stream", chunks=text_stream(identity)) + ) as wire: + model: Final = scenario.model( + api_base=wire.url + "/v1", input_cost_per_token=0.001, output_cost_per_token=0.002 + ) + with OpenAI( + api_key=gateway.key, base_url=str(gateway.client.base_url), timeout=5, max_retries=0 + ) as client: + stream: Final = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": identity}], + stream=True, + **({} if include is None else {"stream_options": {"include_usage": include}}), + ) with stream: chunks: Final = tuple(stream) - assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == "Hello 雪 café" + assert ( + "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + == "Hello 雪 café" + ) assert {chunk.id for chunk in chunks} == {identity} usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None) assert len(usages) == (1 if include else 0) @@ -101,28 +188,369 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat requests: Final = wire.drain() assert len(requests) == 1 assert json.loads(requests[0].body)["stream_options"]["include_usage"] is True - rows: Final = eventually(lambda identity=identity: read_rows('SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,)), lambda values: len(values) == 1, seconds=70) + rows: Final = eventually( + lambda identity=identity: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) assert rows[0]["prompt_tokens"] == 11 and rows[0]["completion_tokens"] == 4 assert float(rows[0]["spend"]) == pytest.approx(0.019) +@pytest.mark.covers("other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream") +def test_messages_stream_completes_through_trailing_empty_choices_usage_chunk(gateway: Gateway) -> None: + identity: Final = "messages-empty-choices-" + uuid.uuid4().hex + metadata: Final = ( + b"data: " + + json.dumps( + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {}}], + }, + ensure_ascii=False, + ).encode() + + b"\n\n" + ) + frames: Final = (metadata, *text_stream(identity)) + with ( + wire_server(lambda request: Reply(content_type="text/event-stream", chunks=frames)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model="azure/gpt-4o-mini", api_base=wire.url + "/v1") + with gateway.client.stream( + "POST", + "/v1/messages", + json={ + "model": model, + "max_tokens": 64, + "stream": True, + "messages": [{"role": "user", "content": identity}], + }, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read().decode() + events: Final = tuple( + json.loads(line.removeprefix("data: ")) for line in response.iter_lines() if line.startswith("data: ") + ) + assert tuple(event["type"] for event in events) == ( + "message_start", + "content_block_start", + "content_block_delta", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ), f"observed events: {events!r}" + assert ( + "".join(event["delta"]["text"] for event in events if event["type"] == "content_block_delta") == "Hello 雪 café" + ) + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["usage"] == {"input_tokens": 11, "output_tokens": 4} + requests: Final = wire.drain() + assert len(requests) == 1 + outbound: Final = json.loads(requests[0].body) + assert outbound["stream"] is True and outbound["stream_options"] == {"include_usage": True}, ( + f"observed outbound body: {outbound!r}" + ) + + +@pytest.mark.covers("other.streaming.responses_bridge.empty_choices_chunks_complete_stream") +def test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks(gateway: Gateway) -> None: + identity: Final = "responses-empty-choices-" + uuid.uuid4().hex + metadata: Final = ( + b"data: " + + json.dumps( + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {}}], + }, + ensure_ascii=False, + ).encode() + + b"\n\n" + ) + frames: Final = (metadata, *text_stream(identity)) + with ( + wire_server(lambda request: Reply(content_type="text/event-stream", chunks=frames)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model="deepseek/gpt-4o-mini", api_base=wire.url + "/v1") + with gateway.client.stream( + "POST", + "/v1/responses", + json={"model": model, "input": identity, "stream": True}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read().decode() + events: Final = tuple( + json.loads(line.removeprefix("data: ")) + for line in response.iter_lines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert ( + "".join(event["delta"] for event in events if event["type"] == "response.output_text.delta") == "Hello 雪 café" + ), f"observed events: {events!r}" + assert tuple(event["type"] for event in events if event["type"] != "response.output_text.delta") == ( + "response.created", + "response.in_progress", + "response.output_item.added", + "response.content_part.added", + "response.output_text.done", + "response.content_part.done", + "response.output_item.done", + "response.completed", + ), f"observed events: {events!r}" + assert events[-1]["type"] == "response.completed" + assert events[-1]["response"]["usage"] == { + "input_tokens": 11, + "output_tokens": 4, + "output_tokens_details": {"reasoning_tokens": 0, "text_tokens": 4}, + "total_tokens": 15, + } + requests: Final = wire.drain() + assert len(requests) == 1 + outbound: Final = json.loads(requests[0].body) + assert outbound["stream"] is True and outbound["stream_options"] == {"include_usage": True}, ( + f"observed outbound body: {outbound!r}" + ) + + +def provider_cost_object_stream(identity: str, total_cost: float) -> tuple[bytes, ...]: + cost: Final = { + "input_tokens_cost": 0.0001, + "output_tokens_cost": 0.0002, + "request_cost": 0.012, + "total_cost": total_cost, + } + usage: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "sonar", + "choices": [], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15, "cost": cost}, + } + return ( + frame(identity, {"role": "assistant", "content": "Hello "}), + frame(identity, {"content": "from search"}), + frame(identity, {}, finish="stop"), + b"data: " + json.dumps(usage).encode() + b"\n\n", + b"data: [DONE]\n\n", + ) + + +def sse_data_lines(text: str) -> tuple[str, ...]: + return tuple(line.removeprefix("data: ") for line in text.splitlines() if line.startswith("data: ")) + + +@pytest.mark.covers("other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost") +def test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost(gateway: Gateway) -> None: + identity: Final = "stream-cost-object-" + uuid.uuid4().hex + total_cost: Final = 0.0123 + with ( + gateway.scenario() as scenario, + wire_server( + lambda request: Reply( + content_type="text/event-stream", chunks=provider_cost_object_stream(identity, total_cost) + ) + ) as wire, + ): + model: Final = scenario.model(model="perplexity/sonar", api_base=wire.url + "/v1") + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": identity}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + text: Final = response.read().decode() + assert response.status_code == 200, text + lines: Final = sse_data_lines(text) + assert lines[-1] == "[DONE]", text + events: Final = tuple(json.loads(line) for line in lines[:-1]) + assert [event for event in events if "error" in event] == [], text + assert ( + "".join(choice["delta"].get("content") or "" for event in events for choice in event["choices"]) + == "Hello from search" + ), text + assert [ + choice.get("finish_reason") + for event in events + for choice in event["choices"] + if choice.get("finish_reason") + ] == ["stop"], text + usages: Final = tuple(event["usage"] for event in events if event.get("usage") is not None) + assert len(usages) == 1, text + assert (usages[0]["prompt_tokens"], usages[0]["completion_tokens"], usages[0]["total_tokens"]) == (11, 4, 15), ( + text + ) + requests: Final = wire.drain() + assert len(requests) == 1 + outbound: Final = json.loads(requests[0].body) + assert outbound["model"] == "sonar" and outbound["stream"] is True, outbound + assert outbound["messages"] == [{"role": "user", "content": identity}], outbound + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (11, 4) + assert float(rows[0]["spend"]) == pytest.approx(total_cost) + + +@pytest.mark.covers( + "other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend" +) +def test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback( + gateway: Gateway, + tmp_path: Path, +) -> None: + identity: Final = "stream-empty-fallback-" + uuid.uuid4().hex + empty_first: Final = ( + b"data: " + + json.dumps( + { + "id": identity + "-primary", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "usage": {"prompt_tokens": 11, "completion_tokens": 0, "total_tokens": 11}, + } + ).encode() + + b"\n\n" + ) + with ( + wire_server( + lambda request: Reply( + content_type="text/event-stream", + chunks=(empty_first, b":" + b"x" * 4_000_000 + b"\n\n", empty_first), + abort_after=2, + ) + ) as primary, + wire_server( + lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity)) + ) as fallback, + ): + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": name, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "synthetic-fallback-key", + "api_base": server.url + "/v1", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + } + for name, server in (("primary", primary), ("fallback", fallback)) + ] + config["router_settings"] = { + "num_retries": 0, + "disable_cooldowns": True, + "fallbacks": [{"primary": ["fallback"]}], + } + path: Final = tmp_path / "fallbacks.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate: + body: Final = { + "model": "primary", + "messages": [{"role": "user", "content": identity}], + "stream": True, + "stream_options": {"include_usage": True}, + } + with candidate.client.stream( + "POST", "/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {candidate.key}"} + ) as response: + lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data:")) + assert response.status_code == 200, lines + assert lines[-1] == "data: [DONE]", lines + events: Final = tuple(json.loads(line.removeprefix("data:")) for line in lines[:-1]) + assert all("error" not in event for event in events), lines + assert ( + "".join(choice["delta"].get("content") or "" for event in events for choice in event["choices"]) + == "Hello 雪 café" + ), lines + usages: Final = tuple(event["usage"] for event in events if event.get("usage") is not None) + assert (usages[-1]["prompt_tokens"], usages[-1]["completion_tokens"]) == (11, 4), lines + assert tuple( + json.loads(request.body)["messages"] + for request in primary.drain() + if request.target.endswith("/chat/completions") + ) == (body["messages"],) + assert tuple( + json.loads(request.body)["messages"] + for request in fallback.drain() + if request.target.endswith("/chat/completions") + ) == (body["messages"],) + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens, status FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"], rows[0]["status"]) == (11, 4, "success"), ( + rows + ) + assert float(rows[0]["spend"]) == pytest.approx(0.019), rows + + @pytest.mark.covers("other.streaming.failure.truncated_transport_raises_and_control_recovers") def test_truncated_http_stream_is_an_error_and_next_stream_succeeds() -> None: import litellm for truncated in (True, False): - with wire_server(lambda request, truncated=truncated: Reply(content_type="text/event-stream", chunks=text_stream("stream-truncated"), abort_after=1 if truncated else None)) as wire: - stream: Final = litellm.completion(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="synthetic-stream-key", messages=[{"role": "user", "content": "truncation control"}], stream=True, timeout=5, num_retries=0) + with wire_server( + lambda request, truncated=truncated: Reply( + content_type="text/event-stream", + chunks=text_stream("stream-truncated"), + abort_after=1 if truncated else None, + ) + ) as wire: + stream: Final = litellm.completion( + model="openai/gpt-4o-mini", + api_base=wire.url + "/v1", + api_key="synthetic-stream-key", + messages=[{"role": "user", "content": "truncation control"}], + stream=True, + timeout=5, + num_retries=0, + ) try: if truncated: - with pytest.raises(litellm.exceptions.MidStreamFallbackError, match="incomplete chunked read") as failure: + with pytest.raises( + litellm.exceptions.MidStreamFallbackError, match="incomplete chunked read" + ) as failure: tuple(stream) assert isinstance(failure.value.original_exception, litellm.APIConnectionError) assert failure.value.generated_content == "Hello " assert failure.value.is_pre_first_chunk is False else: chunks: Final = tuple(stream) - assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == "Hello 雪 café" + assert ( + "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + == "Hello 雪 café" + ) assert any(choice.finish_reason == "stop" for chunk in chunks for choice in chunk.choices) finally: asyncio.run(stream.aclose()) @@ -134,9 +562,23 @@ def test_client_cancellation_releases_the_actual_provider_connection() -> None: import litellm gate: Final = threading.Event() - frames: Final = (frame("stream-cancel", {"role": "assistant", "content": "first"}), b":" + b"x" * 4_000_000 + b"\n\n", b"data: [DONE]\n\n") - with wire_server(lambda request: Reply(content_type="text/event-stream", chunks=frames, gate_after_first=gate)) as wire: - stream: Final = litellm.completion(model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="synthetic-stream-key", messages=[{"role": "user", "content": "cancellation control"}], stream=True, timeout=5, num_retries=0) + frames: Final = ( + frame("stream-cancel", {"role": "assistant", "content": "first"}), + b":" + b"x" * 4_000_000 + b"\n\n", + b"data: [DONE]\n\n", + ) + with wire_server( + lambda request: Reply(content_type="text/event-stream", chunks=frames, gate_after_first=gate) + ) as wire: + stream: Final = litellm.completion( + model="openai/gpt-4o-mini", + api_base=wire.url + "/v1", + api_key="synthetic-stream-key", + messages=[{"role": "user", "content": "cancellation control"}], + stream=True, + timeout=5, + num_retries=0, + ) try: first: Final = next(stream) assert first.choices[0].delta.content == "first" diff --git a/tests/integration/streaming/test_stream_parallel_slot_release.py b/tests/integration/streaming/test_stream_parallel_slot_release.py new file mode 100644 index 00000000000..a1dc038208d --- /dev/null +++ b/tests/integration/streaming/test_stream_parallel_slot_release.py @@ -0,0 +1,86 @@ +import json +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + + +def frame(identity: str, delta: dict[str, str], *, finish: str | None = None) -> bytes: + event: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + return b"data: " + json.dumps(event).encode() + b"\n\n" + + +@pytest.mark.covers("streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails") +def test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot( + gateway: Gateway, tmp_path: Path +) -> None: + identity: Final = "stream-slot-" + uuid.uuid4().hex + prompt: Final = "slot release control " + identity + + def analyzer(request: Request) -> Reply: + assert request.target == "/analyze" + assert json.loads(request.body)["text"] == prompt + return Reply(status=500, body=json.dumps({"error": "synthetic analyzer outage"}).encode()) + + def provider(request: Request) -> Reply: + assert request.target == "/v1/chat/completions" + body: Final = json.loads(request.body) + assert body["messages"] == [{"role": "user", "content": prompt}] + assert body["stream"] is True + return Reply( + content_type="text/event-stream", + chunks=( + frame(identity, {"role": "assistant", "content": "Hello"}), + frame(identity, {"content": " slot"}), + frame(identity, {}, finish="stop"), + b"data: [DONE]\n\n", + ), + ) + + with wire_server(analyzer) as policy, wire_server(provider) as upstream: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["guardrails"] = [ + { + "guardrail_name": identity, + "litellm_params": { + "guardrail": "presidio", + "mode": "logging_only", + "default_on": True, + "presidio_filter_scope": "input", + "pii_entities_config": {"EMAIL_ADDRESS": "MASK"}, + "presidio_analyzer_api_base": policy.url + "/", + "presidio_anonymizer_api_base": policy.url + "/", + }, + } + ] + path: Final = tmp_path / "failing_logging_guardrail.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + key: Final = scenario.key(max_parallel_requests=1) + body: Final = {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True} + first: Final = candidate.request("POST", "/v1/chat/completions", body, key=key) + assert first.status_code == 200, first.text + assert first.text.endswith("data: [DONE]\n\n"), first.text + assert len(upstream.drain()) == 1 + eventually(lambda: policy.received.qsize(), lambda count: count >= 1) + assert {scan.target for scan in policy.drain()} == {"/analyze"} + second: Final = eventually( + lambda: candidate.request("POST", "/v1/chat/completions", body, key=key), + lambda response: response.status_code == 200, + seconds=20, + return_last_on_timeout=True, + ) + assert second.status_code == 200, second.text + assert second.text.endswith("data: [DONE]\n\n"), second.text diff --git a/tests/integration/streaming/test_ttft_keepalive.py b/tests/integration/streaming/test_ttft_keepalive.py new file mode 100644 index 00000000000..3f8c1bbc112 --- /dev/null +++ b/tests/integration/streaming/test_ttft_keepalive.py @@ -0,0 +1,62 @@ +import json +import threading +import uuid +from collections.abc import Callable, Iterable, Iterator +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from integration.streaming.test_stream_contracts import text_stream + +KEEPALIVE_SECONDS: Final = 1 + + +def _reply_after_first_ping(identity: str, first_ping_seen: threading.Event) -> Callable[[Request], Reply]: + def respond(_request: Request) -> Reply: + first_ping_seen.wait(timeout=10) + return Reply(content_type="text/event-stream", chunks=text_stream(identity)) + + return respond + + +def _frames_setting(first_ping_seen: threading.Event, lines: Iterable[str]) -> Iterator[str]: + for line in lines: + if line == ": ping": + first_ping_seen.set() + yield line + + +@pytest.mark.covers("streaming.keepalive.sse_pings_fill_silent_time_to_first_token") +def test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent( + gateway: Gateway, +) -> None: + identity: Final = "stream-ttft-keepalive-" + uuid.uuid4().hex + first_ping_seen: Final = threading.Event() + with gateway.scenario() as scenario: + with wire_server(_reply_after_first_ping(identity, first_ping_seen)) as wire: + model: Final = scenario.model(api_base=wire.url + "/v1", keepalive_seconds=KEEPALIVE_SECONDS) + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": identity}], "stream": True}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read().decode() + frames: Final = tuple( + _frames_setting(first_ping_seen, (line for line in response.iter_lines() if line)) + ) + first_data: Final = next(index for index, line in enumerate(frames) if line.startswith("data:")) + assert first_data >= 1, f"No keepalive reached the client before the first data frame: {frames}" + assert frames[:first_data] == (": ping",) * first_data, frames + assert frames[-1] == "data: [DONE]", frames + deltas: Final = tuple(json.loads(line.removeprefix("data: ")) for line in frames[first_data:-1]) + assert ( + "".join(choice["delta"].get("content", "") for chunk in deltas for choice in chunk["choices"]) + == "Hello 雪 café" + ), frames + requests: Final = wire.drain() + assert len(requests) == 1 + outbound: Final = json.loads(requests[0].body) + assert outbound["model"] == "gpt-4o-mini" and outbound["stream"] is True, outbound + assert "keepalive_seconds" not in outbound, outbound diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 8b45ae08813..c85ad7fc779 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -3331,7 +3331,7 @@ def test_gemini_fine_tuned_model_request_consistency(): Assert the same transformation is applied to Fine tuned gemini 2.0 flash and gemini 2.0 flash - Request 1: Fine tuned: vertex_ai/gemini/ft-uuid - - Request 2: vertex_ai/gemini-2.0-flash-001 + - Request 2: vertex_ai/gemini-2.5-flash """ litellm.set_verbose = True load_vertex_ai_credentials() @@ -3403,7 +3403,7 @@ def test_gemini_fine_tuned_model_request_consistency(): with patch.object(client, "post", new=MagicMock()) as mock_post_2: try: response_2 = completion( - model="vertex_ai/gemini-2.0-flash-001", + model="vertex_ai/gemini-2.5-flash", messages=messages, tools=tools, tool_choice="auto", diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 3ce99f893d8..c24e6c32369 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -5,7 +5,7 @@ import litellm.cost_calculator import asyncio import time -from typing import Optional +from typing import Final, Optional from unittest.mock import MagicMock, patch import pytest @@ -21,6 +21,9 @@ import json import httpx from litellm.types.utils import PromptTokensDetails from litellm.litellm_core_utils.litellm_logging import CustomLogger +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_model_response_object, +) class CustomLoggingHandler(CustomLogger): @@ -328,35 +331,39 @@ def test_whisper_azure(): assert round(cost, 5) == round(expected_cost, 5) -def test_dalle_3_azure_cost_tracking(): - litellm.set_verbose = True - # model = "azure/dall-e-3-test" - # response = litellm.image_generation( - # model=model, - # prompt="A cute baby sea otter", - # api_version="2023-12-01-preview", - # api_base=os.getenv("AZURE_SWEDEN_API_BASE"), - # api_key=os.getenv("AZURE_SWEDEN_API_KEY"), - # base_model="dall-e-3", - # ) - # print(f"response: {response}") - response = litellm.ImageResponse( - created=1710265780, - data=[ - { - "b64_json": None, - "revised_prompt": "A close-up image of an adorable baby sea otter. Its fur is thick and fluffy to provide buoyancy and insulation against the cold water. Its eyes are round, curious and full of life. It's lying on its back, floating effortlessly on the calm sea surface under the warm sun. Surrounding the otter are patches of colorful kelp drifting along the gentle waves, giving the scene a touch of vibrancy. The sea otter has its small paws folded on its chest, and it seems to be taking a break from its play.", - "url": "test-azure-blob-url-with-sas-token", - } - ], +def test_gpt_image_2_azure_cost_tracking(): + azure_image_generation_response: Final = { + "created": 1758585600, + "data": [{"b64_json": "iVBORw0KGgo=", "revised_prompt": None, "url": None}], + "output_format": "png", + "quality": "low", + "size": "1024x1024", + "usage": { + "input_tokens": 12, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 12}, + "output_tokens": 196, + "output_tokens_details": {"image_tokens": 196, "text_tokens": 0}, + "total_tokens": 208, + }, + } + response: Final = convert_to_model_response_object( + response_object=azure_image_generation_response, + model_response_object=litellm.ImageResponse(), + response_type="image_generation", + hidden_params={"model": "gpt-image-2", "custom_llm_provider": "azure"}, ) - response.usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - response._hidden_params = {"model": "dall-e-3", "model_id": None} - print(f"response hidden params: {response._hidden_params}") - cost = litellm.completion_cost( - completion_response=response, call_type="image_generation" + + cost: Final = litellm.completion_cost( + completion_response=response, + model="azure/my-gpt-image-2-deployment", + custom_llm_provider="azure", + base_model="gpt-image-2", + call_type="image_generation", ) - assert cost > 0 + + pricing: Final = litellm.model_cost["azure/gpt-image-2"] + expected_cost: Final = pricing["input_cost_per_token"] * 12 + pricing["output_cost_per_image_token"] * 196 + assert round(cost, 8) == round(expected_cost, 8) def test_replicate_llama3_cost_tracking(): diff --git a/tests/local_testing/whitelisted_bedrock_models.txt b/tests/local_testing/whitelisted_bedrock_models.txt index 7615a540b23..9b66569ff9e 100644 --- a/tests/local_testing/whitelisted_bedrock_models.txt +++ b/tests/local_testing/whitelisted_bedrock_models.txt @@ -139,3 +139,4 @@ bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b bedrock/eu-west-1/qwen.qwen3-next-80b-a3b bedrock/eu-west-2/qwen.qwen3-next-80b-a3b bedrock/sa-east-1/qwen.qwen3-next-80b-a3b +bedrock/eu-west-2/nvidia.nemotron-super-3-120b diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index f92c610df24..ad77f9d4d1b 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1,12 +1,11 @@ import json import os +from typing import Final +from unittest.mock import MagicMock, patch import httpx import pytest -from typing import Final -from unittest.mock import MagicMock, patch - import litellm from litellm import ModelResponse from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig @@ -802,13 +801,13 @@ def test_output_config_format_translated_to_native_output_config_converse(): } result = config._transform_request( - model="bedrock/converse/us.anthropic.claude-opus-4-7", + model="bedrock/converse/us.anthropic.claude-sonnet-4-6", messages=[{"role": "user", "content": "hi"}], optional_params={ "maxTokens": 256, "thinking": {"type": "adaptive"}, "output_config": { - "effort": "xhigh", + "effort": "max", "format": {"type": "json_schema", "schema": schema}, }, }, @@ -817,7 +816,7 @@ def test_output_config_format_translated_to_native_output_config_converse(): ) additional = result.get("additionalModelRequestFields", {}) - assert additional.get("output_config") == {"effort": "xhigh"} + assert additional.get("output_config") == {"effort": "max"} assert "format" not in additional["output_config"] assert result["outputConfig"]["textFormat"]["type"] == "json_schema" parsed_schema = json.loads( @@ -4292,6 +4291,84 @@ def test_translate_response_format_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) +BEDROCK_OPUS_4_7_AND_4_8_MODELS: Final = ( + "anthropic.claude-opus-4-7", + "global.anthropic.claude-opus-4-7", + "us.anthropic.claude-opus-4-7", + "eu.anthropic.claude-opus-4-7", + "au.anthropic.claude-opus-4-7", + "jp.anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "global.anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + "eu.anthropic.claude-opus-4-8", + "au.anthropic.claude-opus-4-8", + "jp.anthropic.claude-opus-4-8", + "us-gov.anthropic.claude-opus-4-8", + "us-gov-west-1/anthropic.claude-opus-4-8", + "us-gov-east-1/anthropic.claude-opus-4-8", +) + +CAPITAL_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "capital", + "schema": { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": False, + }, + }, +} + + +def _converse_request_for_json_schema(model: str, stream: bool) -> tuple[dict, dict]: + config = AmazonConverseConfig() + optional_params = config.map_openai_params( + non_default_params={"response_format": CAPITAL_RESPONSE_FORMAT, "stream": stream}, + optional_params={}, + model=model, + drop_params=False, + ) + request = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + return optional_params, request + + +@pytest.mark.parametrize("model", BEDROCK_OPUS_4_7_AND_4_8_MODELS) +@pytest.mark.parametrize("stream", [False, True]) +def test_opus_4_7_and_4_8_json_schema_sent_as_forced_tool_not_output_config(monkeypatch, model, stream): + """Regression for issue #27846: Bedrock rejects outputConfig on Opus 4.7 and 4.8 + (``output_config.format: Extra inputs are not permitted``), so json_schema has to + go out as the forced json_tool_call tool, streamed through fake_stream.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + optional_params, request = _converse_request_for_json_schema(model=model, stream=stream) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} + assert optional_params.get("fake_stream", False) is stream + + +def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + optional_params, request = _converse_request_for_json_schema(model="us.anthropic.claude-sonnet-4-6", stream=True) + + assert request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "capital" + assert "toolConfig" not in request + assert "fake_stream" not in optional_params + + def test_translate_response_format_fallback_tool_call(): """For unsupported models, should fall back to tool-call approach.""" config = AmazonConverseConfig() @@ -5417,9 +5494,14 @@ def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacki old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + cost_map = dict(litellm.get_model_cost_map(url="")) + cost_map["jp.anthropic.claude-opus-4-7"] = { + k: v + for k, v in cost_map["jp.anthropic.claude-opus-4-7"].items() + if k != "cache_creation_input_token_cost_above_1hr" + } + litellm.model_cost = cost_map try: - assert "cache_creation_input_token_cost_above_1hr" not in litellm.model_cost["jp.anthropic.claude-opus-4-7"] assert "cache_creation_input_token_cost_above_1hr" in litellm.model_cost["anthropic.claude-opus-4-7"] config = AmazonConverseConfig() messages = [ @@ -7500,3 +7582,92 @@ def test_eager_input_streaming_non_boolean_is_a_bad_request(): "us.anthropic.claude-sonnet-4-5-20250929-v1:0", [_eager_openai_tool(eager_input_streaming="true")], ) + + +@pytest.mark.parametrize("model", ("anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-7")) +def test_converse_accepts_anthropic_default_temperature(model: str) -> None: + result: Final = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="bedrock", + temperature=1, + drop_params=False, + ) + + assert result["temperature"] == 1 + + +def test_get_supported_openai_params_drops_sampling_params_for_gpt5_models(): + config = AmazonConverseConfig() + for model in [ + "bedrock/converse/global.openai.gpt-5.6-luna", + "global.openai.gpt-5.6-luna", + "global.openai.gpt-5.6-sol", + "us.openai.gpt-5.6-terra", + "eu.openai.gpt-5.6-luna", + "openai.gpt-5.6-luna", + "bedrock/openai.gpt-5.6-luna", + ]: + supported = config.get_supported_openai_params(model=model) + assert "temperature" not in supported + assert "top_p" not in supported + + supported_oss = config.get_supported_openai_params(model="openai.gpt-oss-120b-1:0") + assert "temperature" in supported_oss + assert "top_p" in supported_oss + + +def test_map_openai_params_drops_temperature_and_top_p_when_drop_params_true(): + config = AmazonConverseConfig() + for model in [ + "bedrock/converse/global.openai.gpt-5.6-luna", + "openai.gpt-5.6-luna", + "eu.openai.gpt-5.6-luna", + ]: + result = config.map_openai_params( + non_default_params={"temperature": 1.0, "top_p": 0.9, "max_tokens": 50}, + optional_params={}, + model=model, + drop_params=True, + ) + assert "temperature" not in result + assert "topP" not in result + assert result.get("maxTokens") == 50 + + +def test_map_openai_params_raises_unsupported_params_when_drop_params_false(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + config = AmazonConverseConfig() + for model in [ + "bedrock/converse/global.openai.gpt-5.6-luna", + "openai.gpt-5.6-luna", + ]: + with pytest.raises(litellm.utils.UnsupportedParamsError) as exc_info: + config.map_openai_params( + non_default_params={"temperature": 1.0}, + optional_params={}, + model=model, + drop_params=False, + ) + assert "does not support temperature=1.0" in str(exc_info.value) + + +def test_map_openai_params_retains_sampling_params_for_supported_models(): + config = AmazonConverseConfig() + result = config.map_openai_params( + non_default_params={"temperature": 0.7, "top_p": 0.8}, + optional_params={}, + model="openai.gpt-oss-120b-1:0", + drop_params=False, + ) + assert result.get("temperature") == 0.7 + assert result.get("topP") == 0.8 + + +def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "global.custom-test-reasoning-model", + {"supports_sampling_params": False}, + ) + assert AmazonConverseConfig._supports_sampling_params("custom-test-reasoning-model") is False + assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index c3f8c2ba903..466e9b4fda8 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -1,4 +1,10 @@ +import base64 +import binascii import datetime +import json +import struct +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx @@ -13,6 +19,7 @@ from litellm.llms.bedrock.chat.invoke_handler import ( make_sync_call, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.utils import ModelResponseStream def test_transform_thinking_blocks_with_redacted_content(): @@ -704,3 +711,91 @@ async def test_async_invoke_streaming_non_200_forwards_bedrock_response_headers( ) assert exc_info.value.response.headers["x-amzn-requestid"] == "req-non200-async" + + +def _bedrock_event_stream_frame(chunk: Mapping[str, object]) -> bytes: + def header(name: str, value: str) -> bytes: + return bytes([len(name)]) + name.encode() + bytes([7]) + struct.pack(">H", len(value)) + value.encode() + + headers: Final = header(":event-type", "chunk") + header(":content-type", "application/json") + header( + ":message-type", "event" + ) + payload: Final = json.dumps({"bytes": base64.b64encode(json.dumps(chunk).encode()).decode()}).encode() + prelude: Final = struct.pack(">II", 12 + len(headers) + len(payload) + 4, len(headers)) + body: Final = prelude + struct.pack(">I", binascii.crc32(prelude)) + headers + payload + return body + struct.pack(">I", binascii.crc32(body)) + + +def _openai_stream_chunk(delta: Mapping[str, str], finish_reason: str | None = None) -> Mapping[str, object]: + return { + "id": "chatcmpl-1", + "object": "chat.completion.chunk", + "created": 1, + "model": "moonshot.kimi-k2-thinking", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +_MOONSHOT_RAW_STREAM: Final = b"".join( + _bedrock_event_stream_frame(chunk) + for chunk in ( + _openai_stream_chunk({"role": "assistant", "reasoning_content": "thinking"}), + _openai_stream_chunk({"content": '{"city": '}), + _openai_stream_chunk({"content": '"San Francisco"}'}), + _openai_stream_chunk({}, "stop"), + ) +) + + +def _assert_moonshot_stream_content(chunks: Sequence[ModelResponseStream]) -> None: + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == '{"city": "San Francisco"}' + assert "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) == "thinking" + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"] + + +@pytest.fixture +def _aws_test_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATEST") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + + +@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}]) +def test_moonshot_invoke_stream_yields_openai_shaped_chunks( + _aws_test_credentials: None, response_format: Mapping[str, str] | None +) -> None: + raw_stream: Final = _MOONSHOT_RAW_STREAM + response: Final = MagicMock(status_code=200, headers={}) + response.iter_bytes = lambda chunk_size=None: iter([raw_stream]) + client: Final = HTTPHandler() + client.post = MagicMock(return_value=response) + + stream: Final = litellm.completion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=[{"role": "user", "content": "weather as json"}], + stream=True, + client=client, + **({"response_format": response_format} if response_format else {}), + ) + _assert_moonshot_stream_content(list(stream)) + + +@pytest.mark.asyncio +async def test_moonshot_invoke_async_stream_yields_openai_shaped_chunks(_aws_test_credentials: None) -> None: + async def _aiter_bytes(chunk_size: int | None = None) -> AsyncIterator[bytes]: + yield _MOONSHOT_RAW_STREAM + + response: Final = MagicMock(status_code=200, headers={}) + response.aiter_bytes = _aiter_bytes + client: Final = AsyncHTTPHandler() + client.post = AsyncMock(return_value=response) + + stream: Final = await litellm.acompletion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=[{"role": "user", "content": "weather as json"}], + stream=True, + response_format={"type": "json_object"}, + client=client, + ) + + _assert_moonshot_stream_content([chunk async for chunk in stream]) diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index ea8b722b849..f4d51d975bb 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1,12 +1,17 @@ import asyncio +import base64 import copy import json import os +import struct +import zlib from datetime import datetime from types import SimpleNamespace +from collections.abc import AsyncIterator, Mapping, Sequence from typing import Final from unittest.mock import Mock +import httpx import pytest # Ensure the project root is on the import path so `litellm` can be imported when @@ -3395,3 +3400,97 @@ def test_bedrock_invoke_eager_input_streaming_beta_not_duplicated_with_client_he ) assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA] + + +def _bedrock_event_frame(payload: Mapping[str, object]) -> bytes: + def _header(name: str, value: str) -> bytes: + return ( + bytes([len(name)]) + + name.encode() + + bytes([7]) + + struct.pack(">H", len(value)) + + value.encode() + ) + + headers: Final = ( + _header(":message-type", "event") + + _header(":event-type", "chunk") + + _header(":content-type", "application/json") + ) + body: Final = json.dumps( + {"bytes": base64.b64encode(json.dumps(payload).encode()).decode()} + ).encode() + prelude: Final = struct.pack(">II", 12 + len(headers) + len(body) + 4, len(headers)) + prelude_crc: Final = struct.pack(">I", zlib.crc32(prelude)) + message_crc: Final = struct.pack(">I", zlib.crc32(prelude + prelude_crc + headers + body)) + return prelude + prelude_crc + headers + body + message_crc + + +class _GatedAsyncByteStream(httpx.AsyncByteStream): + def __init__(self, chunks: Sequence[bytes], gate: asyncio.Event) -> None: + self._chunks = chunks + self._gate = gate + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._chunks[0] + await self._gate.wait() + for chunk in self._chunks[1:]: + yield chunk + + async def aclose(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_get_async_streaming_response_iterator_yields_small_frame_before_upstream_pauses(): + gate: Final = asyncio.Event() + response: Final = httpx.Response( + 200, + stream=_GatedAsyncByteStream( + chunks=( + _bedrock_event_frame( + { + "type": "message_start", + "message": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "content": [], + "model": "us.anthropic.claude-sonnet-4-6", + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + } + ), + _bedrock_event_frame( + { + "type": "message_stop", + "usage": {"input_tokens": 3, "output_tokens": 9}, + } + ), + ), + gate=gate, + ), + ) + + iterator: Final = AmazonAnthropicClaudeMessagesConfig().get_async_streaming_response_iterator( + model="us.anthropic.claude-sonnet-4-6", + httpx_response=response, + request_body={"model": "us.anthropic.claude-sonnet-4-6"}, + litellm_logging_obj=LiteLLMLoggingObj( + model="bedrock/us.anthropic.claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + stream=True, + call_type="chat", + start_time=datetime.now(), + litellm_call_id="test_small_frame_before_upstream_pauses", + function_id="test_small_frame_before_upstream_pauses", + ), + ) + + first: Final = await asyncio.wait_for(anext(iterator), timeout=10) + assert first.startswith(b"event: message_start\n"), first + + gate.set() + remaining: Final = tuple([chunk async for chunk in iterator]) + assert any(chunk.startswith(b"event: message_stop\n") for chunk in remaining), remaining + await iterator.aclose() diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8b0662de3f9..88ba7fc37d9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -2461,6 +2461,12 @@ def test_is_gemini_3_or_newer(): assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-pro") == False assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("4965075652664360960") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini/4965075652664360960") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini/ft-uuid") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("gemma-3-27b-it") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini/gemma-3-27b-it") == False + # Edge cases assert VertexGeminiConfig._is_gemini_3_or_newer("") == False @@ -2504,6 +2510,26 @@ def test_gemini_3_reasoning_effort_maps_to_thinking_level(model: str): assert "thinkingBudget" not in mapped["thinkingConfig"] +@pytest.mark.parametrize( + "model", + ["4965075652664360960", "gemini/4965075652664360960", "gemini/ft-uuid", "gemma-3-27b-it"], +) +def test_fine_tuned_endpoint_and_gemma_get_no_gemini_3_default_temperature(model: str): + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + mapped = VertexGeminiConfig().map_openai_params( + non_default_params={"max_tokens": 10}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert mapped["max_output_tokens"] == 10 + assert "temperature" not in mapped + + def _tool_call_messages(tool_call_id: str): diff --git a/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py b/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py index 4e2059ac30b..96770ee01c4 100644 --- a/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py @@ -1,17 +1,20 @@ import asyncio import hashlib import json -from typing import Iterable, List, Optional, Tuple +import time +from collections.abc import Iterable from unittest.mock import patch import pytest from redis.asyncio import Redis +import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( AUTH_CACHE_INVALIDATION_CHANNEL, AuthCacheInvalidationSubscriber, + evict_and_broadcast, publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -19,13 +22,29 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache class _RecordingRedisClient(Redis): def __init__(self) -> None: - self.published: List[Tuple[str, str]] = [] + self.published: list[tuple[str, str]] = [] async def publish(self, channel: str, message: str) -> int: self.published.append((channel, message)) return 1 +class _WedgedPublishRedisClient(Redis): + def __init__(self) -> None: + self.attempted: list[str] = [] + self.in_flight = 0 + self.max_in_flight = 0 + self.release = asyncio.Event() + + async def publish(self, channel: str, message: str) -> int: + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + self.attempted.append(message) + await self.release.wait() + self.in_flight -= 1 + return 1 + + class _FailingPublishRedisClient(Redis): def __init__(self) -> None: pass @@ -36,16 +55,16 @@ class _FailingPublishRedisClient(Redis): class _QueuePubSub: def __init__(self, initial_messages: Iterable[object] = ()) -> None: - self.queue: "asyncio.Queue[object]" = asyncio.Queue() + self.queue: asyncio.Queue[object] = asyncio.Queue() for message in initial_messages: self.queue.put_nowait(message) - self.subscribed_channels: List[str] = [] + self.subscribed_channels: list[str] = [] self.closed = False async def subscribe(self, *channels: str) -> None: self.subscribed_channels.extend(channels) - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> object | None: try: return await asyncio.wait_for(self.queue.get(), timeout) except asyncio.TimeoutError: @@ -64,7 +83,7 @@ class _ScriptedPubSubRedisClient(Redis): class _FakeRedisCache: - def __init__(self, client: object, namespace: Optional[str] = None) -> None: + def __init__(self, client: object, namespace: str | None = None) -> None: self._client = client self.namespace = namespace @@ -222,3 +241,49 @@ async def test_subscriber_ignores_malformed_messages() -> None: subscriber._apply_message(None) assert cache.in_memory_cache.get_cache("project_id:p-1") is not None + + +@pytest.mark.asyncio +async def test_evict_and_broadcast_evicts_locally_and_returns_while_redis_publish_never_answers() -> None: + cache = UserApiKeyCache() + cache.set_cache("user-wedged", UserAPIKeyAuth(user_id="user-wedged"), model_type=UserAPIKeyAuth) + client = _WedgedPublishRedisClient() + + with patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + return_value=_FakeRedisCache(client=client), + ): + started = time.monotonic() + await evict_and_broadcast(cache_keys=("user-wedged",), user_api_key_cache=cache) + elapsed = time.monotonic() - started + + assert elapsed < 0.1, f"handler waited {elapsed:.3f}s on a publish that never answers" + assert cache.get_cache("user-wedged", model_type=UserAPIKeyAuth) is None + assert client.attempted == [json.dumps({"cache_key": "user-wedged"})], "publish was not handed to redis" + client.release.set() + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_publish_holds_at_most_sixteen_redis_connections_while_redis_is_wedged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + client = _WedgedPublishRedisClient() + + with patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + return_value=_FakeRedisCache(client=client), + ): + for i in range(64): + await publish_auth_cache_invalidation(cache_key=f"user-{i}") + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert client.max_in_flight == 16, f"publish tasks held {client.max_in_flight} redis connections at once" + assert len(client.attempted) == 16, "waiters called publish before a semaphore slot freed" + client.release.set() + await asyncio.gather(*pubsub_module._pending_publishes) # pyright: ignore[reportPrivateUsage] # drain module-level tasks + + assert len(client.attempted) == 64 diff --git a/tests/test_litellm/proxy/policy_engine/test_policy_matcher.py b/tests/test_litellm/proxy/policy_engine/test_policy_matcher.py index b07137893ec..27153e67ab5 100644 --- a/tests/test_litellm/proxy/policy_engine/test_policy_matcher.py +++ b/tests/test_litellm/proxy/policy_engine/test_policy_matcher.py @@ -6,14 +6,23 @@ Tests: - Scope matching via attachments (teams, keys, models) """ +import logging +from typing import Final + import pytest +from hypothesis import given, settings +from hypothesis import strategies as st import litellm.proxy.policy_engine.attachment_registry as attachment_registry_module import litellm.proxy.policy_engine.policy_registry as policy_registry_module from litellm.proxy.policy_engine.attachment_registry import AttachmentRegistry from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher from litellm.proxy.policy_engine.policy_registry import PolicyRegistry +from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.types.proxy.policy_engine import ( + Policy, + PolicyCondition, + PolicyGuardrails, PolicyMatchContext, PolicyScope, ) @@ -221,6 +230,34 @@ def _global_registries(monkeypatch): return policies +def _inherited_registries(monkeypatch, parent_condition=None): + policies = PolicyRegistry() + policies.load_policies( + { + "parent": { + "guardrails": {"add": ["y"]}, + **({"condition": parent_condition} if parent_condition else {}), + }, + "child": { + "inherit": "parent", + "guardrails": {"add": ["x"]}, + "condition": {"model": "claude.*"}, + }, + "fallback": {"guardrails": {"add": ["z"]}}, + } + ) + attachments = AttachmentRegistry() + attachments.load_attachments( + [ + {"policy": "child", "scope": "*"}, + {"policy": "fallback", "scope": "*", "default": True}, + ] + ) + monkeypatch.setattr(policy_registry_module, "get_policy_registry", lambda: policies) + monkeypatch.setattr(attachment_registry_module, "get_attachment_registry", lambda: attachments) + return policies + + class TestGetMatchingPoliciesFallback: def test_condition_failing_opt_in_falls_back_to_default(self, monkeypatch): _global_registries(monkeypatch) @@ -244,3 +281,154 @@ class TestGetMatchingPoliciesFallback: PolicyMatcher.get_matching_policies(context=context) assert len(calls) == 1 + + def test_condition_missing_child_with_unconditional_parent_still_matches(self, monkeypatch): + _inherited_registries(monkeypatch) + context = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-5.5") + + assert PolicyMatcher.get_matching_policies(context=context) == ["child"] + + def test_child_whose_whole_chain_misses_falls_back_to_default(self, monkeypatch): + _inherited_registries(monkeypatch, parent_condition={"model": "claude.*"}) + context = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-5.5") + + assert PolicyMatcher.get_matching_policies(context=context) == ["fallback"] + + def test_get_policies_with_matching_conditions_keeps_missing_policy_out(self): + policies = { + "real": Policy( + guardrails=PolicyGuardrails(add=["g"]), + condition=PolicyCondition(model="claude.*"), + ), + } + context = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-5.5") + + assert ( + PolicyMatcher.get_policies_with_matching_conditions( + policy_names=["nope"], context=context, policies=policies + ) + == [] + ) + + +_MODELS: Final = ("gpt-4o", "gpt-5.5", "claude-opus-4-1") + + +def _policy_forest(draw: st.DrawFn) -> dict[str, Policy]: # mutable-ok: PolicyResolver takes dict[str, Policy] + names: Final = tuple(f"p{i}" for i in range(draw(st.integers(min_value=1, max_value=6)))) + return { # mutable-ok: PolicyResolver takes dict[str, Policy] + name: Policy( + inherit=draw(st.sampled_from((None, *names[:i]))), + guardrails=PolicyGuardrails(add=[f"g-{name}"]), # mutable-ok: pydantic list field + condition=draw(st.sampled_from((None, *(PolicyCondition(model=m) for m in _MODELS)))), + ) + for i, name in enumerate(names) + } + + +@st.composite +def _forest_and_request( + draw: st.DrawFn, +) -> tuple[dict[str, Policy], tuple[str, ...], PolicyMatchContext]: # mutable-ok: PolicyResolver takes dict + policies: Final = _policy_forest(draw) + attached: Final = tuple(draw(st.lists(st.sampled_from(sorted(policies)), unique=True))) + context: Final = PolicyMatchContext(team_alias="t", key_alias="k", model=draw(st.sampled_from(_MODELS))) + return policies, attached, context + + +def _own_condition_applies(policy: Policy, context: PolicyMatchContext) -> bool: + return policy.condition is None or policy.condition.model == context.model + + +def _applicable_chain( + policies: dict[str, Policy], # mutable-ok: PolicyResolver takes dict[str, Policy] + name: str, + context: PolicyMatchContext, +) -> tuple[str, ...]: + chain: Final = PolicyResolver.resolve_inheritance_chain(policy_name=name, policies=policies) + return tuple(member for member in chain if _own_condition_applies(policies[member], context)) + + +class TestChainMatchingProperties: + @given(_forest_and_request()) + @settings(max_examples=400, deadline=None) + def test_chain_matching_only_widens_to_applicable_ancestor_guardrails( + self, + case: tuple[dict[str, Policy], tuple[str, ...], PolicyMatchContext], # mutable-ok: PolicyResolver takes dict + ): + policies, attached, context = case + head: Final = tuple( + PolicyMatcher.get_policies_with_matching_conditions( + policy_names=attached, context=context, policies=policies + ) + ) + base: Final = tuple(name for name in attached if _own_condition_applies(policies[name], context)) + expected_head: Final = tuple(name for name in attached if _applicable_chain(policies, name, context)) + + assert head == expected_head, "a policy applies exactly when some chain member's own condition applies" + assert frozenset(base) <= frozenset(head), "head must never drop a policy base applied" + + for name in head: + resolved = PolicyResolver.resolve_policy_guardrails(policy_name=name, policies=policies, context=context) + assert sorted(resolved.guardrails) == sorted( + f"g-{member}" for member in _applicable_chain(policies, name, context) + ) + if name not in base: + assert f"g-{name}" not in resolved.guardrails, "a condition-missed child must not add its own guardrail" + + +class TestAncestorAdmissionLogging: + @staticmethod + def _chain() -> dict[str, Policy]: # mutable-ok: PolicyResolver takes dict[str, Policy] + return { # mutable-ok: PolicyResolver takes dict[str, Policy] + "parent": Policy(guardrails=PolicyGuardrails(add=["g-parent"])), # mutable-ok: pydantic list field + "child": Policy( + inherit="parent", + guardrails=PolicyGuardrails(add=["g-child"]), # mutable-ok: pydantic list field + condition=PolicyCondition(model="gpt-5.5"), + ), + } + + def test_logs_when_admitted_through_ancestor_only(self, caplog): + context: Final = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-4o") + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + result: Final = PolicyMatcher.policy_applies(context, self._chain())("child") + records: Final = [r for r in caplog.records if "applied through ancestor" in r.getMessage()] + assert result is True + assert len(records) == 1 + assert "applied through ancestor 'parent'" in records[0].getMessage() + assert "'child'" in records[0].getMessage() + + def test_no_log_when_own_condition_matches(self, caplog): + context: Final = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-5.5") + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + result: Final = PolicyMatcher.policy_applies(context, self._chain())("child") + assert result is True + assert not [r for r in caplog.records if "applied through ancestor" in r.getMessage()] + + def test_no_log_when_no_chain_member_applies(self, caplog): + policies: Final = { # mutable-ok: PolicyResolver takes dict[str, Policy] + "parent": Policy( + guardrails=PolicyGuardrails(add=["g-parent"]), # mutable-ok: pydantic list field + condition=PolicyCondition(model="claude-opus-4-1"), + ), + "child": Policy( + inherit="parent", + guardrails=PolicyGuardrails(add=["g-child"]), # mutable-ok: pydantic list field + condition=PolicyCondition(model="gpt-5.5"), + ), + } + context: Final = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-4o") + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + result: Final = PolicyMatcher.policy_applies(context, policies)("child") + assert result is False + assert not [r for r in caplog.records if "applied through ancestor" in r.getMessage()] + + def test_condition_filter_logs_nothing(self, caplog): + context: Final = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-4o") + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + result: Final = PolicyMatcher.get_policies_with_matching_conditions( + policy_names=["child"], context=context, policies=self._chain() + ) + assert result == ["child"] + assert not [r for r in caplog.records if "applied through ancestor" in r.getMessage()] diff --git a/tests/test_litellm/proxy/policy_engine/test_policy_resolver.py b/tests/test_litellm/proxy/policy_engine/test_policy_resolver.py index b9ce22d749e..3d2f547a744 100644 --- a/tests/test_litellm/proxy/policy_engine/test_policy_resolver.py +++ b/tests/test_litellm/proxy/policy_engine/test_policy_resolver.py @@ -11,6 +11,8 @@ import pytest from litellm.proxy.policy_engine.policy_resolver import PolicyResolver from litellm.types.proxy.policy_engine import ( + GuardrailPipeline, + PipelineStep, Policy, PolicyCondition, PolicyGuardrails, @@ -199,3 +201,55 @@ class TestPolicyResolverWithConditions: ) assert "pii_blocker" in resolved_gpt35.guardrails assert "child_guardrail" not in resolved_gpt35.guardrails + + def test_resolve_guardrails_for_context_with_condition_missing_child_keeps_inherited_parent(self): + """Test a matched child whose condition misses still contributes unconditional parent guardrails.""" + policies = { + "parent": Policy( + guardrails=PolicyGuardrails(add=["y"]), + ), + "child": Policy( + inherit="parent", + guardrails=PolicyGuardrails(add=["x"]), + condition=PolicyCondition(model="claude.*"), + ), + } + + context_miss = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-5.5") + assert PolicyResolver.resolve_guardrails_for_context( + context=context_miss, policies=policies, policy_names=["child"] + ) == ["y"] + + context_hit = PolicyMatchContext(team_alias="t", key_alias="k", model="claude-haiku") + assert set( + PolicyResolver.resolve_guardrails_for_context( + context=context_hit, policies=policies, policy_names=["child"] + ) + ) == {"x", "y"} + + def test_resolve_pipelines_for_context_skips_pipeline_when_own_condition_misses(self): + """Test a matched child whose own condition misses does not run its pipeline.""" + pipeline = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="child-guard")]) + policies = { + "parent": Policy( + guardrails=PolicyGuardrails(add=["y"]), + ), + "child": Policy( + inherit="parent", + pipeline=pipeline, + condition=PolicyCondition(model="gpt-5.5"), + ), + } + + context_miss = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-4o") + assert ( + PolicyResolver.resolve_pipelines_for_context( + context=context_miss, policies=policies, policy_names=["child"] + ) + == [] + ) + + context_hit = PolicyMatchContext(team_alias="t", key_alias="k", model="gpt-5.5") + assert PolicyResolver.resolve_pipelines_for_context( + context=context_hit, policies=policies, policy_names=["child"] + ) == [("child", pipeline)] diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 8fb53d4b0a0..0d4b9e8d21f 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -4377,6 +4377,45 @@ def test_match_and_track_policies_preserves_attachment_and_request_body_order(): assert applied_policy_names == policy_names +def test_match_and_track_policies_keeps_condition_missing_child_alongside_unconditional_sibling(): + from litellm.proxy.policy_engine.attachment_registry import AttachmentRegistry + from litellm.types.proxy.policy_engine import ( + Policy, + PolicyCondition, + PolicyGuardrails, + PolicyMatchContext, + ) + + policies = { + "baseline": Policy(guardrails=PolicyGuardrails(add=["baseline_guardrail"])), + "parent": Policy(guardrails=PolicyGuardrails(add=["pii_blocker"])), + "child": Policy( + inherit="parent", + guardrails=PolicyGuardrails(add=["child_guard"]), + condition=PolicyCondition(model="claude.*"), + ), + } + attachment_registry = AttachmentRegistry() + attachment_registry.load_attachments( + [ + {"policy": "baseline", "scope": "*"}, + {"policy": "child", "scope": "*"}, + ] + ) + data = {"metadata": {}} + + applied_policy_names, _ = _match_and_track_policies( + data=data, + context=PolicyMatchContext(model="gpt-5.5"), + request_body_policies=[], + policies_override=policies, + attachment_registry_override=attachment_registry, + ) + + assert applied_policy_names == ["baseline", "child"] + assert data["metadata"]["applied_policies"] == ["baseline", "child"] + + @pytest.mark.asyncio async def test_add_guardrails_from_policy_engine_keeps_a_policy_added_guardrail_its_pipeline_also_steps(): from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry @@ -4419,6 +4458,48 @@ async def test_add_guardrails_from_policy_engine_keeps_a_policy_added_guardrail_ assert [pipeline.mode for _policy_name, pipeline in data["metadata"]["_guardrail_pipelines"]] == ["post_call"] +@pytest.mark.asyncio +async def test_add_guardrails_from_policy_engine_applies_inherited_parent_guardrail_when_child_condition_misses(): + from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + from litellm.types.proxy.policy_engine import ( + Policy, + PolicyAttachment, + PolicyCondition, + PolicyGuardrails, + ) + + data = {"model": "gpt-5.5", "messages": [{"role": "user", "content": "Hello"}], "metadata": {}} + policy_registry = get_policy_registry() + policy_registry._policies = { + "parent": Policy(guardrails=PolicyGuardrails(add=["pii_blocker"])), + "child": Policy( + inherit="parent", + guardrails=PolicyGuardrails(add=["child_guard"]), + condition=PolicyCondition(model="claude.*"), + ), + } + policy_registry._initialized = True + attachment_registry = get_attachment_registry() + attachment_registry._attachments = [PolicyAttachment(policy="child", scope="*")] + attachment_registry._initialized = True + + try: + await add_guardrails_from_policy_engine( + data=data, + metadata_variable_name="metadata", + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + ) + finally: + policy_registry._policies = {} + policy_registry._initialized = False + attachment_registry._attachments = [] + attachment_registry._initialized = False + + assert "pii_blocker" in data["metadata"]["guardrails"] + assert "child_guard" not in data["metadata"]["guardrails"] + + @pytest.mark.asyncio async def test_add_guardrails_from_policy_engine_accepts_dynamic_policies_and_pops_from_data(): """ diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py index d7c52c528eb..0046721fd9c 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -149,6 +149,49 @@ async def test_post_call_failure_hook_attributes_single_router_deployment( ) +@pytest.mark.asyncio +async def test_pre_routing_reject_spend_log_keeps_public_model_group(proxy_logging, make_user_api_key_auth, monkeypatch): + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=401, detail="blocked key"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + assert recorded[0]["standard_logging_object"]["model_group"] == "internal-model" + now: Final = datetime.now() + payload = get_logging_payload( + kwargs={**recorded[0], "completion_start_time": now}, response_obj=None, start_time=now, end_time=now + ) + assert payload["model"] == "openai/gpt-4.1" + assert payload["model_group"] == "internal-model" + + @pytest.mark.asyncio async def test_post_call_failure_hook_keeps_router_stamped_metadata_for_post_call_failures( proxy_logging, make_user_api_key_auth, monkeypatch diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index 20484e787bd..2f7d4b350be 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -2189,6 +2189,7 @@ async def test_new_vector_store_persists_embedding_reference_without_credentials mock_registry = MagicMock() mock_registry.add_vector_store_to_registry = MagicMock() + mock_registry.is_config_vector_store.return_value = False with ( patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), @@ -2267,6 +2268,7 @@ async def test_new_vector_store_auto_resolves_from_router(): mock_registry = MagicMock() mock_registry.add_vector_store_to_registry = MagicMock() + mock_registry.is_config_vector_store.return_value = False with ( patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), @@ -3061,3 +3063,196 @@ def test_vector_store_search_rejects_caller_embedding_selection_params(blocked_k assert response.status_code == 400, response.json() assert blocked_key in str(response.json()) + + +class TestConfigOwnedVectorStores: + """Stores declared under ``vector_store_registry`` in config.yaml are owned by the config file""" + + CONFIG_ID = "vs_from_config" + DB_ID = "vs_from_db" + + def _registry(self): + from litellm.vector_stores.vector_store_registry import VectorStoreRegistry + + registry = VectorStoreRegistry(vector_stores=[]) + registry.load_vector_stores_from_config( + [ + { + "vector_store_name": "config-store", + "litellm_params": {"vector_store_id": self.CONFIG_ID, "custom_llm_provider": "openai"}, + } + ] + ) + registry.add_vector_store_to_registry(self._db_row(self.DB_ID, "db-store")) + registry.add_vector_store_to_registry(self._db_row("vs_stale", "deleted-elsewhere")) + return registry + + @staticmethod + def _db_row(vector_store_id: str, vector_store_name: str) -> dict: + return { + "vector_store_id": vector_store_id, + "custom_llm_provider": "openai", + "vector_store_name": vector_store_name, + "litellm_params": {}, + "created_at": datetime.now(timezone.utc), + "updated_at": datetime.now(timezone.utc), + } + + @staticmethod + def _admin() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + + @pytest.mark.asyncio + async def test_list_keeps_config_store_that_has_no_db_row(self): + from litellm.proxy.vector_store_endpoints.management_endpoints import list_vector_stores + + registry = self._registry() + prisma = MagicMock() + prisma.db.litellm_managedvectorstorestable.find_many = AsyncMock(return_value=[self._db_row(self.DB_ID, "db-store")]) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", registry), + ): + first = await list_vector_stores(user_api_key_dict=self._admin()) + second = await list_vector_stores(user_api_key_dict=self._admin()) + + assert [(vs["vector_store_id"], vs["is_config"]) for vs in first["data"]] == [(self.DB_ID, False), (self.CONFIG_ID, True)] + assert second["data"] == first["data"] + assert [vs["vector_store_id"] for vs in registry.vector_stores] == [self.CONFIG_ID, self.DB_ID] + + @pytest.mark.asyncio + async def test_list_keeps_config_store_and_db_row_with_same_id_does_not_overwrite_it(self): + from litellm.proxy.vector_store_endpoints.management_endpoints import list_vector_stores + + registry = self._registry() + prisma = MagicMock() + prisma.db.litellm_managedvectorstorestable.find_many = AsyncMock( + return_value=[self._db_row(self.DB_ID, "db-store"), self._db_row(self.CONFIG_ID, "renamed-in-db")] + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", registry), + ): + response = await list_vector_stores(user_api_key_dict=self._admin()) + + by_id = {vs["vector_store_id"]: vs for vs in response["data"]} + assert set(by_id) == {self.CONFIG_ID, self.DB_ID}, response + assert (by_id[self.CONFIG_ID]["vector_store_name"], by_id[self.CONFIG_ID]["is_config"]) == ("config-store", True) + assert (by_id[self.DB_ID]["vector_store_name"], by_id[self.DB_ID]["is_config"]) == ("db-store", False) + assert [vs["vector_store_id"] for vs in registry.vector_stores] == [self.CONFIG_ID, self.DB_ID] + assert registry.get_litellm_managed_vector_store_from_registry(self.CONFIG_ID)["vector_store_name"] == "config-store" + + @pytest.mark.asyncio + async def test_info_reports_config_ownership(self): + from litellm.proxy.vector_store_endpoints.management_endpoints import get_vector_store_info + from litellm.types.vector_stores import VectorStoreInfoRequest + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", self._registry()), + ): + config_info = await get_vector_store_info( + data=VectorStoreInfoRequest(vector_store_id=self.CONFIG_ID), user_api_key_dict=self._admin() + ) + db_info = await get_vector_store_info( + data=VectorStoreInfoRequest(vector_store_id=self.DB_ID), user_api_key_dict=self._admin() + ) + + assert config_info["vector_store"].is_config is True + assert db_info["vector_store"].is_config is False + + @pytest.mark.asyncio + async def test_new_with_config_store_id_is_rejected_before_db_write(self): + prisma = MagicMock() + prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_managedvectorstorestable.create = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", self._registry()), + pytest.raises(HTTPException) as exc_info, + ): + await new_vector_store( + vector_store={"vector_store_id": self.CONFIG_ID, "custom_llm_provider": "openai"}, + user_api_key_dict=self._admin(), + ) + + assert exc_info.value.status_code == 400, exc_info.value.detail + assert exc_info.value.detail["vector_store_id"] == self.CONFIG_ID + assert "config file" in exc_info.value.detail["error"] + prisma.db.litellm_managedvectorstorestable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_update_of_config_store_is_rejected_before_db_write(self): + from litellm.proxy.vector_store_endpoints.management_endpoints import update_vector_store + from litellm.types.vector_stores import VectorStoreUpdateRequest + + prisma = MagicMock() + prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_managedvectorstorestable.update = AsyncMock() + registry = self._registry() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", registry), + pytest.raises(HTTPException) as exc_info, + ): + await update_vector_store( + data=VectorStoreUpdateRequest(vector_store_id=self.CONFIG_ID, vector_store_name="renamed"), + user_api_key_dict=self._admin(), + ) + + assert exc_info.value.status_code == 400, exc_info.value.detail + assert exc_info.value.detail["vector_store_id"] == self.CONFIG_ID + prisma.db.litellm_managedvectorstorestable.update.assert_not_called() + assert registry.get_litellm_managed_vector_store_from_registry(self.CONFIG_ID)["vector_store_name"] == "config-store" + + @pytest.mark.asyncio + async def test_delete_of_config_store_is_rejected_and_store_stays_registered(self): + from litellm.proxy.vector_store_endpoints.management_endpoints import delete_vector_store + from litellm.types.vector_stores import VectorStoreDeleteRequest + + prisma = MagicMock() + prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_managedvectorstorestable.delete = AsyncMock() + registry = self._registry() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", registry), + pytest.raises(HTTPException) as exc_info, + ): + await delete_vector_store( + data=VectorStoreDeleteRequest(vector_store_id=self.CONFIG_ID), user_api_key_dict=self._admin() + ) + + assert exc_info.value.status_code == 400, exc_info.value.detail + assert exc_info.value.detail["vector_store_id"] == self.CONFIG_ID + prisma.db.litellm_managedvectorstorestable.delete.assert_not_called() + assert registry.is_config_vector_store(self.CONFIG_ID) is True + + @pytest.mark.asyncio + async def test_delete_of_db_store_still_works(self): + from litellm.proxy.vector_store_endpoints.management_endpoints import delete_vector_store + from litellm.types.vector_stores import VectorStoreDeleteRequest + + row = MagicMock() + row.model_dump = MagicMock(return_value=self._db_row(self.DB_ID, "db-store")) + prisma = MagicMock() + prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=row) + prisma.db.litellm_managedvectorstorestable.delete = AsyncMock() + registry = self._registry() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: proxy_server global, no seam + patch.object(litellm, "vector_store_registry", registry), + ): + response = await delete_vector_store( + data=VectorStoreDeleteRequest(vector_store_id=self.DB_ID), user_api_key_dict=self._admin() + ) + + assert response["status"] == "success", response + prisma.db.litellm_managedvectorstorestable.delete.assert_awaited_once_with(where={"vector_store_id": self.DB_ID}) + assert registry.get_litellm_managed_vector_store_from_registry(self.DB_ID) is None diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index b6b486e73f0..45c35dc6802 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -11,6 +11,7 @@ from litellm.rust_bridge.catalog import ( CacheRule, Context, Delivery, + LoggerContext, Route, RouteContext, RouteRule, @@ -89,6 +90,13 @@ def test_backend_rollouts_stay_on_python_when_global_rust_is_enabled( assert catalog.decision(context) is Decision.PYTHON +def test_logger_rollout_obeys_the_global_switch() -> None: + assert catalog.rollout(LoggerContext()) is Rollout.RUST_OPT_IN + assert catalog.decision(LoggerContext()) is Decision.PYTHON + configuration.rust(True) + assert catalog.decision(LoggerContext()) is Decision.RUST_WITH_FALLBACK + + def test_response_cache_rules_select_the_whole_backend_runtime() -> None: rules: Final = ( CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({"local"})), diff --git a/tests/test_litellm/rust_bridge/test_logger.py b/tests/test_litellm/rust_bridge/test_logger.py new file mode 100644 index 00000000000..aff385b9344 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_logger.py @@ -0,0 +1,137 @@ +import logging +from typing import Final + +import pytest + +import litellm +from litellm._logging import ( + DiagnosticProcessingFilter, + _python_process_diagnostic, + redact_secrets, + session_id_var, + trace_id_var, + verbose_logger, +) +from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH +from litellm.litellm_core_utils.secret_redaction import ( + _python_redact_internal_details, + _python_redact_string, + _python_redact_structured_value, +) +from litellm.rust_bridge import diagnostics, logger + + +def test_native_records_preserve_metadata_and_redact_before_custom_handlers( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(verbose_logger, "handlers", []) + secret: Final = "sk-" + "a" * 48 + message: Final = f"Authorization: Bearer {secret}" + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logger.emit( + logging.WARNING, message, "native.rs", 42, "litellm_http", {"retry": True, "api_key": secret}, ("", "") + ) + + record: Final = caplog.records[0] + assert len(caplog.records) == 1 + assert record.getMessage() == redact_secrets(message) + assert secret not in record.getMessage() + assert (record.pathname, record.lineno, record.funcName) == ("native.rs", 42, "litellm_http") + assert record.__dict__["rust_fields"]["retry"] is True + assert secret not in str(record.__dict__["rust_fields"]) + + +def test_native_context_is_scoped_and_respects_correlation_setting( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + context_before: Final = logger.context() + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logger.emit(logging.WARNING, "native", "native.rs", 1, "litellm_http", {}, ("session", "trace")) + monkeypatch.setattr(litellm, "request_correlation_in_logs", False) + logger.emit(logging.WARNING, "disabled", "native.rs", 2, "litellm_http", {}, ("hidden", "hidden")) + + first, second = caplog.records + assert (first.__dict__["session_id"], first.__dict__["trace_id"]) == ("session", "trace") + assert "session_id" not in second.__dict__ + assert "trace_id" not in second.__dict__ + assert (session_id_var.get(), trace_id_var.get()) == context_before + + +def test_native_logging_observes_level_changes(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + assert not logger.enabled(logging.WARNING) + logger.emit(logging.WARNING, "filtered", "native.rs", 1, "litellm_http", {}, ("", "")) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logger.enabled(logging.WARNING) + logger.emit(logging.WARNING, "visible", "native.rs", 1, "litellm_http", {}, ("", "")) + + assert [record.getMessage() for record in caplog.records] == ["visible"] + + +@pytest.mark.parametrize( + "text", + ( + "Authorization: Bearer abcdefghijklmnop", + "s3_secret_access_key=secret123", + "postgres://user:pass@database.internal/name", + '{"type":"service_account","private_key":"secret123"}', + "GET /v1?key=abcdefghij&page=2", + ), +) +def test_native_credential_patterns_match_python(text: str) -> None: + pytest.importorskip("litellm.rust_bridge._native") + from litellm.rust_bridge._native import NativeDiagnosticProcessor + + processor: Final = NativeDiagnosticProcessor(MINIMUM_CUSTOM_KEY_LENGTH) + assert processor.redact_text(text) == _python_redact_string(text) + assert processor.redact_structured_text("api_key", "secret123") == _python_redact_structured_value( + "api_key", "secret123" + ) + + +def test_native_client_redaction_matches_python() -> None: + pytest.importorskip("litellm.rust_bridge._native") + from litellm.rust_bridge._native import NativeDiagnosticProcessor + + text: Final = "error at /etc/secrets/config on db.internal\nTraceback (most recent call last):\nsecret" + processor: Final = NativeDiagnosticProcessor(MINIMUM_CUSTOM_KEY_LENGTH) + assert processor.redact_client_message(text) == _python_redact_internal_details(text) + + +def test_native_diagnostic_batch_matches_python() -> None: + pytest.importorskip("litellm.rust_bridge._native") + from litellm.rust_bridge._native import NativeDiagnosticProcessor + + message: Final = "é" * 110 + "sk-" + "q" * 48 + "界" * 1000 + exception: Final = "document=" + "Q" * 200 + stack: Final = "api_key=secret123" + leaves: Final = (("api_key", "secret123"), (None, "safe")) + processor: Final = NativeDiagnosticProcessor(MINIMUM_CUSTOM_KEY_LENGTH) + rust: Final = processor.process_diagnostic(message, exception, stack, leaves, (True, 20, 500)) + python: Final = _python_process_diagnostic(message, exception, stack, leaves, True, 20, 500) + + assert rust[:3] == python[:3] + assert tuple(rust[3]) == python[3] + assert rust[4] == python[4] + assert "sk-qq" not in rust[0] + assert len(rust[0]) <= 500 + assert rust[3] == ["REDACTED", "safe"] + + +def test_missing_native_diagnostic_processor_falls_back_before_record_mutation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_RUST", "1") + diagnostics.PROCESSOR.override(None) + try: + record: Final = logging.makeLogRecord({"name": "LiteLLM", "levelno": logging.INFO, "msg": "api_key=secret123"}) + assert DiagnosticProcessingFilter().filter(record) is True + assert record.getMessage() == "REDACTED" + finally: + diagnostics.PROCESSOR.reset() + + +def test_unsupported_unicode_uses_safe_python_redaction(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "1") + assert redact_secrets("broken\ud800 api_key=secret123") == "broken\ud800 REDACTED" diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index 3a86a69ed8b..216961ab640 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -1,4 +1,3 @@ -import logging from typing import Final import httpx @@ -11,6 +10,7 @@ from litellm.rust_bridge import settings from litellm.secret_managers.main import get_secret_str from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem + def test_url_policy_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "user_url_validation", False) monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["docs.internal:8443"]) @@ -57,13 +57,6 @@ def test_http_settings_ignores_environment_overrides(monkeypatch: pytest.MonkeyP assert result.ssl_verify is True -def test_warn_reaches_the_litellm_logger(caplog: pytest.LogCaptureFixture) -> None: - with caplog.at_level(logging.WARNING, logger="LiteLLM"): - settings.warn("ssl_ecdh_curve 'secp521r1' is not supported") - - assert [record.getMessage() for record in caplog.records] == ["ssl_ecdh_curve 'secp521r1' is not supported"] - - class _VaultSecrets(CustomSecretManager): def __init__(self, secrets: dict[str, str]) -> None: super().__init__(secret_manager_name="rust_bridge_settings_test") diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index c6b57fa7604..ddcdeb61e02 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -38,6 +38,7 @@ from litellm.types.utils import ( Usage, ) from litellm.types.videos.main import VideoObject +from litellm.utils import supports_prompt_caching @pytest.fixture @@ -872,6 +873,40 @@ def test_default_image_cost_calculator(monkeypatch): assert cost == 10485760 +@pytest.mark.parametrize( + ("model", "quality", "size", "priced_key", "pixels"), + [ + ("azure/dall-e-3", "standard", "1024x1024", "azure/standard/1024-x-1024/dall-e-3", 1024 * 1024), + ("azure/dall-e-3", "hd", "1024x1792", "azure/hd/1024-x-1792/dall-e-3", 1024 * 1792), + ("dall-e-3", "hd", "1024x1792", "azure/hd/1024-x-1792/dall-e-3", 1024 * 1792), + ], +) +def test_default_image_cost_calculator_matches_provider_first_quality_key( + monkeypatch, model: str, quality: str, size: str, priced_key: str, pixels: int +): + from litellm.cost_calculator import default_image_cost_calculator + + monkeypatch.setattr( + litellm, + "model_cost", + { + "azure/standard/1024-x-1024/dall-e-3": {"litellm_provider": "azure", "input_cost_per_pixel": 1e-08}, + "azure/hd/1024-x-1792/dall-e-3": {"litellm_provider": "azure", "input_cost_per_pixel": 3e-08}, + }, + ) + + cost = default_image_cost_calculator( + model=model, + custom_llm_provider="azure", + quality=quality, + n=1, + size=size, + optional_params={}, + ) + + assert cost == litellm.model_cost[priced_key]["input_cost_per_pixel"] * pixels + + def test_cost_calculator_with_cache_creation(): from litellm import completion_cost from litellm.types.utils import Choices, Message, Usage @@ -4502,3 +4537,99 @@ def test_cost_per_token_bedrock_nemotron_super_3_uses_eu_west_2_entry_not_us_rat assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"]) assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"]) + + +GPT_REALTIME_2_FAMILY: Final = ( + "azure/gpt-realtime-2.1", + "azure/gpt-realtime-2.1-mini", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", +) + + +def test_gpt_realtime_2_family_prices_audio_cache_writes_and_reads_alike(_local_model_cost_map: None) -> None: + audio_cache_rates: Final = { + model: ( + litellm.model_cost[model].get("cache_read_input_audio_token_cost"), + litellm.model_cost[model].get("cache_creation_input_audio_token_cost"), + ) + for model in GPT_REALTIME_2_FAMILY + } + + # Azure publishes one cached-audio meter per gpt-realtime-2 deployment, + # https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/, checked 2026-09-23 + assert all(read is not None and write == read for read, write in audio_cache_rates.values()), audio_cache_rates + assert len(audio_cache_rates) == len(GPT_REALTIME_2_FAMILY) + + +GEMINI_LIVE_NATIVE_AUDIO_CASES: Final = ( + ("gemini-live-2.5-flash-native-audio", "vertex_ai"), + ("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"), + ("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"), +) + + +@pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES) +def test_gemini_live_native_audio_carries_no_cached_input_rate( + _local_model_cost_map: None, model: str, provider: str +) -> None: + # the Vertex pricing table prints N/A for cached input on every Live row, + # https://cloud.google.com/vertex-ai/generative-ai/pricing, checked 2026-09-23 + assert litellm.get_model_info(model, custom_llm_provider=provider)["cache_read_input_token_cost"] is None + + prompt_usd, _ = cost_per_token( + model=model, + prompt_tokens=101_000, + completion_tokens=0, + custom_llm_provider=provider, + usage_object=Usage( + prompt_tokens=101_000, + completion_tokens=0, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000), + ), + ) + fresh_usd, _ = cost_per_token( + model=model, + prompt_tokens=101_000, + completion_tokens=0, + custom_llm_provider=provider, + usage_object=Usage(prompt_tokens=101_000, completion_tokens=0), + ) + + assert prompt_usd == pytest.approx(fresh_usd), ( + "with no cached rate the cached tokens bill at the input rate, so a phantom discount cannot appear" + ) + assert prompt_usd > 0 + + +@pytest.mark.parametrize(("model", "provider"), GEMINI_LIVE_NATIVE_AUDIO_CASES) +def test_gemini_live_native_audio_declares_prompt_caching_unsupported( + _local_model_cost_map: None, model: str, provider: str +) -> None: + # the Vertex context-caching supported-model lists contain no Live model while 2.5 Flash is listed, + # https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-overview, checked 2026-09-23 + assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_prompt_caching"] is False + assert supports_prompt_caching(model=model, custom_llm_provider=provider) is False + assert supports_prompt_caching(model="gemini-2.5-flash", custom_llm_provider="vertex_ai") is True, ( + "control: the helper swallows a lookup error into False, so without this a broken lookup reads as a pass" + ) + + +@pytest.mark.parametrize( + "model", + ["gemini-live-2.5-flash-native-audio", "vertex_ai/gemini-live-2.5-flash-native-audio"], +) +def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_card( + _local_model_cost_map: None, model: str +) -> None: + info = litellm.get_model_info(model) + + # the Vertex model card for gemini-live-2.5-flash-native-audio publishes these limits and flags, + # https://cloud.google.com/vertex-ai/generative-ai/docs/models, checked 2026-09-23 + assert info["max_input_tokens"] == 131072 + assert info["max_output_tokens"] == 65536 + assert info["max_tokens"] == 65536 + assert info["supports_response_schema"] is False + assert info["supports_url_context"] is False + assert info["supports_pdf_input"] is False diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 7cecdaec25d..d9cfe88d52f 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -19,6 +19,16 @@ from litellm._logging import ( _COLOR_LOG_FORMAT, _MAX_SCRUBBED_ACCESS_ARG, _PLAIN_LOG_FORMAT, + ALL_LOGGERS, + AccessLogPathFilter, + AccessLogRedactionFilter, + CorrelationContextFilter, + CorrelationPlainFormatter, + DiagnosticProcessingFilter, + JsonFormatter, + LevelRoutingStreamHandler, + SecretRedactionFilter, + StdoutLogTruncationFilter, _get_uvicorn_json_log_config, _initialize_loggers_with_handler, _parse_json_logs_env, @@ -33,15 +43,6 @@ from litellm._logging import ( verbose_logger, verbose_proxy_logger, verbose_router_logger, - ALL_LOGGERS, - AccessLogPathFilter, - AccessLogRedactionFilter, - CorrelationContextFilter, - CorrelationPlainFormatter, - JsonFormatter, - LevelRoutingStreamHandler, - SecretRedactionFilter, - StdoutLogTruncationFilter, ) from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD from litellm.integrations.custom_logger import CustomLogger @@ -824,6 +825,75 @@ def test_secret_filter_keeps_truncated_traceback(monkeypatch): assert "sk-1234567890abcdefghij" not in record.exc_text +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_diagnostic_redaction_precedes_a_credential_cut(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") + monkeypatch.setenv("MAX_BASE64_LENGTH_STDOUT_LOG", "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + secret = "sk-" + "q" * 48 + record = _make_record(logging.INFO, "%s", ("é" * 110 + secret + "界" * 1000,)) + + assert DiagnosticProcessingFilter().filter(record) is True + + assert len(record.getMessage()) <= 500 + assert "sk-qq" not in record.getMessage() + + +def test_correlation_id_redacts_before_its_length_bound(monkeypatch): + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + secret = "sk-" + "q" * 48 + token = set_trace_id("x" * 250 + secret) + try: + assert "sk-qq" not in trace_id_var.get() + assert len(trace_id_var.get()) <= 256 + finally: + trace_id_var.reset(token) + + +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_malformed_interpolation_still_scrubs_a_record(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "bad % api_key=secret123", ("value",)) + record.color_message = "bad % api_key=secret123" + + assert DiagnosticProcessingFilter().filter(record) is True + + assert record.getMessage() == "REDACTED" + assert record.color_message == "REDACTED" + + +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_key_pattern_template_keeps_the_rendered_redacted_line(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.INFO, "password=%s ok", ("hunter2",)) + record.color_message = "password=%s ok" + + assert DiagnosticProcessingFilter().filter(record) is True + + assert record.getMessage() == "REDACTED ok" + assert record.color_message == "REDACTED ok" + + +def test_disabled_diagnostic_call_does_not_render_arguments(caplog): + class Unrenderable: + def __str__(self): + raise AssertionError("disabled call rendered its argument") + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + verbose_logger.debug("hidden %s", Unrenderable()) + + assert not caplog.records + + def test_truncation_filter_survives_json_reconfiguration(): """The cap lives on the loggers, so swapping handlers (JSON mode) can't drop it.""" _turn_on_json() @@ -983,10 +1053,10 @@ _REQUEST_DUMP = "{'model': 'gpt-4', 'messages': [{'role': 'user', 'content': 'he (CorrelationPlainFormatter(_PLAIN_LOG_FORMAT), JsonFormatter()), ids=("plain", "json"), ) -def test_scrubbed_record_is_scanned_for_secrets_once(monkeypatch, formatter): - """Every pass of the secret regex over a multi-megabyte debug line costs seconds of - event-loop time, so a formatter must not rescan what SecretRedactionFilter scrubbed.""" +def test_scrubbed_record_scans_the_large_rendered_value_once(monkeypatch, formatter): + """The raw format template gets its own check, while the large rendered value gets one scan.""" counting = _CountingPattern(secret_redaction._SECRET_RE) + monkeypatch.setenv("LITELLM_RUST", "0") monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting) monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,)) @@ -997,14 +1067,15 @@ def test_scrubbed_record_is_scanned_for_secrets_once(monkeypatch, formatter): assert _REQUEST_DUMP in rendered assert "litellm_redacted" not in rendered - assert counting.calls == 1 - assert counting.scanned_chars == len(f"receiving data: {_REQUEST_DUMP}") + assert counting.calls == 2 + assert counting.scanned_chars == len(f"receiving data: {_REQUEST_DUMP}") + len("receiving data: %s") def test_stamped_record_is_not_scanned_again(monkeypatch): """JSON mode puts the filter on a third-party logger and again on the root handler its records propagate to, so the second filter must trust the stamp instead of rescanning.""" counting = _CountingPattern(secret_redaction._SECRET_RE) + monkeypatch.setenv("LITELLM_RUST", "0") monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting) monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,)) @@ -1012,13 +1083,14 @@ def test_stamped_record_is_not_scanned_again(monkeypatch): assert SecretRedactionFilter().filter(record) is True assert SecretRedactionFilter().filter(record) is True - assert counting.calls == 1 + assert counting.calls == 2 def test_caller_supplied_stamp_never_skips_the_scrub(monkeypatch): """The stamp is a private sentinel, so a caller passing extra={"litellm_redacted": True} still gets the full scrub, and only the filter's own stamp lets a later pass skip it.""" counting = _CountingPattern(secret_redaction._SECRET_RE) + monkeypatch.setenv("LITELLM_RUST", "0") monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting) monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) record = _make_record(logging.DEBUG, "api_key=sk-1234567890abcdefghij") @@ -1619,3 +1691,75 @@ def test_access_log_path_filter_keeps_a_record_without_a_string_path_arg(monkeyp exc_info=None, ) assert AccessLogPathFilter().filter(record) is True + + +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_diagnostic_filter_scrubs_exc_stack_and_nested_extras(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + secret = "sk-" + "q" * 48 + try: + raise ValueError(f"upstream rejected {secret}") + except ValueError: + record = _make_record(logging.ERROR, "call failed", exc_info=sys.exc_info()) + record.stack_info = f"Stack (most recent call last): {secret}" + record.payload = { + "api_key": secret, + "items": [secret, "ok"], + "tags": {secret}, + "pair": (secret, "ok"), + "count": 2, + } + + assert DiagnosticProcessingFilter().filter(record) is True + + assert secret not in (record.exc_text or "") + assert secret not in (record.stack_info or "") + assert secret not in repr(record.payload) + assert record.payload["count"] == 2 + + +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_diagnostic_filter_stamps_records_so_a_second_pass_is_free(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "api_key=secret123") + diagnostic_filter = DiagnosticProcessingFilter() + + assert diagnostic_filter.filter(record) is True + assert diagnostic_filter.filter(record) is True + assert record.getMessage() == "REDACTED" + + +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_json_formatter_scrubs_unfiltered_extras(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + secret = "sk-" + "q" * 48 + record = _make_record(logging.INFO, "response complete") + record.payload = {"api_key": secret, "nested": {"list": [secret]}} + + rendered = JsonFormatter().format(record) + + assert secret not in rendered + assert "REDACTED" in rendered + + +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_diagnostic_filter_redacts_a_non_string_message_object(monkeypatch, native): + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + secret = "sk-" + "q" * 48 + record = _make_record(logging.ERROR, {"api_key": secret}) + + assert DiagnosticProcessingFilter().filter(record) is True + + assert secret not in record.getMessage() diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index f58ade11d1c..71369c33a6b 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -19,7 +19,11 @@ from litellm._logging import ( verbose_proxy_logger, verbose_router_logger, ) -from litellm.litellm_core_utils.secret_redaction import redact_internal_details, redact_string +from litellm.litellm_core_utils.secret_redaction import ( + redact_internal_details, + redact_string, + redact_structured_value, +) SECRET = "sk-proj-abc123def456ghi789jklmnopqrst" @@ -71,6 +75,17 @@ def test_redact_string_catches_secret_patterns(): assert redact_string(normal) == normal +@pytest.mark.parametrize("native", (False, True), ids=("python", "rust")) +def test_diagnostic_redaction_policy_matches_across_backends(monkeypatch: pytest.MonkeyPatch, native: bool) -> None: + if native: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1" if native else "0") + + assert redact_string("GET /v1?api_key=abcdefgh12345&page=2") == "GET /v1?REDACTED&page=2" + assert redact_structured_value("db_url", "postgresql://reader@example.org/database") == "REDACTED" + assert redact_internal_details("failed at /etc/service/keys on db.internal") == "failed at REDACTED on REDACTED" + + @pytest.mark.parametrize( "connection_string", [ diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 89472cbd20f..ce280cc3513 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -8,7 +8,7 @@ import logging import os import queue import threading -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone from pathlib import PurePath @@ -5283,23 +5283,31 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error( ) -> None: """Regression: an error raised after the deployment call already succeeded (e.g. inside async_post_call_success_deployment_hook or post_call_processing) is not a deployment - attempt failure and must not reach async_post_call_failure_deployment_hook.""" + attempt failure and must not reach async_post_call_failure_deployment_hook. The raising + callback is a guardrail because a plain logger's success hook error is isolated and + logged instead of propagating out of the call.""" - class ExplodingSuccessLogger(CustomLogger): + class ExplodingSuccessGuardrail(CustomGuardrail): def __init__(self) -> None: - super().__init__() - self.failure_calls: list[Exception] = [] + super().__init__(guardrail_name="exploding") + self.failure_calls: tuple[Exception, ...] = () - async def async_post_call_success_deployment_hook(self, request_data, response, call_type): + async def async_post_call_success_deployment_hook( + self, request_data: Mapping[str, object], response: LLMResponseTypes, call_type: CallTypes | None + ) -> LLMResponseTypes | None: raise RuntimeError("boom in success hook, model call itself succeeded") async def async_post_call_failure_deployment_hook( - self, request_data, exception, call_type, fallback_depth=None - ): - self.failure_calls.append(exception) + self, + request_data: Mapping[str, object], + exception: Exception, + call_type: CallTypes | None, + fallback_depth: int | None = None, + ) -> None: + self.failure_calls = (*self.failure_calls, exception) - exploding_logger = ExplodingSuccessLogger() - monkeypatch.setattr(litellm, "callbacks", [exploding_logger]) + exploding_guardrail: Final = ExplodingSuccessGuardrail() + monkeypatch.setattr(litellm, "callbacks", [exploding_guardrail]) with pytest.raises(RuntimeError, match="boom in success hook"): await litellm.acompletion( @@ -5308,7 +5316,7 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error( mock_response="this call succeeds", ) - assert exploding_logger.failure_calls == [] + assert exploding_guardrail.failure_calls == () @pytest.mark.asyncio diff --git a/tests/test_litellm/vector_stores/test_vector_store_registry.py b/tests/test_litellm/vector_stores/test_vector_store_registry.py index f19c3706845..762176d6a81 100644 --- a/tests/test_litellm/vector_stores/test_vector_store_registry.py +++ b/tests/test_litellm/vector_stores/test_vector_store_registry.py @@ -8,7 +8,7 @@ from fastapi.testclient import TestClient from datetime import datetime, timezone -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import litellm from litellm.types.vector_stores import LiteLLM_ManagedVectorStore @@ -182,3 +182,70 @@ def test_search_uses_registry_credentials(): assert getattr(called_params, "aws_region_name") == "us-east-1" finally: litellm.vector_store_registry = original_registry + + +def _config_registry(vector_store_id: str = "vs_from_config") -> VectorStoreRegistry: + registry = VectorStoreRegistry(vector_stores=[]) + registry.load_vector_stores_from_config( + [ + { + "vector_store_name": "config-store", + "litellm_params": {"vector_store_id": vector_store_id, "custom_llm_provider": "openai"}, + } + ] + ) + return registry + + +def _db_store(vector_store_id: str, vector_store_name: str) -> LiteLLM_ManagedVectorStore: + return LiteLLM_ManagedVectorStore( + vector_store_id=vector_store_id, + custom_llm_provider="openai", + vector_store_name=vector_store_name, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + + +def test_config_loaded_store_is_marked_config_owned_and_db_store_is_not(): + registry = _config_registry() + registry.add_vector_store_to_registry(_db_store("vs_from_db", "db-store")) + + assert registry.get_litellm_managed_vector_store_from_registry("vs_from_config")["is_config"] is True + assert registry.is_config_vector_store("vs_from_config") is True + assert registry.is_config_vector_store("vs_from_db") is False + assert registry.is_config_vector_store("vs_unknown") is False + + +def test_db_row_does_not_overwrite_config_owned_store_in_registry(): + registry = _config_registry() + registry.add_vector_store_to_registry(_db_store("vs_from_db", "db-store")) + + registry.update_vector_store_in_registry("vs_from_config", _db_store("vs_from_config", "renamed-in-db")) + registry.update_vector_store_in_registry("vs_from_db", _db_store("vs_from_db", "renamed-in-db")) + + assert registry.get_litellm_managed_vector_store_from_registry("vs_from_config") == { + **registry.get_litellm_managed_vector_store_from_registry("vs_from_config"), + "vector_store_name": "config-store", + "is_config": True, + } + assert registry.get_litellm_managed_vector_store_from_registry("vs_from_db")["vector_store_name"] == "renamed-in-db" + + +@pytest.mark.asyncio +async def test_config_owned_store_survives_db_liveness_check_while_missing_db_store_is_evicted(): + registry = _config_registry() + registry.add_vector_store_to_registry(_db_store("vs_from_db", "db-store")) + prisma_client = MagicMock() + prisma_client.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=None) + + to_run = await registry.pop_vector_stores_to_run_with_db_fallback( + non_default_params={"vector_store_ids": ["vs_from_config", "vs_from_db"]}, + prisma_client=prisma_client, + ) + + assert [vs["vector_store_id"] for vs in to_run] == ["vs_from_config"] + assert [vs["vector_store_id"] for vs in registry.vector_stores] == ["vs_from_config"] + prisma_client.db.litellm_managedvectorstorestable.find_unique.assert_awaited_once_with( + where={"vector_store_id": "vs_from_db"} + ) diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index cf2fd78a896..3846a94c9fe 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -541,26 +541,48 @@ def test_output_config_format_converted_for_bedrock_chat_invoke_request(): assert json.loads(last_content[-1]["text"]) == schema -def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(): +@pytest.mark.parametrize("model", ["anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-8"]) +def test_output_config_format_inlined_for_bedrock_chat_invoke_opus_4_7_and_4_8(local_model_cost_map, model): + """Bedrock rejects ``output_config.format`` on Claude Opus 4.7 and 4.8, so the + Invoke chat path inlines the schema into the last user message and keeps effort, + driven by the cost map alone (no capability stub).""" + schema = {"type": "object", "properties": {"answer": {"type": "string"}}} + + result = AmazonAnthropicClaudeConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "test"}], + optional_params={ + "max_tokens": 100, + "output_config": {"effort": "xhigh", "format": {"type": "json_schema", "schema": schema}}, + }, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"effort": "xhigh"} + assert json.loads(result["messages"][-1]["content"][-1]["text"]) == schema + + +def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map): """Bedrock Invoke chat path forwards ``output_config.format`` alongside effort - for models with native structured-output support (Claude Opus 4.7).""" + for models with native structured-output support (Claude Sonnet 4.6).""" schema_format = { "type": "json_schema", "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, } result = AmazonAnthropicClaudeConfig().transform_request( - model="anthropic.claude-opus-4-7", + model="us.anthropic.claude-sonnet-4-6", messages=[{"role": "user", "content": "test"}], optional_params={ "max_tokens": 100, - "output_config": {"effort": "xhigh", "format": schema_format}, + "output_config": {"effort": "max", "format": schema_format}, }, litellm_params={}, headers={}, ) - assert result.get("output_config") == {"effort": "xhigh", "format": schema_format} + assert result.get("output_config") == {"effort": "max", "format": schema_format} assert "answer" not in json.dumps(result["messages"]) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTable.test.tsx index 7aae01254da..33e982e2735 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTable.test.tsx @@ -59,7 +59,16 @@ describe("VectorStoreTable", () => { it("should render every column header", () => { render(); - for (const header of ["Vector Store ID", "Name", "Description", "Files", "Provider", "Created At", "Updated At"]) { + for (const header of [ + "Vector Store ID", + "Name", + "Description", + "Source", + "Files", + "Provider", + "Created At", + "Updated At", + ]) { expect(screen.getByText(header)).toBeInTheDocument(); } }); @@ -112,6 +121,36 @@ describe("VectorStoreTable", () => { expect(mockOnDelete).toHaveBeenCalledWith("vs-newer"); }); + it("should label each row's source as Config or DB", () => { + const configStore: VectorStore = { ...mockVectorStores[1], vector_store_id: "vs-config", is_config: true }; + render(); + const rows = screen.getAllByRole("row").slice(1); + const dbRow = rows.find((row) => within(row).queryByText("vs-newer")); + const configRow = rows.find((row) => within(row).queryByText("vs-config")); + expect(within(dbRow!).getByText("DB")).toBeInTheDocument(); + expect(within(dbRow!).queryByText("Config")).not.toBeInTheDocument(); + expect(within(configRow!).getByText("Config")).toBeInTheDocument(); + expect(within(configRow!).queryByText("DB")).not.toBeInTheDocument(); + }); + + it("should keep edit and delete disabled for a config-defined store while copy still works", async () => { + const user = userEvent.setup(); + const configStore: VectorStore = { ...mockVectorStores[1], vector_store_id: "vs-config", is_config: true }; + render(); + await user.click(screen.getByTestId("vector-store-actions-vs-config")); + const editItem = await screen.findByTestId("vector-store-action-edit"); + const deleteItem = screen.getByTestId("vector-store-action-delete"); + expect(editItem).toHaveAttribute("aria-disabled", "true"); + expect(deleteItem).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByText(/Read only: this vector store is defined in the config file/)).toBeVisible(); + await user.click(editItem); + await user.click(deleteItem); + expect(mockOnEdit).not.toHaveBeenCalled(); + expect(mockOnDelete).not.toHaveBeenCalled(); + await user.click(screen.getByTestId("vector-store-action-copy")); + expect(await window.navigator.clipboard.readText()).toBe("vs-config"); + }); + it("should copy the vector store ID through the actions menu", async () => { const user = userEvent.setup(); render(); @@ -119,4 +158,12 @@ describe("VectorStoreTable", () => { await user.click(await screen.findByTestId("vector-store-action-copy")); expect(await window.navigator.clipboard.readText()).toBe("vs-newer"); }); + + it("should not show the read-only hint for a database-backed store", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByTestId("vector-store-actions-vs-newer")); + await screen.findByTestId("vector-store-action-edit"); + expect(screen.queryByText(/Read only: this vector store is defined in the config file/)).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTableColumns.tsx index a4c18e48e1a..d0ed333d3d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTableColumns.tsx @@ -4,7 +4,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Copy, MoreHorizontal, Pencil, Trash2 } from "lucide-react"; import { DataTableSortHeader } from "@/components/shared/DataTable"; -import { CellTooltip, DateCell, IdentityCell } from "@/components/shared/table_cells"; +import { CellTooltip, DateCell, IdentityCell, StatusBadge } from "@/components/shared/table_cells"; import { getVectorStoreProviderLogoAndName } from "@/components/vector_store_providers"; import { buttonVariants } from "@/components/ui/button"; import { @@ -18,6 +18,9 @@ import { VectorStore } from "@/components/vector_store_management/types"; import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; +const CONFIG_STORE_HINT = + "Read only: this vector store is defined in the config file and cannot be edited or deleted on the dashboard."; + function VectorStoreProviderCell({ provider }: { provider: string }) { const { displayName, logo } = getVectorStoreProviderLogoAndName(provider); return ( @@ -64,6 +67,7 @@ interface VectorStoreRowActionsProps { } function VectorStoreRowActions({ vectorStore, onEdit, onDelete }: VectorStoreRowActionsProps) { + const isFromConfig = vectorStore.is_config ?? false; return ( - onEdit(vectorStore.vector_store_id)}> + onEdit(vectorStore.vector_store_id)} + > Edit @@ -89,11 +97,17 @@ function VectorStoreRowActions({ vectorStore, onEdit, onDelete }: VectorStoreRow onDelete(vectorStore.vector_store_id)} > Delete + {isFromConfig && ( +
+ {CONFIG_STORE_HINT} +
+ )}
); @@ -158,6 +172,18 @@ export const getVectorStoreTableColumns = ({ ); }, }, + { + id: "source", + accessorFn: (row) => row.is_config ?? false, + meta: { title: "Source", skeleton: "badge" }, + header: ({ column }) => , + size: 110, + enableSorting: true, + cell: ({ row }) => { + const isFromConfig = row.original.is_config ?? false; + return ; + }, + }, { id: "files", meta: { title: "Files" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.test.tsx index 697d5f9692b..1600518ecdf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.test.tsx @@ -59,6 +59,61 @@ describe("VectorStoreInfoView", () => { expect(await screen.findByText("Vector Store ID: vs-1")).toBeInTheDocument(); }); + it("should render a config-defined store read-only for an admin, even when opened in edit mode", async () => { + mockVectorStoreInfoCall.mockResolvedValue({ + vector_store: { + vector_store_id: "vs-config", + vector_store_name: "config-store", + custom_llm_provider: "openai", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + is_config: true, + }, + }); + render( + , + ); + expect(await screen.findByText("Vector Store ID: vs-config")).toBeInTheDocument(); + expect(screen.getByText("Read only: defined in the config file")).toBeInTheDocument(); + expect(screen.getByText("Config")).toBeInTheDocument(); + expect(screen.queryByText("DB")).not.toBeInTheDocument(); + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Edit Vector Store" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Save/ })).not.toBeInTheDocument(); + }); + + it("should still offer editing for a database-backed store", async () => { + mockVectorStoreInfoCall.mockResolvedValue({ + vector_store: { + vector_store_id: "vs-db", + vector_store_name: "db-store", + custom_llm_provider: "openai", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + is_config: false, + }, + }); + render( + , + ); + expect(await screen.findByText("Vector Store ID: vs-db")).toBeInTheDocument(); + expect(screen.queryByText("Read only: defined in the config file")).not.toBeInTheDocument(); + expect(screen.getByText("DB")).toBeInTheDocument(); + expect(screen.getAllByRole("button", { name: "Edit Vector Store" }).length).toBeGreaterThan(0); + }); + it("should show a not-found state with a working back button when the fetch fails instead of loading forever", async () => { const user = userEvent.setup(); const onClose = vi.fn(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx index 58fd2ce1143..92d4145b9f2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/vector_store_info.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from "react"; -import { ArrowLeft, CircleHelp } from "lucide-react"; +import { ArrowLeft, CircleHelp, Lock } from "lucide-react"; import { z } from "zod/v4"; import { vectorStoreInfoCall, @@ -15,6 +15,8 @@ import VectorStoreTester from "./VectorStoreTester"; import { toast } from "@/lib/toast"; import { FieldGroup } from "@/components/ui/field"; import { FormField } from "@/components/shared/form/FormField"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { StatusBadge } from "@/components/shared/table_cells"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; @@ -200,6 +202,9 @@ const VectorStoreInfoView: React.FC = ({ return
Loading...
; } + const canEdit = is_admin && !vectorStoreDetails.is_config; + const showEditForm = isEditing && canEdit; + return (
@@ -208,14 +213,31 @@ const VectorStoreInfoView: React.FC = ({ Back to Vector Stores -

Vector Store ID: {vectorStoreDetails.vector_store_id}

+
+

Vector Store ID: {vectorStoreDetails.vector_store_id}

+ +

{vectorStoreDetails.vector_store_description || "No description"}

- {is_admin && !isEditing && } + {canEdit && !isEditing && }
+ {vectorStoreDetails.is_config && ( + + + Read only: defined in the config file + + This vector store comes from the proxy config YAML, so it cannot be edited or deleted on the dashboard. + Change or remove it in the config file and restart the proxy. + + + )} + @@ -227,7 +249,7 @@ const VectorStoreInfoView: React.FC = ({ - {isEditing ? ( + {showEditForm ? (

Edit Vector Store

@@ -373,7 +395,7 @@ const VectorStoreInfoView: React.FC = ({

Vector Store Details

- {is_admin && } + {canEdit && }
diff --git a/ui/litellm-dashboard/src/components/vector_store_management/types.tsx b/ui/litellm-dashboard/src/components/vector_store_management/types.tsx index 2fe1c732ecc..bfa803f02ea 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/types.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/types.tsx @@ -23,6 +23,7 @@ export interface VectorStore { updated_at: string; created_by?: string; updated_by?: string; + is_config?: boolean; } export interface VectorStoreInfoRequest { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ebd87b30901..94b5238ea93 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21168,7 +21168,9 @@ export interface paths { * List Vector Stores * @description List all available vector stores with optional filtering and pagination. * Combines both in-memory vector stores and those stored in the database. - * Database is the source of truth - deleted stores are removed from memory, updated stores sync to memory. + * Database is the source of truth for stores it owns: deleted stores are removed from memory, updated stores + * sync to memory. Stores declared in the config file are owned by the config file, are always listed, and are + * never overwritten by database rows. * * Parameters: * - page: int - Page number for pagination (default: 1) @@ -22471,7 +22473,9 @@ export interface paths { * List Vector Stores * @description List all available vector stores with optional filtering and pagination. * Combines both in-memory vector stores and those stored in the database. - * Database is the source of truth - deleted stores are removed from memory, updated stores sync to memory. + * Database is the source of truth for stores it owns: deleted stores are removed from memory, updated stores + * sync to memory. Stores declared in the config file are owned by the config file, are always listed, and are + * never overwritten by database rows. * * Parameters: * - page: int - Page number for pagination (default: 1) @@ -30876,6 +30880,8 @@ export interface components { created_at?: string | null; /** Custom Llm Provider */ custom_llm_provider?: string; + /** Is Config */ + is_config?: boolean; /** Litellm Credential Name */ litellm_credential_name?: string | null; /** Litellm Params */ @@ -30947,6 +30953,11 @@ export interface components { created_at?: string | null; /** Custom Llm Provider */ custom_llm_provider: string; + /** + * Is Config + * @default false + */ + is_config: boolean; /** Litellm Credential Name */ litellm_credential_name?: string | null; /** Litellm Params */ diff --git a/uv.lock b/uv.lock index 32d7580f61b..85e2b6d4e52 100644 --- a/uv.lock +++ b/uv.lock @@ -4499,7 +4499,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.103.0" +version = "1.104.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -4959,12 +4959,12 @@ proxy-dev = [ [[package]] name = "litellm-enterprise" -version = "0.1.69" +version = "0.1.70" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.100" +version = "0.4.101" source = { editable = "litellm-proxy-extras" } [[package]] diff --git a/whitelisted_bedrock_models.txt b/whitelisted_bedrock_models.txt index 254842e2714..4bca4af2435 100644 --- a/whitelisted_bedrock_models.txt +++ b/whitelisted_bedrock_models.txt @@ -90,6 +90,7 @@ bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0 bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0 bedrock/eu-west-2/minimax.minimax-m2.1 bedrock/eu-west-2/minimax.minimax-m2.5 +bedrock/eu-west-2/nvidia.nemotron-super-3-120b bedrock/eu-west-2/qwen.qwen3-coder-next bedrock/eu-west-2/qwen.qwen3-next-80b-a3b bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2 @@ -234,6 +235,7 @@ bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0 bedrock/us-gov-west-1/anthropic.claude-sonnet-5 bedrock/us-gov-west-1/anthropic.claude-opus-4-8 bedrock/us-gov-west-1/anthropic.claude-opus-5 +bedrock/us-gov-west-1/anthropic.claude-opus-5-5 bedrock/us-gov-west-1/anthropic.claude-fable-5-1 bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2 @@ -244,5 +246,6 @@ bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0 bedrock/us-gov-east-1/anthropic.claude-sonnet-5 bedrock/us-gov-east-1/anthropic.claude-opus-4-8 bedrock/us-gov-east-1/anthropic.claude-opus-5 +bedrock/us-gov-east-1/anthropic.claude-opus-5-5 bedrock/us-gov-east-1/anthropic.claude-fable-5-1 bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b