diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a707eb8044c..f6c06f7d83f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ -/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri -/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri +/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri +/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /helm/ @yassin-berriai /terraform/ @yassin-berriai diff --git a/.github/workflows/interactions-spec-canary.yml b/.github/workflows/interactions-spec-canary.yml new file mode 100644 index 00000000000..7ec3defdcb7 --- /dev/null +++ b/.github/workflows/interactions-spec-canary.yml @@ -0,0 +1,91 @@ +name: Interactions spec canary + +on: + schedule: + - cron: "17 7 * * *" + pull_request: + paths: + - .github/workflows/interactions-spec-canary.yml + - tests/unit/interactions/** + - litellm/types/interactions/** + workflow_dispatch: + +permissions: {} + +concurrency: interactions-spec-canary + +jobs: + interactions-spec-canary: + name: interactions-spec-canary + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + compliance: ${{ steps.compliance.outcome }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project + + - name: Run the compliance tests against Google's live spec + id: compliance + run: | + uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \ + --interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \ + --tb=short -vv -p no:rerunfailures -p no:pytest-retry \ + --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev + + tracking-issue: + name: tracking-issue + needs: interactions-spec-canary + if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + steps: + - name: File the tracking issue unless the compliance step passed, close it when it did + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + script: | + const title = "Interactions spec canary is red"; + const outcome = process.env.COMPLIANCE_OUTCOME; + const runUrl = process.env.RUN_URL; + const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`; + const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 }); + const open = found.items.find((issue) => issue.title === title); + if (outcome === "success") { + if (open === undefined) return; + await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` }); + await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" }); + return; + } + const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`; + if (open !== undefined) { + await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` }); + return; + } + await github.rest.issues.create({ + ...context.repo, + title, + assignees: ["mateo-berri"], + body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`, + }); diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2276efc4e34..76af64c8ccd 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -425,6 +425,7 @@ jobs: tests/unit/realtime_api tests/unit/rerank_api tests/unit/search + tests/unit/text_completions tests/unit/vector_stores tests/unit/videos python-version: "3.12" diff --git a/Dockerfile b/Dockerfile index 890c90fe449..713d7f8f5ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -61,6 +61,7 @@ RUN apk add --no-cache \ python-3.13-dev \ rust \ openssl \ + libssl3 \ openssl-dev \ nodejs \ npm \ @@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder COPY --from=uvbin /uv /usr/local/bin/uv RUN apk add --no-cache python-3.13 -ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \ - https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz +ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \ + https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \ uv venv /opt/liteadmin --python python3.13 && \ uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \ @@ -141,7 +142,7 @@ USER root RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app diff --git a/README.md b/README.md index 5fd84b529dc..4feb6e00ffa 100644 --- a/README.md +++ b/README.md @@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws ### Verify Docker Image Signatures -All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). +All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065). **Verify using the pinned commit hash (recommended):** -A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key: ```bash cosign verify \ - --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + --key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \ ghcr.io/berriai/litellm: ``` diff --git a/security.md b/SECURITY.md similarity index 98% rename from security.md rename to SECURITY.md index c73379a9c01..d3603448d46 100644 --- a/security.md +++ b/SECURITY.md @@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod - Audit Logs with retention policy - Control Allowed IP Addresses that can access your Cloud LiteLLM Instance -For security inquiries, please contact us at support@berri.ai +For security inquiries, please contact us at security@berri.ai #### Supported data regions for LiteLLM Cloud diff --git a/backend/Dockerfile b/backend/Dockerfile index 84c90e16f67..46d65c432bb 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ + apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/cosign.pub b/cosign.pub index 2c2a555ab09..7603d69641f 100644 --- a/cosign.pub +++ b/cosign.pub @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb -POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g== +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg +JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ== -----END PUBLIC KEY----- diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 56210458532..69d9c26b2a8 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -59,6 +59,7 @@ RUN apk add --no-cache \ python-3.13 \ python-3.13-dev \ openssl \ + libssl3 \ openssl-dev \ nodejs \ npm \ @@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 086c38e0d60..54bf8c807aa 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \ coreutils \ curl \ openssl \ + libssl3 \ libsndfile \ nodejs \ npm && break || sleep 5; \ @@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \ + apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \ done COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip index a5733f0e1a0..25a7ed2fa97 100644 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ b/docker/build_from_pip/Dockerfile.build_from_pip @@ -30,7 +30,7 @@ RUN uv venv --python python && \ "google-cloud-aiplatform==1.133.0" \ "google-genai==1.37.0" \ "anthropic[vertex]==0.84.0" \ - "grpcio==1.78.0" \ + "grpcio==1.84.1" \ "prometheus-client==0.20.0" \ "langfuse==2.59.7" \ "opentelemetry-api==1.28.0" \ diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 051b1c30c8f..390a5f5666f 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) ARG PGBOUNCER_VERSION=1.26.0 @@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \ + apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 98f36607fff..c868d3e4b8a 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -8202,18 +8202,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index f04eb115071..9e5e0a96b73 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" percent-encoding = "2.3" -webpki-roots = "1" +webpki-roots = "1.0.10" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" fancy-regex = "0.19.2" diff --git a/litellm-rust/crates/gateway-inference/src/caching.rs b/litellm-rust/crates/gateway-inference/src/caching.rs index 1abea61b164..426b8b91182 100644 --- a/litellm-rust/crates/gateway-inference/src/caching.rs +++ b/litellm-rust/crates/gateway-inference/src/caching.rs @@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result { #[derive(Clone, Default)] pub(crate) struct CacheHeaders(std::sync::Arc>); -impl litellm_host::interceptors::Interceptors for CacheHeaders { +impl litellm_host::interceptors::Interceptors for CacheHeaders { async fn before_provider_request( &self, wire: litellm_host::interceptors::WireRequest, _: litellm_host::interceptors::RequestContext, - ) -> Result { + ) -> Result { Ok(wire) } async fn after_provider_response( &self, _: litellm_host::interceptors::RawResponse, - ) -> Result<(), litellm_inference::RouteError> { + ) -> Result<(), litellm_inference::Error> { Ok(()) } async fn result_ready( &self, facts: litellm_host::interceptors::ExecutionFacts, - ) -> Result<(), litellm_inference::RouteError> { + ) -> Result<(), litellm_inference::Error> { if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source { let _ = self.0.set(key); } diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index 39656b933eb..ec82a6ede12 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -4,7 +4,7 @@ use axum::{ response::{IntoResponse, Response}, }; use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error as InferenceError; use litellm_llms::base_llm::ocr::error::Error as OcrError; use serde_json::{Map, Value, json}; @@ -19,7 +19,7 @@ pub enum Error { )] UnknownModel(String), #[error(transparent)] - Route(#[from] RouteError), + Route(#[from] InferenceError), #[error(transparent)] Ocr(#[from] OcrError), #[error("{0} is not implemented by the Rust gateway")] @@ -36,8 +36,8 @@ impl IntoResponse for Error { } } -impl From> for Error { - fn from(error: litellm_host_http::Error) -> Self { +impl From> for Error { + fn from(error: litellm_host_http::Error) -> Self { match error { litellm_host_http::Error::Call(error) => Self::Route(error), litellm_host_http::Error::Protocol => Self::Internal(error.to_string()), @@ -50,7 +50,7 @@ impl Error { match self { Self::Auth(error) => error.status(), Self::Unsupported(_) - | Self::Route(RouteError::Unsupported(_)) + | Self::Route(InferenceError::Unsupported(_)) | Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED, Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE, Self::Ocr( @@ -64,10 +64,10 @@ impl Error { .and_then(|status| StatusCode::from_u16(status).ok()) .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST, - Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => { + Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => { StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY) } - Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => { + Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => { StatusCode::UNAUTHORIZED } Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST, @@ -97,7 +97,9 @@ impl Error { /// body already in that shape passes through, any other has its message extracted. pub fn body(&self, request_id: Option<&str>) -> Value { let raw = match self { - Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(), + Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => { + body.clone() + } other => other.to_string(), }; let parsed = serde_json::from_str::(&raw).ok(); @@ -175,7 +177,7 @@ mod tests { use super::*; fn upstream(status: u16, body: &str) -> Error { - Error::Route(RouteError::Transport(TransportError::Http { + Error::Route(InferenceError::Transport(TransportError::Http { status, body: body.into(), })) @@ -225,16 +227,16 @@ mod tests { #[rstest] #[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)] - #[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)] + #[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)] #[case::missing_key( - Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { + Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { provider: "Anthropic", environment_variable: "ANTHROPIC_API_KEY", })), StatusCode::UNAUTHORIZED, )] #[case::lost_connection( - Error::Route(RouteError::Transport(TransportError::Network("reset".into()))), + Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))), StatusCode::INTERNAL_SERVER_ERROR, )] fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) { diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index f6b391e0ba1..3be83840901 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -1,7 +1,7 @@ use litellm_host::observation::ObservationSender; pub mod route; pub mod types; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; mod common_utils; pub mod constants; pub(crate) mod handler; diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index 3fc432f8b46..70da8bc3255 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -21,7 +21,7 @@ use litellm_host::{ lifecycle::{CallEvent, ExecutionEvent}, observation::observation_channel, }; -use litellm_inference::RouteError; +use litellm_inference::Error; use rstest::{fixture, rstest}; use serde_json::{Value, json}; @@ -166,12 +166,12 @@ struct ChangingHooks { facts: std::sync::Mutex>, } -impl Interceptors for ChangingHooks { +impl Interceptors for ChangingHooks { async fn before_provider_request( &self, mut wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { let call = self.calls.fetch_add(1, Ordering::SeqCst); if self.rewrite { wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 }); @@ -179,11 +179,11 @@ impl Interceptors for ChangingHooks { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.facts.lock().unwrap().push(facts); Ok(()) } diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index f91d70445f3..bfefb9c63b0 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use crate::common_utils::messages_provider; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; pub use types::{ MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, messages_body, diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 32c962091ad..480b3c4f3b7 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -17,7 +17,7 @@ use litellm_host::interceptors::{ ResultSource, WireRequest, }; use litellm_inference::{ - RouteError, + Error, caching::{CacheRequest, execute_unary}, }; use rstest::{fixture, rstest}; @@ -133,12 +133,12 @@ struct ChangingHooks { facts: std::sync::Mutex>, } -impl Interceptors for ChangingHooks { +impl Interceptors for ChangingHooks { async fn before_provider_request( &self, mut wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { let call = self.calls.fetch_add(1, Ordering::SeqCst); if self.rewrite { wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 }); @@ -146,11 +146,11 @@ impl Interceptors for ChangingHooks { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.facts.lock().unwrap().push(facts); Ok(()) } diff --git a/litellm-rust/crates/inference-ocr/AGENTS.md b/litellm-rust/crates/inference-ocr/AGENTS.md index d74b7e7af06..e08f561e75d 100644 --- a/litellm-rust/crates/inference-ocr/AGENTS.md +++ b/litellm-rust/crates/inference-ocr/AGENTS.md @@ -3,4 +3,4 @@ - The route returns its completed response directly - Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`) - Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors` -- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error` +- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error` diff --git a/litellm-rust/crates/inference-ocr/src/provider_config.rs b/litellm-rust/crates/inference-ocr/src/provider_config.rs index 2f0346f6e47..09041e9b72b 100644 --- a/litellm-rust/crates/inference-ocr/src/provider_config.rs +++ b/litellm-rust/crates/inference-ocr/src/provider_config.rs @@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>( provider: LlmProviders::Mistral, } } - Err(litellm_inference::RouteError::InvalidProvider(provider)) => { + Err(litellm_inference::Error::InvalidProvider(provider)) => { return Err(Error::InvalidProvider(provider)); } Err(error) => return Err(Error::InvalidProvider(error.to_string())), diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index 6a9d5372311..b6e151ecec3 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -1,5 +1,5 @@ use litellm_host::observation::ObservationSender; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; pub mod ids; pub mod route; diff --git a/litellm-rust/crates/inference-responses/tests/caching.rs b/litellm-rust/crates/inference-responses/tests/caching.rs index f45cefdc13b..462461695c3 100644 --- a/litellm-rust/crates/inference-responses/tests/caching.rs +++ b/litellm-rust/crates/inference-responses/tests/caching.rs @@ -18,7 +18,7 @@ use litellm_host::interceptors::{ WireRequest, }; use litellm_inference::{ - RouteError, + Error, caching::{CacheRequest, execute_unary}, }; use rstest::{fixture, rstest}; @@ -124,12 +124,12 @@ struct ChangingHooks { facts: std::sync::Mutex>, } -impl Interceptors for ChangingHooks { +impl Interceptors for ChangingHooks { async fn before_provider_request( &self, mut wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { let call = self.calls.fetch_add(1, Ordering::SeqCst); if self.rewrite { wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 }); @@ -137,11 +137,11 @@ impl Interceptors for ChangingHooks { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.facts.lock().unwrap().push(facts); Ok(()) } diff --git a/litellm-rust/crates/inference-transcription/src/lib.rs b/litellm-rust/crates/inference-transcription/src/lib.rs index 47e7bc70b31..f11e3abcd54 100644 --- a/litellm-rust/crates/inference-transcription/src/lib.rs +++ b/litellm-rust/crates/inference-transcription/src/lib.rs @@ -1,5 +1,5 @@ pub mod types; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; mod constants; mod handler; diff --git a/litellm-rust/crates/inference/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md index 2c4bd00108b..7f2bd0387be 100644 --- a/litellm-rust/crates/inference/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -2,7 +2,7 @@ - `gateway-inference` (serving) -> `inference-` (one API's call) -> `inference` (shared base) - This crate is the shared base every format crate builds on - - `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`) + - `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`) - diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`) - response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture - Shared test helpers belong in litellm-inference-testing, used only as a dev dependency @@ -49,14 +49,14 @@ - `litellm-inference-` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...) - Provider code never imports from `inference` or `inference-*` - Handlers belong in `inference-*` or `llms`, never in a host crate -- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `::Error` name +- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `::Error` name ## Error placement - The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to - A failure is declared once, by the lowest crate that raises it - every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error` - - `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises + - `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises - Scope follows the concept, not the first caller - an error type under `litellm-llms`'s `/` is private to that provider: no other provider and nothing in `base_llm` may import it - a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer diff --git a/litellm-rust/crates/inference/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs index 6dfda20ead4..09d03c06215 100644 --- a/litellm-rust/crates/inference/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value; use tokio_util::codec::Decoder; -use crate::RouteError; +use crate::Error; -pub trait Cachable: Protocol { +pub trait Cachable: Protocol { const SURFACE: &'static str; fn reusable(_response: &Self::Response) -> bool { @@ -145,15 +145,15 @@ pub async fn execute_unary( request: CacheRequest, cache: Option>, options: Option, - interceptors: &impl Interceptors, + interceptors: &impl Interceptors, observers: Option<&ObservationSender>, provider: F, -) -> Result +) -> Result where P: Cachable, P::Response: Serialize + DeserializeOwned, F: FnOnce() -> Fut, - Fut: Future>, + Fut: Future>, { let identity = request.identity.clone(); crate::diagnostic::provider(&identity.model, &identity.provider); @@ -191,15 +191,15 @@ pub async fn execute_streaming( request: CacheRequest, cache: Option>, options: Option, - interceptors: &impl Interceptors, + interceptors: &impl Interceptors, observers: Option<&ObservationSender>, provider: F, -) -> Result, RouteError> +) -> Result, Error> where P: StreamCachable, P::Response: Serialize + DeserializeOwned, F: FnOnce() -> Fut, - Fut: Future, RouteError>>, + Fut: Future, Error>>, { let identity = request.identity.clone(); crate::diagnostic::provider(&identity.model, &identity.provider); @@ -299,9 +299,9 @@ impl CallCache

{ } fn capture_stream( - chunks: futures_util::stream::BoxStream<'static, Result>, + chunks: futures_util::stream::BoxStream<'static, Result>, session: CacheSession, -) -> futures_util::stream::BoxStream<'static, Result> { +) -> futures_util::stream::BoxStream<'static, Result> { stream::try_unfold( (chunks, Some(Vec::::new()), session), |(mut chunks, captured, session)| async move { @@ -330,7 +330,7 @@ fn capture_stream( { session.store(entry).await; } - Ok::<_, RouteError>(None) + Ok::<_, Error>(None) } } }, @@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool { async fn publish( facts: ExecutionFacts, - interceptors: &impl Interceptors, + interceptors: &impl Interceptors, observers: Option<&ObservationSender>, -) -> Result<(), RouteError> { +) -> Result<(), Error> { if let Some(observers) = observers { observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady { facts: facts.clone(), diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs index 6fcbb3051b4..655b768dde1 100644 --- a/litellm-rust/crates/inference/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -5,7 +5,7 @@ use litellm_host::{ observation::ObservationSender, }; -use crate::{CallOptions, RouteError}; +use crate::{CallOptions, Error}; pub struct CallContext<'a, I> { pub interceptors: &'a I, @@ -13,7 +13,7 @@ pub struct CallContext<'a, I> { pub cache: CachePolicy, } -impl<'a, I: Interceptors> CallContext<'a, I> { +impl<'a, I: Interceptors> CallContext<'a, I> { pub fn new(interceptors: &'a I, options: CallOptions) -> Self { Self { interceptors, @@ -22,7 +22,7 @@ impl<'a, I: Interceptors> CallContext<'a, I> { } } - pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { if let Some(observers) = &self.observers { observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady { facts: facts.clone(), @@ -31,7 +31,7 @@ impl<'a, I: Interceptors> CallContext<'a, I> { self.interceptors.result_ready(facts).await } - pub async fn response_received(&self, body: &str) -> Result<(), RouteError> { + pub async fn response_received(&self, body: &str) -> Result<(), Error> { let raw = RawResponse { body: body.to_owned(), }; @@ -43,6 +43,6 @@ impl<'a, I: Interceptors> CallContext<'a, I> { self.interceptors .after_provider_response(raw) .await - .map_err(RouteError::post_call) + .map_err(Error::post_call) } } diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index e5f02754f3d..895c8c55e01 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError; use litellm_llms::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] -pub enum RouteError { +pub enum Error { #[error("expected {expected}, got {actual}")] InvalidType { expected: &'static str, @@ -38,10 +38,10 @@ pub enum RouteError { #[error(transparent)] Secret(#[from] SecretError), #[error("post-call hook failed: {0}")] - PostCallHook(#[source] Arc), + PostCallHook(#[source] Arc), } -impl From for RouteError { +impl From for Error { fn from(fault: litellm_host::machine::MachineFault) -> Self { use litellm_host::machine::MachineFault; Self::InvalidRequest(match fault { @@ -51,7 +51,7 @@ impl From for RouteError { } } -impl RouteError { +impl Error { pub fn post_call(error: Self) -> Self { Self::PostCallHook(Arc::new(error)) } @@ -76,7 +76,7 @@ impl RouteError { } } -impl From for RouteError { +impl From for Error { fn from(error: LlmError) -> Self { match error { LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual }, @@ -99,7 +99,7 @@ impl SecretError { } } -impl From for RouteError { +impl From for Error { fn from(error: litellm_secrets::Error) -> Self { Self::Secret(SecretError(Arc::new(error))) } @@ -115,22 +115,22 @@ impl Eq for SecretError {} #[cfg(test)] mod tests { - use super::RouteError; + use super::Error; use litellm_llms::{Error as LlmError, ErrorDetail}; use rstest::rstest; #[test] fn a_missing_api_key_is_the_environment_not_the_request() { assert!( - !RouteError::Auth(litellm_auth::Error::MissingApiKey { + !Error::Auth(litellm_auth::Error::MissingApiKey { provider: "Anthropic", environment_variable: "ANTHROPIC_API_KEY", }) .is_request() ); - assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request()); - assert!(RouteError::InvalidRequest("top_k".into()).is_request()); - assert!(!RouteError::InvalidResponse("bad json".into()).is_request()); + assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request()); + assert!(Error::InvalidRequest("top_k".into()).is_request()); + assert!(!Error::InvalidResponse("bad json".into()).is_request()); } #[rstest] #[case::request(true)] @@ -139,7 +139,7 @@ mod tests { let source = serde_json::from_str::("{").unwrap_err(); let source_message = source.to_string(); let detail = ErrorDetail::invalid("test payload", source); - let error = RouteError::from(if request { + let error = Error::from(if request { LlmError::InvalidRequest(detail) } else { LlmError::InvalidResponse(detail) diff --git a/litellm-rust/crates/inference/src/lib.rs b/litellm-rust/crates/inference/src/lib.rs index a168802e7ec..80771e206ca 100644 --- a/litellm-rust/crates/inference/src/lib.rs +++ b/litellm-rust/crates/inference/src/lib.rs @@ -7,7 +7,7 @@ pub mod outbound; pub mod provider; pub mod resources; -pub use error::RouteError; +pub use error::Error; #[derive(Clone, Default)] pub struct CallOptions { diff --git a/litellm-rust/crates/inference/src/provider.rs b/litellm-rust/crates/inference/src/provider.rs index 7e1269db1cf..305aca85f97 100644 --- a/litellm-rust/crates/inference/src/provider.rs +++ b/litellm-rust/crates/inference/src/provider.rs @@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{ CustomLlmProvider, LlmProviders, get_custom_llm_provider, }; -use crate::error::RouteError as Error; +use crate::error::Error; #[derive(Debug)] pub struct ResolvedProvider<'a> { diff --git a/litellm-rust/crates/inference/tests/caching.rs b/litellm-rust/crates/inference/tests/caching.rs index 1bd53695c30..a0a6a757565 100644 --- a/litellm-rust/crates/inference/tests/caching.rs +++ b/litellm-rust/crates/inference/tests/caching.rs @@ -25,7 +25,7 @@ use litellm_host::{ protocol::Protocol, }; use litellm_inference::{ - RouteError, + Error, caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary}, }; use rstest::{fixture, rstest}; @@ -36,7 +36,7 @@ struct TestRoute; impl Protocol for TestRoute { type Request = Value; type Response = Value; - type Error = RouteError; + type Error = Error; type HostCall = Infallible; type Chunk = Bytes; type StreamHead = (); @@ -204,7 +204,7 @@ async fn streamed( .chunks(3) .map(|bytes| Ok(Bytes::copy_from_slice(bytes))) .collect::>(); - let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure"))); + let ending = fail.then_some(Err(Error::Unsupported("test transport failure"))); Ok(CallOutput::Stream { head: (), chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(), @@ -215,7 +215,7 @@ async fn streamed( .unwrap() } -async fn consume(output: OutputOf) -> Result, RouteError> { +async fn consume(output: OutputOf) -> Result, Error> { let CallOutput::Stream { chunks, .. } = output else { panic!("expected a stream"); }; @@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc for CacheHitAccounting { - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { +impl Interceptors for CacheHitAccounting { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.calls.fetch_add(1, Ordering::SeqCst); let ResultSource::Cache { key } = facts.source else { panic!("expected cache source") @@ -482,7 +482,7 @@ impl Interceptors for CacheHitAccounting { ); self.key.set(key).unwrap(); if self.reject { - return Err(RouteError::Unsupported("cache accounting rejected")); + return Err(Error::Unsupported("cache accounting rejected")); } Ok(()) } @@ -491,11 +491,11 @@ impl Interceptors for CacheHitAccounting { &self, wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } } @@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure( if reject { assert!(matches!( result, - Err(RouteError::Unsupported("cache accounting rejected")) + Err(Error::Unsupported("cache accounting rejected")) )); } else { assert_eq!(result.unwrap(), expected); @@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure( impl Protocol for UnaryTestRoute { type Request = Value; type Response = Value; - type Error = RouteError; + type Error = Error; type HostCall = Infallible; type Chunk = Infallible; type StreamHead = Infallible; diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index b1509313b88..8429c98a88a 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -235,7 +235,10 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse], - Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[], + Self::ComputerUse20250124 + | Self::ComputerUse20251124 + | Self::Context1m20250807 + | Self::ThinkingBindingControls20260801 => &[], Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse], Self::Effort20251124 => &[BetaProvider::VertexAi], Self::FastMode20260201 | Self::Oauth20250420 => &[ @@ -272,7 +275,6 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi], - Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi], Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks], Self::ToolSearchTool20251019 => &[ BetaProvider::Anthropic, diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index dc63f0d6353..4dab0297e73 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,5 +1,5 @@ use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error; use pyo3::{ exceptions::{PyRuntimeError, PyValueError}, prelude::*, @@ -19,9 +19,9 @@ pyo3::create_exception!( "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." ); -pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr { +pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr { match error { - RouteError::Transport(TransportError::Http { status, body }) => { + Error::Transport(TransportError::Http { status, body }) => { RustUpstreamError::new_err((status, body)) } other => by_fault(other.is_request(), other.to_string()), @@ -42,13 +42,13 @@ mod tests { use super::*; #[rstest::rstest] - #[case::unsupported(RouteError::Unsupported("test capability"), true)] - #[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)] - #[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)] + #[case::unsupported(Error::Unsupported("test capability"), true)] + #[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)] + #[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)] #[case::connection(TransportError::Connect("unreachable".into()).into(), false)] #[case::network(TransportError::Network("timed out".into()).into(), false)] - #[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)] - fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) { + #[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)] + fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) { Python::initialize(); Python::attach(|py| { let failure = route_error_to_pyerr(error); @@ -86,14 +86,12 @@ mod tests { fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() { Python::initialize(); Python::attach(|py| { - let missing = - route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey { - provider: "Anthropic", - environment_variable: "ANTHROPIC_API_KEY", - })); + let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + })); assert!(missing.is_instance_of::(py)); - let invalid = - route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader)); + let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader)); assert!(invalid.is_instance_of::(py)); }); } diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index 3b590a522fe..c9eb60e47ed 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_host::{call::HostedMachine, protocol::Protocol}; use litellm_host_python::{PythonBinding, PythonHostCalls, from_py}; use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error; use litellm_secrets::source::SecretSource; use pyo3::{ exceptions::PyValueError, @@ -119,14 +119,14 @@ impl InferenceHost { public_response(py, self.module, response) } - pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult { - if let RouteError::Secret(source) = &error + pub fn error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Error::Secret(source) = &error && let Some(original) = crate::secrets::python_error(py, source.source_error()) { return Ok(original); } let native = match error { - RouteError::Transport(TransportError::Http { status, body }) => { + Error::Transport(TransportError::Http { status, body }) => { let error = RustUpstreamError::new_err((status, body)); error .value(py) @@ -144,7 +144,7 @@ impl InferenceHost { } pub(super) trait InferenceRoute: Sized + 'static { - type Protocol: Protocol; + type Protocol: Protocol; const OPERATION: LoggingOperation; const SYNC_CALL_TYPE: &'static str; const ASYNC_CALL_TYPE: &'static str; diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 236b18fe428..d68de6c9d4d 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -67,7 +67,7 @@ "structured-outputs-2025-11-13": "structured-outputs-2025-11-13", "text_editor_20241022": null, "text_editor_20250124": null, - "thinking-binding-controls-2026-08-01": null, + "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": null, "web-fetch-2025-09-10": "web-fetch-2025-09-10", diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 96c38b69104..61e36bd1e96 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -36,6 +36,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode # Cache for the loaded configuration _BETA_HEADERS_CONFIG: dict | None = None @@ -145,7 +146,7 @@ def get_beta_headers_config(url: str) -> dict: Dict containing the beta headers configuration """ # Check if local-only mode is enabled - if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true": # verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)") return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..99944662bf8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2264,6 +2264,8 @@ def response_cost_calculator( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") region_name: str | None = None, + custom_cost_per_token: CostPerToken | None = None, + custom_cost_per_second: float | None = None, ) -> float: """ Returns @@ -2292,6 +2294,8 @@ def response_cost_calculator( optional_params=optional_params, custom_pricing=custom_pricing, base_model=base_model, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, litellm_model_name=litellm_model_name, diff --git a/litellm/integrations/deepeval/types.py b/litellm/integrations/deepeval/types.py index eb4bb1f9b2a..a547863e26a 100644 --- a/litellm/integrations/deepeval/types.py +++ b/litellm/integrations/deepeval/types.py @@ -1,6 +1,6 @@ # Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py from enum import Enum -from typing import Any, ClassVar, Literal +from typing import ClassVar, Literal from pydantic import ConfigDict, Field @@ -55,7 +55,7 @@ class TraceApi(LiteLLMBaseModel): tool_spans: list[BaseApiSpan] = Field(alias="toolSpans") start_time: str = Field(alias="startTime") end_time: str = Field(alias="endTime") - metadata: dict[str, Any] | None = Field(None) + metadata: dict[str, object] | None = Field(None) tags: list[str] | None = Field(None) environment: str | None = Field(None) diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index cb52ad24c8e..c0d42eeb5c5 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi`` is unavailable. This MUST be called at app-creation time — once the lifespan - runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add - middleware after an application has started". + runs, the middleware stack is frozen and middleware added later never runs, so + a late call is skipped without registering anything. No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global ``ProxyTracerProvider``; the proxy publishes the real provider as the global @@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No try: if not is_otel_v2_enabled(): return + if app.middleware_stack is not None: + verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started") + return # Lazy: only the V2-enabled path needs the optional # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the diff --git a/litellm/integrations/otel/plumbing/otlp_json.py b/litellm/integrations/otel/plumbing/otlp_json.py index bc6d7d435b0..fc0351d5049 100644 --- a/litellm/integrations/otel/plumbing/otlp_json.py +++ b/litellm/integrations/otel/plumbing/otlp_json.py @@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex). import base64 import json +import logging from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final, TypeAlias @@ -13,9 +14,13 @@ from typing import Final, TypeAlias import requests from google.protobuf.json_format import MessageToDict from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE +_logger: Final = logging.getLogger(__name__) JSON_CONTENT_TYPE: Final = "application/json" _HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"}) @@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter): def __init__( self, endpoint: str | None, - headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict + headers: Mapping[str, str], certificate_file: str | None = None, session: "requests.Session | None" = None, + *, + max_request_size: int | None = None, ) -> None: - super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session) - self._session.headers["Content-Type"] = JSON_CONTENT_TYPE + super().__init__( + endpoint=endpoint, + headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE}, + certificate_file=certificate_file, + session=session, + max_request_size=max_request_size, + ) - def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes: - return encode_spans_json(spans) + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring batch") + return SpanExportResult.FAILURE + + with self._metrics.export_operation(len(spans)) as result: + try: + serialized_data: Final = encode_spans_json(spans) + except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results + _logger.error("Failed to encode span batch: %s", error) + result.error = error + return SpanExportResult.FAILURE + + if 0 < self._max_request_size < len(serialized_data): + result.error = RequestPayloadTooLargeError( + f"Serialized span request size {len(serialized_data)} bytes " + f"exceeds max_request_size {self._max_request_size} bytes." + ) + _logger.warning("Dropping span batch: %s", result.error) + return SpanExportResult.FAILURE + + export_result: Final = self._client.export(serialized_data) + if not export_result.success: + result.error = export_result.error + result.error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: export_result.status_code} + if export_result.status_code is not None + else None + ) + return SpanExportResult.FAILURE + + return SpanExportResult.SUCCESS diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 3befb73b0cc..141162bdfa5 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -76,6 +76,7 @@ def agentops_preset( def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter.""" + import requests from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, ) @@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """ def __init__(self, *, endpoint: str | None, api_key: str | None) -> None: - super().__init__(endpoint=endpoint) + self._agentops_session = requests.Session() + super().__init__(endpoint=endpoint, session=self._agentops_session) self._agentops_api_key = api_key self._auth_resolved = False @@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: try: token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token") if token: - # ``_session`` is the requests.Session the base exporter - # POSTs through; updating its Authorization header is how the - # minted JWT reaches every subsequent export. - self._session.headers["Authorization"] = f"Bearer {token}" + self._agentops_session.headers["Authorization"] = f"Bearer {token}" except Exception as e: verbose_logger.debug("AgentOps JWT fetch failed: %s", e) diff --git a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py index dab12d447df..46e633d2b96 100644 --- a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py +++ b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py @@ -3,8 +3,8 @@ Streaming iterator for transforming Responses API stream to Interactions API str """ from collections import deque -from collections.abc import AsyncIterator, Iterator -from typing import Any, Final, cast +from collections.abc import AsyncIterator, Iterator, Mapping +from typing import Final, cast from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, @@ -44,7 +44,7 @@ class LiteLLMResponsesInteractionsStreamingIterator: request_input: InteractionInput | None, optional_params: InteractionsAPIOptionalRequestParams, custom_llm_provider: str | None = None, - litellm_metadata: dict[str, Any] | None = None, + litellm_metadata: Mapping[str, object] | None = None, ): self.model = model self.responses_stream_iterator = litellm_custom_stream_wrapper diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 1f49cc2c4bb..e8da59cd33a 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -102,6 +102,7 @@ class ExceptionCheckers: "`inputs` tokens + `max_new_tokens` must be", "exceeds the available context size", # llama.cpp/Lemonade "exceeds the maximum number of tokens allowed", # Gemini + '"code": "context_length_exceeded"', ] for substring in known_exception_substrings: if substring in _error_str_lowercase: diff --git a/litellm/litellm_core_utils/get_blog_posts.py b/litellm/litellm_core_utils/get_blog_posts.py index c8bc184f972..c3b4114d5b9 100644 --- a/litellm/litellm_core_utils/get_blog_posts.py +++ b/litellm/litellm_core_utils/get_blog_posts.py @@ -19,6 +19,7 @@ import httpx from pydantic import ConfigDict from litellm import verbose_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.types.llms.base import LiteLLMBaseModel BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour @@ -129,7 +130,7 @@ class GetBlogPosts: Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS. Fetches from ``url`` otherwise, falling back to local backup on failure. """ - if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true": return cls.load_local_blog_posts() now: Final = time.time() diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a67c1bb7df8..3130d49a5e2 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info( if dynamic_api_key is None and api_key is not None: dynamic_api_key = api_key return model, custom_llm_provider, dynamic_api_key, api_base + + +def get_provider_affinity_header(api_base: str | None) -> str | None: + from litellm.llms.opencode.transformation import OpenCodeProviderAdapter + + return OpenCodeProviderAdapter().get_provider_affinity_header(api_base) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 7b3835dc1ac..1e2a511d439 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -39,6 +39,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.fallback_generalizations import ( set_fallback_generalizations, ) +from litellm.litellm_core_utils.offline_mode import is_offline_mode FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) @@ -456,7 +457,7 @@ async def refetch_model_cost_map( on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the map they already have. """ - if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": _cost_map_source_info.loaded_at = datetime.now(timezone.utc) _cost_map_source_info.source = "local" _cost_map_source_info.url = None @@ -740,7 +741,7 @@ def get_model_cost_map( _cost_map_source_info.loaded_at = datetime.now(timezone.utc) # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. - if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): + if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d75d35e4531..41f13817d75 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -161,6 +161,7 @@ from litellm.types.utils import ( CachingDetails, CallTypes, CostBreakdown, + CostPerToken, CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, EmbeddingResponse, @@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass): prometheusLogger, \ slack_app custom_pricing: bool = False + custom_cost_per_token: CostPerToken | None = None + custom_cost_per_second: float | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass): "call_type": self.call_type, "optional_params": self.optional_params, "custom_pricing": custom_pricing, + "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, + "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, diff --git a/litellm/litellm_core_utils/offline_mode.py b/litellm/litellm_core_utils/offline_mode.py new file mode 100644 index 00000000000..7fdcae8825f --- /dev/null +++ b/litellm/litellm_core_utils/offline_mode.py @@ -0,0 +1,7 @@ +import os + + +def is_offline_mode() -> bool: + """True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather + than from a destination the operator configured, must check this and use its bundled copy or skip instead""" + return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true" diff --git a/litellm/litellm_core_utils/provider_affinity.py b/litellm/litellm_core_utils/provider_affinity.py index 02c4cd69159..e022839c5c7 100644 --- a/litellm/litellm_core_utils/provider_affinity.py +++ b/litellm/litellm_core_utils/provider_affinity.py @@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None: return getattr(value, key, None) -def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None: +def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None: header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None if header is None: - return None + return default_header if not isinstance(header, str): raise TypeError("provider_affinity_header must be a string") return validate_provider_affinity_header_name(header) @@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None: def add_provider_affinity_header( - headers: Mapping[str, object], litellm_params: object | None + headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None ) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers - header_name: Final = _get_provider_affinity_header_name(litellm_params) + header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header) if header_name is None or any(key.lower() == header_name.lower() for key in headers): return dict(headers) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 9f3aa0130d1..dfe0788e50a 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool: _messages_carry_output_config = messages_carry_output_config +def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse: + if "stop_sequence" not in response: + response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place + return response + + class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True @@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> AnthropicMessagesResponse: - """ - No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec - """ try: raw_response_json: Final = raw_response.json() except Exception: raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code) - return AnthropicMessagesResponse(**raw_response_json) + response: Final = AnthropicMessagesResponse(**raw_response_json) + return _ensure_stop_sequence(response) def get_async_streaming_response_iterator( self, diff --git a/litellm/llms/aws_polly/text_to_speech/transformation.py b/litellm/llms/aws_polly/text_to_speech/transformation.py index 01fec02c310..325c9fd364d 100644 --- a/litellm/llms/aws_polly/text_to_speech/transformation.py +++ b/litellm/llms/aws_polly/text_to_speech/transformation.py @@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM): else: mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT - # Extract engine from model name (e.g., "aws_polly/neural" -> "neural") + # model arrives without the "aws_polly/" prefix, e.g. "standard" engine: Final = self._extract_engine_from_model(model) mapped_params["engine"] = engine @@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM): """ Extract engine from model name. + speech() strips the "aws_polly/" prefix before the config is called, + so the bare engine name is what normally arrives here. + Examples: - - aws_polly/neural -> neural - - aws_polly/standard -> standard + - standard -> standard - aws_polly/long-form -> long-form - aws_polly -> neural (default) """ - if "/" in model: - parts: Final = model.split("/") - if len(parts) >= 2: - engine: Final = parts[1].lower() - if engine in self.VALID_ENGINES: - return engine + engine: Final = model.rsplit("/", 1)[-1].lower() + if engine in self.VALID_ENGINES: + return engine return self.DEFAULT_ENGINE def validate_environment( diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 74e65572809..8cae4bf284b 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -3,6 +3,7 @@ from collections.abc import Mapping from copy import deepcopy from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal +from urllib.parse import urlsplit import httpx from openai.types.responses import ResponseReasoningItem @@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base) + responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params) + parsed_base: Final = urlsplit(resolved_api_base or "") + path: Final = parsed_base.path.rstrip("/") + if path.endswith(("/openai/v1", "/openai/v1/responses")): + responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path + return BaseAzureLLM.get_base_azure_url( + api_base=parsed_base._replace(path=responses_path).geturl(), + litellm_params=responses_params, + route="/openai/v1/responses", + default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, + ) + return BaseAzureLLM.get_base_azure_url( api_base=resolved_api_base, - litellm_params=_with_responses_api_version(resolved_api_base, litellm_params), + litellm_params=responses_params, route="/openai/responses", default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, ) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 0fa54cccbc9..83d00a621b4 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -12,6 +12,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( audio_or_image_in_message_content, convert_content_list_to_str, + drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, ) from litellm.llms.azure.common_utils import BaseAzureLLM @@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig): Operates on a deep copy so the caller's messages keep their thinking blocks and provider metadata, which a fallback to another provider still needs. """ - stripped_messages: Final = copy.deepcopy(messages) + stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages)) for message in stripped_messages: message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index 97d402e7a1b..171b193f60c 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -4,7 +4,7 @@ Base OCR types shared by the Rust OCR route and Python consumers. import builtins from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import PrivateAttr @@ -45,7 +45,7 @@ class OCRPageImage(LiteLLMPydanticObjectBase): """Image extracted from OCR page.""" image_base64: str | None = None - bbox: dict[str, Any] | None = None + bbox: dict[str, builtins.object] | None = None model_config = {"extra": "allow"} @@ -80,7 +80,7 @@ class OCRResponse(LiteLLMPydanticObjectBase): pages: list[OCRPage] model: str - document_annotation: Any | None = None + document_annotation: builtins.object | None = None usage_info: OCRUsageInfo | None = None content: str | None = None tables: list[dict[str, builtins.object]] | None = None diff --git a/litellm/llms/base_llm/sandbox/transformation.py b/litellm/llms/base_llm/sandbox/transformation.py index 877edc15ec7..fe0e3abb87e 100644 --- a/litellm/llms/base_llm/sandbox/transformation.py +++ b/litellm/llms/base_llm/sandbox/transformation.py @@ -7,7 +7,7 @@ run code -> delete container; `code_interpreter_tool` combines all three. """ import builtins -from typing import Any, Final +from typing import Final import httpx from pydantic import Field, PrivateAttr @@ -42,8 +42,8 @@ class CodeExecutionResult(LiteLLMPydanticObjectBase): stdout: str = "" stderr: str = "" - results: list[dict[str, Any]] = Field(default_factory=list) - error: dict[str, Any] | None = None + results: list[dict[str, object]] = Field(default_factory=list) + error: dict[str, object] | None = None execution_count: int | None = None object: str = "code_execution" diff --git a/litellm/llms/bedrock/count_tokens/kimi_k3.py b/litellm/llms/bedrock/count_tokens/kimi_k3.py index 5e3fd56e4ba..4f8b4624cd8 100644 --- a/litellm/llms/bedrock/count_tokens/kimi_k3.py +++ b/litellm/llms/bedrock/count_tokens/kimi_k3.py @@ -27,6 +27,7 @@ from typing_extensions import Self import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.prompt_templates.factory import ( chat_completion_tools_for, is_flat_responses_tool, @@ -67,12 +68,14 @@ class KimiK3TokenizerSource: def _hugging_face_ranks_path() -> str: import huggingface_hub - return huggingface_hub.hf_hub_download( - repo_id=KIMI_K3_REPO, - filename=KIMI_K3_RANKS_FILE, - revision=KIMI_K3_REVISION, - local_files_only=litellm.disable_hf_tokenizer_download is True, - ) + if litellm.disable_hf_tokenizer_download is True or is_offline_mode(): + cached: Final = huggingface_hub.try_to_load_from_cache( + repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION + ) + if not isinstance(cached, str): + raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache") + return cached + return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION) HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a7ca0d5f49e..a692f3f964e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.realtime_errors import ( close_after_upstream_handshake_refusal, realtime_error_event, @@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import ( ) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base +from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -2435,13 +2438,13 @@ class BaseLLMHTTPHandler: # Also check for extra_headers in kwargs (from config or direct calls) extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None) # Merge all header sources: forwarded < extra_headers < provider_specific - merged_headers: Final = {} - if forwarded_headers: - merged_headers.update(forwarded_headers) - if extra_headers_from_kwargs: - merged_headers.update(extra_headers_from_kwargs) - if provider_specific_headers: - merged_headers.update(provider_specific_headers) + merged_headers: Final[Mapping[str, object]] = MappingProxyType( + { + **(forwarded_headers or {}), + **(extra_headers_from_kwargs or {}), + **(provider_specific_headers or {}), + } + ) request_params: Final = reduce( delete_nested_value, litellm_params.get("additional_drop_params") or (), @@ -2452,11 +2455,19 @@ class BaseLLMHTTPHandler: and anthropic_messages_provider_config.should_filter_anthropic_beta_headers() ): await aload_beta_headers_config() + affinity_api_base: Final = ( + AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) ( headers, api_base, ) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment( - headers=merged_headers or {}, + headers=( + add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header) + if default_affinity_header is not None + else dict(merged_headers) + ), model=model, messages=messages, optional_params=request_params, diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index f16564ba505..4f308b01159 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -202,7 +202,9 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): optional_params[param] = value return optional_params - def get_error_class(self, error_message: str, status_code: int, headers: Any) -> BaseLLMException: + def get_error_class( + self, error_message: str, status_code: int, headers: dict[str, str] | httpx.Headers + ) -> BaseLLMException: from litellm.llms.openai.openai import OpenAIConfig return OpenAIConfig().get_error_class(error_message=error_message, status_code=status_code, headers=headers) diff --git a/litellm/llms/ollama/completion/handler.py b/litellm/llms/ollama/completion/handler.py index d4110e55bf0..0c13b1cd1d3 100644 --- a/litellm/llms/ollama/completion/handler.py +++ b/litellm/llms/ollama/completion/handler.py @@ -87,7 +87,7 @@ async def ollama_aembeddings( model: str, prompts: list[str], model_response: EmbeddingResponse, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, encoding: TokenEncoder | None, ): @@ -110,7 +110,7 @@ def ollama_embeddings( api_base: str, model: str, prompts: list[str], - optional_params: dict, + optional_params: Mapping[str, object], model_response: EmbeddingResponse, logging_obj: object, encoding: TokenEncoder | None = None, diff --git a/litellm/llms/opencode/common_utils.py b/litellm/llms/opencode/common_utils.py new file mode 100644 index 00000000000..65a96411e52 --- /dev/null +++ b/litellm/llms/opencode/common_utils.py @@ -0,0 +1,10 @@ +import re + + +def is_opencode_api_base(api_base: str | None) -> bool: + return bool( + api_base + and re.fullmatch( + r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base + ) + ) diff --git a/litellm/llms/opencode/transformation.py b/litellm/llms/opencode/transformation.py new file mode 100644 index 00000000000..9b62e7aca95 --- /dev/null +++ b/litellm/llms/opencode/transformation.py @@ -0,0 +1,6 @@ +from litellm.llms.opencode.common_utils import is_opencode_api_base + + +class OpenCodeProviderAdapter: + def get_provider_affinity_header(self, api_base: str | None) -> str | None: + return "x-opencode-session" if is_opencode_api_base(api_base) else None diff --git a/litellm/main.py b/litellm/main.py index 27700cf9def..327a7a1c4b5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import ( with_control_options, ) from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_provider_affinity_header, invalid_provider_routing_hints_message, ) from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( VertexAIModelRoute, @@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import ( resolve_chat_reasoning, ) from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper from litellm.types.completion import ( CompletionDispatchContext, _CompletionDispatchResult, @@ -169,7 +172,6 @@ from litellm.utils import ( ProviderConfigManager, TextChoices, TextCompletionResponse, - TextCompletionStreamWrapper, TranscriptionResponse, Usage, add_provider_specific_params_to_optional_params, @@ -5990,19 +5992,28 @@ def completion( }, ) litellm_params: Final = with_control_options(requested_litellm_params, control_options) - if litellm_params.get("provider_affinity_header") is not None: + affinity_api_base: Final = ( + str(client.base_url) + if isinstance(client, openai.OpenAI | openai.AsyncOpenAI) + else get_openai_credentials(api_base=api_base).api_base + if custom_llm_provider == "openai" + else api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) + if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None: try: headers = add_provider_affinity_header( headers=headers or litellm.headers or MappingProxyType({}), litellm_params=MappingProxyType( { - "provider_affinity_header": litellm_params["provider_affinity_header"], + "provider_affinity_header": litellm_params.get("provider_affinity_header"), "litellm_session_id": kwargs.get("litellm_session_id"), "session_id": kwargs.get("session_id"), "metadata": metadata, "litellm_metadata": kwargs.get("litellm_metadata"), } ), + default_header=default_affinity_header, ) except ValueError as affinity_error: raise litellm.BadRequestError( diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index cdb9a925403..2e9d6a77a62 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target: _FEATURES: Final[detectors.Detectors] = MappingProxyType( { **detectors.COMMON, + Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity, Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, Feature.RESPONSE_CACHE: detectors.response_cache, Feature.MODIFY_PARAMS: detectors.modify_params, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb9bfa6ffe4..14fe355a805 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12827,6 +12827,7 @@ "supports_vision": true }, "azure_ai/Phi-4": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -12840,6 +12841,7 @@ "supports_vision": false }, "azure_ai/Phi-4-mini-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12851,6 +12853,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -12865,6 +12868,7 @@ "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12876,6 +12880,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -24892,27 +24897,118 @@ ] }, "elevenlabs/eleven_v3": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] }, "elevenlabs/eleven_multilingual_v2": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4": { + "input_cost_per_character": 8e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4_turbo": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v3_conversational": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] @@ -36861,17 +36957,17 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, - "cache_creation_input_token_cost_decisions": 1e-07, + "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, - "cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07, + "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-08, "cache_creation_input_token_cost_priority": 2.5e-07, "cache_read_input_token_cost": 1e-08, - "cache_read_input_token_cost_decisions": 1e-07, + "cache_read_input_token_cost_decisions": 1e-08, "cache_read_input_token_cost_above_272k_tokens": 2e-08, - "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, "cache_read_input_token_cost_batches": 5e-09, @@ -42337,6 +42433,134 @@ ], "supports_audio_output": true }, + "mistral/voxtral-mini-transcribe-3": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-3-0": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3-0": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3-0": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-3": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-mini-tts-3-0": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "mistral/voxtral-small-2507": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -45623,6 +45847,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -45648,6 +45873,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -48499,6 +48725,17 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.together.ai/pricing", + "supports_prompt_caching": true + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -49352,6 +49589,7 @@ "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49371,6 +49609,7 @@ "supports_tool_choice": true }, "us-gov.openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -52906,12 +53145,14 @@ }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, + "output_cost_per_token_batches": 5.75e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -52926,12 +53167,14 @@ }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, + "output_cost_per_token_batches": 3.5e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -59917,6 +60160,7 @@ } }, "bedrock_mantle/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -59942,6 +60186,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -59967,6 +60212,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7e-08, "input_cost_per_token_flex": 7e-08, @@ -59990,6 +60236,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -67841,6 +68088,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -67860,6 +68108,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68138,6 +68387,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68157,6 +68407,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68545,6 +68796,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68564,6 +68816,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -68704,6 +68957,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68723,6 +68977,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -73330,6 +73585,7 @@ ] }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py index 435db632026..b2bfb572b7d 100644 --- a/litellm/models/managed_files.py +++ b/litellm/models/managed_files.py @@ -6,7 +6,7 @@ Canonical definitions for the ``litellm_managed*`` tables. Re-exported from """ from datetime import datetime -from typing import Any, Literal +from typing import Literal from litellm.types.llms.base import LiteLLMPydanticObjectBase from litellm.types.llms.openai import OpenAIFileObject, ResponsesAPIResponse @@ -39,7 +39,7 @@ class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): """Table for managing vector stores with target_model_names support.""" unified_resource_id: str - resource_object: Any | None = None + resource_object: object | None = None model_mappings: dict[str, str] flat_model_resource_ids: list[str] created_by: str | None = None @@ -54,11 +54,11 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase): custom_llm_provider: str vector_store_name: str | None = None vector_store_description: str | None = None - vector_store_metadata: dict[str, Any] | None = None + vector_store_metadata: dict[str, object] | None = None created_at: datetime | None = None updated_at: datetime | None = None litellm_credential_name: str | None = None - litellm_params: dict[str, Any] | None = None + litellm_params: dict[str, object] | None = None team_id: str | None = None user_id: str | None = None is_config: bool = False diff --git a/litellm/proxy/_experimental/out/assets/aria-agent.png b/litellm/proxy/_experimental/out/assets/aria-agent.png new file mode 100644 index 00000000000..870773933e3 Binary files /dev/null and b/litellm/proxy/_experimental/out/assets/aria-agent.png differ diff --git a/litellm/proxy/_experimental/out/assets/logos/claude.svg b/litellm/proxy/_experimental/out/assets/logos/claude.svg new file mode 100644 index 00000000000..ede30652b25 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/claude.svg @@ -0,0 +1,27 @@ + +Claude diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 0e2eed73364..28c701e0fc2 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.spend_tracking.vantage_endpoints", path_prefixes=("/vantage",), ), - LazyFeature( - name="usage_ai", - module_path="litellm.proxy.management_endpoints.usage_endpoints", - path_prefixes=("/usage/ai",), - ), LazyFeature( name="prompts", module_path="litellm.proxy.prompts.prompt_endpoints", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4572e59944b..0dd978ee855 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11407,7 +11407,7 @@ }, "/systemone": { "post": { - "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", "requestBody": { "content": { @@ -11813,7 +11813,6 @@ ] }, "description": "Named noul, choice and score questions. Each key becomes a key in the answers", - "maxProperties": 128, "minProperties": 1, "propertyNames": { "minLength": 1 @@ -12814,7 +12813,7 @@ }, "/v1/systemone": { "post": { - "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", + "description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", "requestBody": { "content": { @@ -13220,7 +13219,6 @@ ] }, "description": "Named noul, choice and score questions. Each key becomes a key in the answers", - "maxProperties": 128, "minProperties": 1, "propertyNames": { "minLength": 1 @@ -57900,163 +57898,6 @@ } } }, - "usage_ai": { - "components": { - "schemas": { - "ChatMessage": { - "properties": { - "content": { - "title": "Content", - "type": "string" - }, - "role": { - "enum": [ - "user", - "assistant" - ], - "title": "Role", - "type": "string" - } - }, - "required": [ - "role", - "content" - ], - "title": "ChatMessage", - "type": "object" - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "title": "Detail", - "type": "array" - } - }, - "title": "HTTPValidationError", - "type": "object" - }, - "UsageAIChatRequest": { - "properties": { - "messages": { - "description": "Chat messages (user/assistant history)", - "items": { - "$ref": "#/components/schemas/ChatMessage" - }, - "title": "Messages", - "type": "array" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Model to use for AI chat", - "title": "Model" - } - }, - "required": [ - "messages" - ], - "title": "UsageAIChatRequest", - "type": "object" - }, - "ValidationError": { - "properties": { - "ctx": { - "title": "Context", - "type": "object" - }, - "input": { - "title": "Input" - }, - "loc": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "title": "Location", - "type": "array" - }, - "msg": { - "title": "Message", - "type": "string" - }, - "type": { - "title": "Error Type", - "type": "string" - } - }, - "required": [ - "loc", - "msg", - "type" - ], - "title": "ValidationError", - "type": "object" - } - } - }, - "paths": { - "/usage/ai/chat": { - "post": { - "description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.", - "operationId": "usage_ai_chat_usage_ai_chat_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UsageAIChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Usage Ai Chat", - "tags": [ - "usage_ai" - ] - } - } - } - }, "vantage": { "components": { "schemas": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 54acf59ef80..28d3c08bb31 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2717,6 +2717,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "hidden model can still be called." ), ) + return_wildcard_routes: bool | None = Field( + None, + description=( + "When true, `/v1/models` lists wildcard routes like `openai/*` unless the request passes " + "`return_wildcard_routes=false`." + ), + ) alerting: list | None = Field( None, description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL", diff --git a/litellm/proxy/a2a/endpoints.py b/litellm/proxy/a2a/endpoints.py index 56b512df934..730d40b46d0 100644 --- a/litellm/proxy/a2a/endpoints.py +++ b/litellm/proxy/a2a/endpoints.py @@ -9,7 +9,7 @@ admin pick which ones to expose through the proxy. The actual merge into a LiteLLM-fronted card happens when the agent is saved via ``POST /v1/agents``. """ -from typing import Any, Final +from typing import Final from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import JSONResponse @@ -50,7 +50,7 @@ class DiscoverAgentRequest(LiteLLMBaseModel): "query parameter." ), ) - params: dict[str, Any] | None = Field( + params: dict[str, object] | None = Field( default=None, description=( "Mode-specific parameters. ``langgraph_platform`` requires " @@ -61,7 +61,7 @@ class DiscoverAgentRequest(LiteLLMBaseModel): class DiscoverAgentResponse(LiteLLMBaseModel): url: str - agent_card: dict[str, Any] + agent_card: dict[str, object] @router.post( diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index ae473a62271..f6637a1fdf3 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -41,7 +41,7 @@ DB_UNAVAILABLE_FALLBACK_USER_ID: Final = "__db_unavailable_fallback__" if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py index 36a569bf18f..d810ba4238b 100644 --- a/litellm/proxy/auth/password_policy.py +++ b/litellm/proxy/auth/password_policy.py @@ -21,6 +21,7 @@ from typing import Final from litellm._logging import verbose_proxy_logger from litellm._version import version from litellm.constants import HIBP_RANGE_API_BASE +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.types.llms.custom_http import httpxSpecialProvider @@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool: - return general_settings.get("password_policy_check_breached_passwords", True) is not False + return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False async def is_password_breached( diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 4e98823ba77..be5b307b5ae 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,6 @@ import json import os import random import time -import traceback from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar @@ -90,6 +89,7 @@ from litellm.proxy.spend_tracking.savings import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable +from litellm.responses.streaming_iterator import spawn_logging_task from litellm.types.utils import CallTypes if TYPE_CHECKING: @@ -313,6 +313,38 @@ def _spend_tables_left_to_send( return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None +def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None: + spend_log_error( + "Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s", + step_name, + payload.get("request_id"), + payload.get("api_key"), + exc=exc, + ) + + +_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata + + +def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata: + """Malformed metadata costs the request its metadata-derived fields, never its daily rollup row.""" + raw: Final = payload.get("metadata") + if not isinstance(raw, str): + return _EMPTY_SPEND_LOG_METADATA + try: + parsed: Final[SpendLogsMetadata] = json.loads(raw) + except json.JSONDecodeError as e: + spend_log_error( + "Spend tracking - unreadable spend log metadata, rollups for this request are written without " + "metadata-derived fields. request_id=%s api_key=%s", + payload.get("request_id"), + payload.get("api_key"), + exc=e, + ) + return _EMPTY_SPEND_LOG_METADATA + return parsed + + def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool: """An earlier billed retry attempt's own spend row, not a request the caller sent.""" return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True @@ -647,8 +679,8 @@ class DBSpendUpdateWriter: "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur." ) - # Single task replaces 11 create_task() calls - asyncio.create_task( + spawn_logging_task( + asyncio.get_running_loop(), self._batch_database_updates( response_cost=response_cost, user_id=user_id, @@ -661,7 +693,8 @@ class DBSpendUpdateWriter: litellm_proxy_budget_name=litellm_proxy_budget_name, payload=payload, request_model_access_groups=get_request_model_access_groups(kwargs), - ) + ), + task_name="spend rollups", ) self._enqueue_tool_registry_upsert( @@ -1086,91 +1119,52 @@ class DBSpendUpdateWriter: project_id: str | None = None, ): """ - Runs all 13 spend-update helpers sequentially inside a single asyncio task. - - Each helper is wrapped in try/except so one failure doesn't prevent the others. - - The deepcopy runs here, off the awaited request path, so the daily spend - helpers get a payload isolated from the spend-log queue entry and the caller. + Runs every spend-update helper for one request. Each helper goes through + _run_spend_update_step so one failure never skips the sibling writes. The deepcopy + isolates the daily spend helpers' payload from the spend-log queue entry and the caller. """ payload_copy: Final = copy.deepcopy(payload) request_tags: Final = payload_copy.get("request_tags") - try: - await self._update_user_db( + await self._run_spend_update_step( + "_update_user_db", + self._update_user_db( response_cost=response_cost, user_id=user_id, prisma_client=prisma_client, litellm_proxy_budget_name=litellm_proxy_budget_name, end_user_id=end_user_id, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_user_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_key_db( - response_cost=response_cost, - hashed_token=hashed_token, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_key_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_team_db( - response_cost=response_cost, - team_id=team_id, - user_id=user_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_team_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_org_db( - response_cost=response_cost, - org_id=org_id, - user_id=user_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_org_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_project_db( - response_cost=response_cost, - project_id=project_id, - prisma_client=prisma_client, - ) - except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes - verbose_proxy_logger.debug( - "_batch_database_updates: _update_project_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_tag_db( - response_cost=response_cost, - request_tags=request_tags, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_tag_db failed: %s", - traceback.format_exc(), - ) - + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_key_db", + self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_team_db", + self._update_team_db( + response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_org_db", + self._update_org_db( + response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_project_db", + self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_tag_db", + self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client), + payload=payload_copy, + ) await self._update_model_access_group_db( response_cost=response_cost, request_model_access_groups=request_model_access_groups, @@ -1178,96 +1172,64 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, router=get_llm_router(), ) - - _agent_id_for_spend: Final = payload_copy.get("agent_id") - try: - await self._update_agent_db( - response_cost=response_cost, - agent_id=_agent_id_for_spend, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_agent_db failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_user_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_end_user_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_agent_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_team_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s", - traceback.format_exc(), - ) + await self._run_spend_update_step( + "_update_agent_db", + self._update_agent_db( + response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_user_transaction", + self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_end_user_transaction", + self.add_spend_log_transaction_to_daily_end_user_transaction( + payload=payload_copy, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_agent_transaction", + self.add_spend_log_transaction_to_daily_agent_transaction( + payload=payload_copy, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_team_transaction", + self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) try: self._record_request_error(payload=payload_copy, prisma_client=prisma_client) - except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes - verbose_proxy_logger.debug( - "_batch_database_updates: _record_request_error failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_org_transaction( - payload=payload_copy, - org_id=org_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_tag_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s", - traceback.format_exc(), - ) - + except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes + _report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_org_transaction", + self.add_spend_log_transaction_to_daily_org_transaction( + payload=payload_copy, org_id=org_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_tag_transaction", + self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client) + @staticmethod + async def _run_spend_update_step( + step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload + ) -> None: + try: + await step + except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes + _report_spend_update_step_failure(step_name, payload=payload, exc=e) + async def _update_key_db( self, response_cost: float | None, @@ -2810,7 +2772,7 @@ class DBSpendUpdateWriter: date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "") if not date: return - metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + metadata: Final = _parse_spend_log_metadata(payload) internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults request_error_accumulator.record( payload=payload, @@ -2861,7 +2823,7 @@ class DBSpendUpdateWriter: # spend, token and per-entity columns written here stay either way. request_status: Final = prisma_client.get_request_status(payload) verbose_proxy_logger.debug("Logged request status: %s", request_status) - _metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + _metadata: Final = _parse_spend_log_metadata(payload) usage_obj: Final = _metadata.get("usage_object", {}) or {} if isinstance(payload["startTime"], datetime): start_time: Final = payload["startTime"].isoformat() diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 366842d23a6..a8684dd9757 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -228,7 +228,7 @@ async def systemone( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): """ - Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + Judge a `state` against one or more named `questions` and get one calibrated answer per question name. Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with `confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8a45f658f36..8dc7501bc3b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = ( "/v1/messages", "responses", "files", + "interactions", ) LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( @@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( LlmProviders.VERTEX_AI.value, ) ) +_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( + (LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value) +) _ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS)) @@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request( custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers ), ProviderSpecificHeader( - custom_llm_provider=LlmProviders.DATABRICKS.value, + custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers, native_anthropic_messages_only=True, ), diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index b563dbc5c4c..d6bf9220551 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2093,7 +2093,9 @@ async def get_user_key_counts( return result -def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None: +def _validate_sort_params( + sort_by: str | None, sort_order: str +) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None: order_by: Final[dict[str, str]] = {} if sort_by is None: @@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str "spend", "user_alias", "user_role", + "max_budget", + "updated_at", + "sso_user_id", + "status", + "key_count", ] if sort_by not in valid_columns: raise HTTPException( @@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"}, ) + if sort_by in {"status", "key_count"}: + return None + if sort_by in {"max_budget", "updated_at", "sso_user_id"}: + nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last" + return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"}) order_by[sort_by] = sort_order.lower() return order_by @@ -2338,16 +2350,40 @@ async def get_users( # Build order_by conditions - order_by: Final[dict[str, str] | None] = ( + order_by: Final = ( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None ) - users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many( - where=where, - skip=skip, - take=page_size, - order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified + from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids + + computed_sort: Final = sort_by in ("status", "key_count") + sorted_ids: Final = ( + await sorted_user_ids( + prisma_client, + sort_by="status" if sort_by == "status" else "key_count", + descending=sort_order.lower() == "desc", + skip=skip, + take=page_size, + role=role, + user_ids=user_ids, + user_email=user_email, + team=team, + sso_user_ids=sso_user_ids, + organization_ids=organization_ids, + search=search, + ) + if computed_sort + else () ) + page_users: Final = await UserRepository(prisma_client).table.find_many( + # bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route + where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where, + skip=0 if computed_sort else skip, + take=page_size, + order=order_by if order_by else {"created_at": "desc"}, + ) + positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)} + users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users # Get total count of user rows total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 824b310a1dd..eebc537509b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -7397,20 +7397,26 @@ async def key_aliases( ) -def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None: - order_by: Final[dict[str, str]] = {} - +def _validate_sort_params( + sort_by: str | None, sort_order: str +) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None: if sort_by is None: return None # Validate sort_by is a valid column - valid_columns: Final = [ + valid_columns: Final = ( "spend", "max_budget", "created_at", "updated_at", "token", "key_alias", - ] + "expires", + "last_active", + "budget_reset_at", + "total_spend", + "tpm_limit", + "rpm_limit", + ) if sort_by not in valid_columns: raise HTTPException( status_code=400, @@ -7424,9 +7430,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"}, ) - order_by[sort_by] = sort_order.lower() - - return order_by + direction: Final = sort_order.lower() + nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit") + if sort_by not in (*nullable_sort_columns, "total_spend"): + return {sort_by: direction} + return [ + {sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction}, + {"token": "asc"}, + ] def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]: @@ -7668,7 +7679,7 @@ async def list_key_helper( verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size) - order_by: Final[dict[str, str] | None] = ( + order_by: Final = ( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None ) diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 1970a0797ef..19a5408496a 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime +from itertools import chain from types import MappingProxyType from typing import Annotated, Final @@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...]) SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields) +# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt. +_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰０𐒠𐴰𐵀𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑛐𑛚𑜰𑣠𑥐𑯰𑱐𑵐𑶠\U00011de0𑽐𖄰𖩠𖫀𖭐𖵰𜳰𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞗱𞥐🯰" +_DECIMAL_DIGITS: Final = "".join( + chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES) +) +_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES) + + +_BUDGET_DURATION_SECONDS: Final = f"""( + SELECT EXTRACT(EPOCH FROM ( + (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + + (parts[1] || ' ' || CASE parts[2] + WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours' + WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval + - (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + )) FROM regexp_match( + CASE lower(trim(budget_duration)) + WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h' + WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d' + ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration + ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END, + '^([0-9]+)(mo|[smhdw])' + ) AS duration(parts) +)""" + + @dataclass(frozen=True, slots=True) class PrismaBudgetListExecutor: """The database half of the budgets list. Every caller-supplied value is bound to a @@ -91,10 +118,16 @@ class PrismaBudgetListExecutor: async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]: clauses, params = where_sql(plan.where) + ordering: Final = ", ".join( + order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST") + if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit") + else order_by_sql((key,)) + for key in plan.order + ).replace('"budget_duration"', _BUDGET_DURATION_SECONDS) sql: Final = ( f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}" + (f" WHERE {clauses}" if clauses else "") - + f" ORDER BY {order_by_sql(plan.order)}" + + f" ORDER BY {ordering}" + f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}" ) return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip)) @@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope: return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}") -# budget_duration is deliberately absent from `sortable`: the column holds strings -# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d". BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( { "budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))), @@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec( resource="budgets", - sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")), + sortable=frozenset( + ("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration") + ), searchable=frozenset(("budget_id",)), filters=BUDGET_FILTERS, default_sort=(SortKey(field="created_at", descending=True),), @@ -154,8 +187,9 @@ async def list_budgets( way to page, sort or filter it. `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, - `rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending, - and defaults to `-created_at`. `budget_id` is appended to every sort as the + `rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending, + and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months + measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the tiebreaker. `q` is a case-insensitive substring match on `budget_id`. `page_size` defaults to 50 and is capped at 100. Filters are `filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]` diff --git a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py index 3bef7c107eb..7cd36d51511 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py @@ -28,6 +28,7 @@ from litellm.constants import ( MAX_COMPETITOR_NAMES, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) @@ -657,7 +658,7 @@ async def get_policy_templates( Fetches from GitHub with automatic fallback to local backup on failure. Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only. """ - use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in ( + use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in ( "true", "1", "yes", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index d9763f72741..c7adae2ce8d 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -6109,13 +6109,52 @@ async def list_team_v2( "total_pages": 0, } - # Build order_by conditions - valid_sort_columns: Final = ["team_id", "team_alias", "created_at"] - order_by = None - if sort_by and sort_by in valid_sort_columns: - if sort_order.lower() not in ["asc", "desc"]: - sort_order = "asc" - order_by = {sort_by: sort_order.lower()} + from litellm.proxy.management_helpers.team_list_sorting import ( + COMPUTED_TEAM_SORTS, + UnsupportedTeamFilter, + sorted_team_ids, + ) + + valid_sort_columns: Final = ( + "team_id", + "team_alias", + "created_at", + "spend", + "max_budget", + "tpm_limit", + "rpm_limit", + "updated_at", + ) + sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc" + nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit") + order_by: Final = ( + ( + {sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}}, + {"team_id": "asc"}, + ) + if nullable_sort + else ({sort_by: sort_direction}, {"team_id": "asc"}) + if sort_by in ("spend", "updated_at") + else {sort_by: sort_direction} + if sort_by in valid_sort_columns + else None + ) + computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS + sorted_ids: Final = ( + await sorted_team_ids( + prisma_client, + where=where_conditions, + sort_by=sort_by or "", + descending=sort_direction == "desc", + skip=skip, + take=page_size, + ) + if computed_sort + else () + ) + + if isinstance(sorted_ids, UnsupportedTeamFilter): + raise HTTPException(status_code=500, detail="Unsupported team sorting filter") # Get teams with pagination if use_deleted_table: @@ -6130,26 +6169,32 @@ async def list_team_v2( total_count = await _deleted_team_db(prisma_client).count(where=where_conditions) else: teams = await _team_db(prisma_client).find_many( - where=where_conditions, - skip=skip, + # bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route + where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions, + skip=0 if computed_sort else skip, take=page_size, - order=order_by if order_by else {"created_at": "desc"}, # Default sort + order=order_by if order_by else {"created_at": "desc"}, include=_INCLUDE_MODEL_TABLE, ) # Get total count for pagination total_count = await _team_db(prisma_client).count(where=where_conditions) + positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)} + sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams + # Calculate total pages total_pages: Final = -(-total_count // page_size) # Ceiling division # Aggregate virtual-key counts per team for the current page. The deleted # table does not carry keys_count, so it is skipped. - keys_count_by_team: dict[str, int] = {} - if not use_deleted_table: - keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams) + keys_count_by_team: Final = ( + await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {} + ) # Convert Prisma models to response models with members_count and keys_count - team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team) + team_list: Final = _convert_teams_to_response_models( + sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team + ) # Resolve resources inherited from access groups (single batch query) if not use_deleted_table: diff --git a/litellm/proxy/management_endpoints/types.py b/litellm/proxy/management_endpoints/types.py index 4414eed97b2..221a6221fab 100644 --- a/litellm/proxy/management_endpoints/types.py +++ b/litellm/proxy/management_endpoints/types.py @@ -5,7 +5,7 @@ Might include fastapi/proxy requirements.txt related imports """ from collections.abc import Iterable, Sequence -from typing import Any, Final, cast +from typing import Final, cast from fastapi_sso.sso.base import OpenID @@ -94,4 +94,4 @@ def get_litellm_user_role(role_str: object) -> LitellmUserRoles | None: class CustomOpenID(OpenID): team_ids: list[str] user_role: LitellmUserRoles | None = None - extra_fields: dict[str, Any] | None = None + extra_fields: dict[str, object] | None = None diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 19bca3d2bf8..fd5eb672811 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -152,7 +152,7 @@ from litellm.proxy.utils import ( from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import SSOConfigRepository from litellm.repositories.team_repository import TeamRepository -from litellm.repositories.user_repository import UserRepository +from litellm.repositories.user_repository import UserRepository, free_sso_user_creation from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403 @@ -1074,25 +1074,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token( ) -async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None: - """Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license.""" - if premium_user is True: - return - if prisma_client is None: - raise ProxyException( - message=CommonProxyErrors.db_not_connected_error.value, - type=ProxyErrorTypes.auth_error, - param="premium_user", - code=status.HTTP_403_FORBIDDEN, - ) - billable_users: Final = await UserRepository(prisma_client).count_billable_users() - if billable_users and billable_users > 5: - raise ProxyException( - message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it", - type=ProxyErrorTypes.auth_error, - param="premium_user", - code=status.HTTP_403_FORBIDDEN, - ) +class _SSOUserAdmissionError(ProxyException): + pass @router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False) @@ -1112,8 +1095,6 @@ async def google_login( from litellm.proxy.proxy_server import ( cli_sso_session_cache, general_settings, - premium_user, - prisma_client, saml_sso_cache, user_custom_ui_sso_sign_in_handler, ) @@ -1128,15 +1109,6 @@ async def google_login( if admin_ui_is_disabled and not is_cli_sso_login: return admin_ui_disabled() - ####### Check if user is a Enterprise / Premium User ####### - if ( - microsoft_client_id is not None - or google_client_id is not None - or generic_client_id is not None - or SAMLAuthHandler.is_saml_configured() - ): - await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client) - ####### Detect DB + MASTER KEY in .env ####### missing_env_vars: Final = show_missing_vars_in_env() if missing_env_vars is not None: @@ -1963,6 +1935,8 @@ async def get_user_info_from_db( ) return user_info + except _SSOUserAdmissionError: + raise except Exception as e: if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e) @@ -2328,7 +2302,6 @@ async def saml_callback(request: Request): general_settings, jwt_handler, master_key, - premium_user, prisma_client, saml_sso_cache, ) @@ -2352,8 +2325,6 @@ async def saml_callback(request: Request): user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data) - await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client) - ui_access_mode: Final = general_settings.get("ui_access_mode", None) relay_state: Final = post_data.get("RelayState") cp_return_to: Final[str | None] = ( @@ -2821,12 +2792,20 @@ async def insert_sso_user( if result_openid and hasattr(result_openid, "provider"): new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")} - response: Final = await new_user( - data=new_user_request, - user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), - ) + from litellm.proxy.proxy_server import premium_user - return response + with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True): + try: + return await new_user( + data=new_user_request, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + except ProxyException as error: + if premium_user is True: + raise + raise _SSOUserAdmissionError( + message=error.message, type=error.type, param=error.param, code=error.code + ) from error DefaultTeamJoin: TypeAlias = Callable[ @@ -3567,6 +3546,8 @@ class SSOAuthenticationHandler: prisma_client=prisma_client, ) return user_info + except _SSOUserAdmissionError: + raise except Exception as e: if ( user_info is None diff --git a/litellm/proxy/management_endpoints/usage_endpoints/__init__.py b/litellm/proxy/management_endpoints/usage_endpoints/__init__.py deleted file mode 100644 index 6e68dcd2a2e..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Usage endpoints package. - -Re-exports the router from endpoints module. -""" - -from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401 - router, -) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py deleted file mode 100644 index 7a45d0438de..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ /dev/null @@ -1,627 +0,0 @@ -""" -AI Usage Chat - uses LLM tool calling to answer questions about -usage/spend data by querying the aggregated daily activity endpoints. -""" - -import json -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence -from datetime import date -from typing import Final, Literal, NamedTuple, Protocol, cast, overload - -from fastapi import HTTPException -from typing_extensions import ReadOnly, TypedDict - -import litellm -from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL -from litellm.proxy._types import CommonProxyErrors -from litellm.types.proxy.management_endpoints.common_daily_activity import ( - SpendAnalyticsPaginatedResponse, -) -from litellm.types.utils import ChatCompletionMessageToolCall - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -USAGE_AI_TEMPERATURE: Final = 0.2 - -TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend" -TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend" -TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend" - -ENTITY_FIELD_USER: Final = "user_id" -ENTITY_FIELD_TEAM: Final = "team_id" -ENTITY_FIELD_TAG: Final = "tag" - -PAGINATED_PAGE_SIZE: Final = 200 -MAX_CHAT_MESSAGES: Final = 20 -TOP_N_MODELS: Final = 15 -TOP_N_PROVIDERS: Final = 10 -TOP_N_KEYS: Final = 10 - -# --------------------------------------------------------------------------- -# Types -# --------------------------------------------------------------------------- - - -class SSEStatusEvent(TypedDict): - type: Literal["status"] - message: str - - -class SSEToolCallEvent(TypedDict, total=False): - type: Literal["tool_call"] - tool_name: str - tool_label: str - arguments: dict[str, str] - status: Literal["running", "complete", "error"] - error: str - - -class SSEChunkEvent(TypedDict): - type: Literal["chunk"] - content: str - - -class SSEDoneEvent(TypedDict): - type: Literal["done"] - - -class SSEErrorEvent(TypedDict): - type: Literal["error"] - message: str - - -SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent - - -class _EntityEntry(TypedDict, total=False): - metrics: ReadOnly[Mapping[str, float]] - metadata: ReadOnly[Mapping[str, str]] - - -class _DayDump(TypedDict, total=False): - breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]] - - -class _EntityTotal(NamedTuple): - """Running per-entity totals accumulated while summarising a usage dump.""" - - alias: str - spend: float - requests: float - tokens: float - - -class _UsageDump(Protocol): - @overload - def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ... - @overload - def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ... - - -class _ToolFunctionDef(TypedDict): - name: ReadOnly[str] - description: ReadOnly[str] - parameters: ReadOnly[Mapping[str, object]] - - -class _ToolDef(TypedDict): - type: ReadOnly[str] - function: ReadOnly[_ToolFunctionDef] - - -class ToolHandler(TypedDict): - fetch: Callable[..., Awaitable[_UsageDump]] - summarise: Callable[[_UsageDump], str] - label: str - - -# --------------------------------------------------------------------------- -# Tool definitions (OpenAI function-calling schema) -# --------------------------------------------------------------------------- - -_DATE_PARAMS: Final = { - "start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"}, - "end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"}, -} - -_TOOL_USAGE: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_usage_data", - "description": ( - "Fetch aggregated global usage/spend data. Returns daily spend, " - "token counts, request counts, and breakdowns by model, provider, " - "and API key. Use for overall spend, top models, top providers." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "user_id": { - "type": "string", - "description": "Optional user ID filter. Omit for global view.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -_TOOL_TEAM: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_team_usage_data", - "description": ( - "Fetch usage/spend data broken down by team. Use for questions " - "like 'which team spends the most' or 'show me team X usage'." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "team_ids": { - "type": "string", - "description": "Optional comma-separated team IDs. Omit for all teams.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -_TOOL_TAG: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_tag_usage_data", - "description": ( - "Fetch usage/spend data broken down by tag. Tags are labels " - "attached to requests (features, environments, credentials)." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "tags": { - "type": "string", - "description": "Optional comma-separated tag names. Omit for all tags.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -TOOLS_BASE: Final = [_TOOL_USAGE] -TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG] - - -def get_tools_for_role(is_admin: bool) -> list[_ToolDef]: - """Return the tool list appropriate for the user's role.""" - return TOOLS_ADMIN if is_admin else TOOLS_BASE - - -_SYSTEM_PROMPT_BASE: Final = ( - "You are an AI assistant embedded in the LiteLLM Usage dashboard. " - "You help users understand their LLM API spend and usage data.\n\n" - "ALWAYS call the appropriate tool(s) first to fetch data before answering. " - "You may call multiple tools if the question spans different dimensions.\n\n" - "Guidelines:\n" - "- Be concise and specific. Use exact numbers from the data.\n" - "- Format costs as dollar amounts (e.g. $12.34).\n" - "- When comparing entities, show a ranked list.\n" - "- If data is empty or no results found, say so clearly.\n" - "- Do not hallucinate data — only use what the tools return.\n" - "- Today's date will be provided below. Use it to interpret relative dates " - "like 'this week', 'this month', 'last 7 days', etc." -) - -_TOOL_DESCRIPTIONS_ADMIN: Final = ( - "You have access to these tools:\n" - "- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n" - "- `get_team_usage_data`: Team-level usage breakdown\n" - "- `get_tag_usage_data`: Tag-level usage breakdown\n\n" -) - -_TOOL_DESCRIPTIONS_BASE: Final = ( - "You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n" -) - - -def _build_system_prompt(is_admin: bool) -> str: - """Build role-appropriate system prompt with today's date.""" - tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE - return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}" - - -# keep a public reference for test assertions -SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE - -# --------------------------------------------------------------------------- -# Data fetchers -# --------------------------------------------------------------------------- - - -def _parse_csv_ids(raw: str | None) -> list[str] | None: - if not raw: - return None - return [t.strip() for t in raw.split(",") if t.strip()] - - -async def _query_activity( - table_name: str, - entity_id_field: str, - entity_id: str | list[str] | None, - start_date: str, - end_date: str, - *, - use_aggregated: bool = False, -) -> SpendAnalyticsPaginatedResponse: - """Shared helper that calls the daily activity query layer.""" - from litellm.proxy.management_endpoints.common_daily_activity import ( - daily_activity_repository, - daily_activity_scope, - get_daily_activity, - get_daily_activity_aggregated, - ) - from litellm.proxy.proxy_server import prisma_client - - if use_aggregated: - if prisma_client is None: - raise HTTPException( - status_code=500, - detail={"error": CommonProxyErrors.db_not_connected_error.value}, - ) - repository: Final = daily_activity_repository(prisma_client) - scope: Final = daily_activity_scope( - table_name, - entity_id_field, - entity_id, - None, - None, - start_date, - end_date, - None, - None, - ) - return await get_daily_activity_aggregated( - repository, - scope, - ) - return await get_daily_activity( - prisma_client=prisma_client, - table_name=table_name, - entity_id_field=entity_id_field, - entity_id=entity_id, - entity_metadata_field=None, - start_date=start_date, - end_date=end_date, - model=None, - api_key=None, - page=1, - page_size=PAGINATED_PAGE_SIZE, - ) - - -async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_USER_SPEND, - ENTITY_FIELD_USER, - user_id, - start_date, - end_date, - use_aggregated=True, - ) - return resp.model_dump(mode="json") - - -async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_TEAM_SPEND, - ENTITY_FIELD_TEAM, - _parse_csv_ids(team_ids), - start_date, - end_date, - ) - return resp.model_dump(mode="json") - - -async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_TAG_SPEND, - ENTITY_FIELD_TAG, - _parse_csv_ids(tags), - start_date, - end_date, - ) - return resp.model_dump(mode="json") - - -# --------------------------------------------------------------------------- -# Summarisers — convert raw JSON to concise text the LLM can reason over -# --------------------------------------------------------------------------- - - -def _accumulate_breakdown( - results: Sequence[_DayDump], dimension: str, fields: Sequence[str] -) -> dict[str, dict[str, float]]: - """Aggregate a single breakdown dimension across days.""" - totals: Final[dict[str, dict[str, float]]] = {} - for day in results: - for key, entry in day.get("breakdown", {}).get(dimension, {}).items(): - if key not in totals: - totals[key] = {f: 0.0 for f in fields} - m = entry.get("metrics", {}) - for f in fields: - totals[key][f] += m.get(f, 0) - return totals - - -def _ranked_lines( - totals: dict[str, dict[str, float]], - fmt: Callable[[str, dict[str, float]], str], - limit: int, -) -> list[str]: - """Sort by spend descending, format each entry, and truncate.""" - return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]] - - -def _summarise_usage_data(data: _UsageDump) -> str: - meta: Final = data.get("metadata", {}) - results: Final = data.get("results", []) - - header: Final = ( - f"Total Spend: ${meta.get('total_spend', 0):.4f}\n" - f"Total Requests: {meta.get('total_api_requests', 0)}\n" - f"Successful: {meta.get('total_successful_requests', 0)} | " - f"Failed: {meta.get('total_failed_requests', 0)}\n" - f"Total Tokens: {meta.get('total_tokens', 0)}" - ) - - models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"]) - providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"]) - - model_lines: Final = _ranked_lines( - models, - lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)", - TOP_N_MODELS, - ) - provider_lines: Final = _ranked_lines( - providers, - lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)", - TOP_N_PROVIDERS, - ) - - sections = [header, ""] - sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""] - sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"]) - return "\n".join(sections) - - -def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str: - """Summarise team/tag entity usage data.""" - results: Final = data.get("results", []) - if not results: - return f"No {entity_label} usage data found for the given date range." - - totals: Final[dict[str, _EntityTotal]] = {} - for day in results: - for eid, entry in day.get("breakdown", {}).get("entities", {}).items(): - previous = totals.get(eid) - m = entry.get("metrics", {}) - totals[eid] = _EntityTotal( - alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid), - spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0), - requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0), - tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0), - ) - - lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""] - for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend): - label = d.alias if d.alias != eid else eid - lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens") - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Tool dispatch registry -# --------------------------------------------------------------------------- - -TOOL_HANDLERS: Final[dict[str, ToolHandler]] = { - "get_usage_data": ToolHandler( - fetch=_fetch_usage_data, - summarise=_summarise_usage_data, - label="global usage data", - ), - "get_team_usage_data": ToolHandler( - fetch=_fetch_team_usage_data, - summarise=lambda data: _summarise_entity_data(data, "Team"), - label="team usage data", - ), - "get_tag_usage_data": ToolHandler( - fetch=_fetch_tag_usage_data, - summarise=lambda data: _summarise_entity_data(data, "Tag"), - label="tag usage data", - ), -} - - -# --------------------------------------------------------------------------- -# SSE streaming -# --------------------------------------------------------------------------- - - -def _sse(event: SSEEvent) -> str: - return f"data: {json.dumps(event)}\n\n" - - -def _resolve_fetch_kwargs( - fn_name: str, - fn_args: Mapping[str, str], - user_id: str | None, - is_admin: bool, -) -> dict[str, str]: - """Build keyword arguments for a tool's fetch function.""" - start_date: Final = fn_args.get("start_date", "") - end_date: Final = fn_args.get("end_date", "") - if not start_date or not end_date: - raise ValueError("Missing required start_date or end_date from tool arguments") - kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date} - if fn_name == "get_usage_data": - if not is_admin: - if user_id is None: - # Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py - # should have already rejected this. If we ever reach here it means - # a future caller invoked the helper without scoping — fail loudly - # rather than issuing an unfiltered global query. - raise ValueError( - "Non-admin caller has user_id=None; refusing to issue an " - "unscoped query. Endpoint-level guard missing." - ) - kwargs["user_id"] = user_id - elif fn_args.get("user_id"): - kwargs["user_id"] = fn_args["user_id"] - elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"): - kwargs["team_ids"] = fn_args["team_ids"] - elif fn_name == "get_tag_usage_data" and fn_args.get("tags"): - kwargs["tags"] = fn_args["tags"] - return kwargs - - -async def _execute_tool_call( - handler: ToolHandler, - fn_name: str, - fn_args: Mapping[str, str], - user_id: str | None, - is_admin: bool, -) -> str: - """Run a single tool and return the summarised result text, or the reason the query - layer rejected its dates so the model can correct them.""" - kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin) - try: - raw_data: Final = await handler["fetch"](**kwargs) - except HTTPException as e: - reason: Final = _invalid_date_reason(e) - if reason is None: - raise - return f"Invalid date range: {reason}" - return handler["summarise"](raw_data) - - -def _invalid_date_reason(error: HTTPException) -> str | None: - """The reason in the daily activity query layer's 400 for invalid or reversed dates.""" - detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict - match detail: - case {"error": str() as reason} if error.status_code == 400: - return reason - case _: - return None - - -async def _process_tool_call( - tc: ChatCompletionMessageToolCall, - chat_messages: list[Mapping[str, object]], - user_id: str | None, - is_admin: bool, -) -> AsyncIterator[str]: - """Execute a single tool call, yielding SSE events for status.""" - fn_name: Final = tc.function.name - fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) - - allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} - handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None - - if fn_name is None or fn_name not in allowed_names or not handler: - chat_messages.append( - { - "role": "tool", - "tool_call_id": tc.id, - "content": f"Tool not available: {fn_name}", - } - ) - return - - tool_event_base: Final = { - "type": "tool_call", - "tool_name": fn_name, - "tool_label": handler["label"], - "arguments": fn_args, - } - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"})) - - try: - tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin) - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"})) - except Exception as e: - verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e) - tool_result = f"Error fetching {handler['label']}. Please try again." - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"})) - - chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result}) - - -async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]: - """Stream the final LLM response after tool results are appended.""" - yield _sse({"type": "status", "message": "Analyzing results..."}) - - response: Final = await litellm.acompletion( - model=model, - messages=chat_messages, - stream=True, - temperature=USAGE_AI_TEMPERATURE, - ) - async for chunk in response: - delta = chunk.choices[0].delta.content - if delta: - yield _sse({"type": "chunk", "content": delta}) - - -async def stream_usage_ai_chat( - messages: list[dict[str, str]], - model: str | None = None, - user_id: str | None = None, - is_admin: bool = False, -) -> AsyncGenerator[str, None]: - """Stream SSE events: status → tool_call → chunk → done.""" - resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL - truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages - chat_messages: Final[list[Mapping[str, object]]] = [ - {"role": "system", "content": _build_system_prompt(is_admin)}, - *truncated, - ] - - try: - yield _sse({"type": "status", "message": "Thinking..."}) - tools: Final = get_tools_for_role(is_admin) - response: Final = await litellm.acompletion( - model=resolved_model, - messages=chat_messages, - tools=tools, - temperature=USAGE_AI_TEMPERATURE, - ) - choice: Final = response.choices[0] - - if not choice.message.tool_calls: - if choice.message.content: - yield _sse({"type": "chunk", "content": choice.message.content}) - yield _sse({"type": "done"}) - return - - chat_messages.append(choice.message.model_dump()) - for tc in choice.message.tool_calls: - async for event in _process_tool_call(tc, chat_messages, user_id, is_admin): - yield event - async for event in _stream_final_response(resolved_model, chat_messages): - yield event - yield _sse({"type": "done"}) - - except Exception as e: - verbose_proxy_logger.error("AI usage chat failed: %s", e) - yield _sse( - { - "type": "error", - "message": "An internal error occurred. Please try again.", - } - ) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py deleted file mode 100644 index cff5bef05a0..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -USAGE AI CHAT ENDPOINTS - -/usage/ai/chat - Stream AI chat responses about usage data -""" - -from typing import Final, Literal - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import StreamingResponse -from pydantic import Field - -import litellm -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.sse_keepalive import ( - SSE_COMMENT_PING, - wrap_sse_stream_with_keepalive_pings, -) -from litellm.types.llms.base import LiteLLMBaseModel - -router: Final = APIRouter() - - -class ChatMessage(LiteLLMBaseModel): - role: Literal["user", "assistant"] - content: str - - -class UsageAIChatRequest(LiteLLMBaseModel): - messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)") - model: str | None = Field(default=None, description="Model to use for AI chat") - - -@router.post( - "/usage/ai/chat", - tags=["Budget & Spend Tracking"], - dependencies=[Depends(user_api_key_auth)], -) -async def usage_ai_chat( - data: UsageAIChatRequest, - request: Request, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - AI chat about usage data. Streams SSE events with the AI response. - The AI agent has access to tools that query aggregated daily activity data. - """ - from litellm.proxy.management_endpoints.common_utils import ( - require_caller_user_id_for_non_admin, - user_api_key_has_admin_view, - ) - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - stream_usage_ai_chat, - ) - - is_admin: Final = user_api_key_has_admin_view(user_api_key_dict) - if is_admin: - user_id = user_api_key_dict.user_id - else: - user_id = require_caller_user_id_for_non_admin(user_api_key_dict) - messages: Final = [{"role": m.role, "content": m.content} for m in data.messages] - - return StreamingResponse( - wrap_sse_stream_with_keepalive_pings( - stream_usage_ai_chat( - messages=messages, - model=data.model, - user_id=user_id, - is_admin=is_admin, - ), - ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, - ping_chunk=SSE_COMMENT_PING, - ), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) diff --git a/litellm/proxy/management_helpers/team_list_sorting.py b/litellm/proxy/management_helpers/team_list_sorting.py new file mode 100644 index 00000000000..caddd8afc08 --- /dev/null +++ b/litellm/proxy/management_helpers/team_list_sorting.py @@ -0,0 +1,109 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm.proxy.utils import PrismaClient + +COMPUTED_TEAM_SORTS: Final = MappingProxyType( + { + "organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') " + 'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), ' + "NULLIF(t.organization_id, ''))", + "members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' " + "THEN jsonb_array_length(t.members_with_roles) ELSE 0 END", + "models": "cardinality(t.models)", + "keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)', + } +) +_WHERE: Final = TypeAdapter(dict[str, JsonValue]) + + +@dataclass(frozen=True, slots=True) +class UnsupportedTeamFilter: + field: str + + +def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter: + if field not in ("team_id", "team_alias", "organization_id"): + return UnsupportedTeamFilter(field) + column: Final = f't."{field}"' + match value: + case str(text): + return f"{column} = ${index}", (text,) + case {"in": list(values)}: + return f"{column} = ANY(${index}::text[])", (values,) + case {"contains": str(text), "mode": "insensitive"}: + return f"{column} ILIKE ${index}", (f"%{text}%",) + case {"startsWith": str(text)}: + return f"{column} LIKE ${index}", (f"{text}%",) + case _: + return UnsupportedTeamFilter(field) + + +def _group_tokens( + field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue] +) -> tuple[str | tuple[str, JsonValue], ...]: + children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value) + if not children: + return ("FALSE" if field == "OR" else "TRUE",) + return ( + "(", + *chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)), + ")", + ) + + +def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter: + pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal + fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying + params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order + while pending: + match pending.pop(): + case str(fragment): + fragments.append(fragment) + case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)): + pending.extend(reversed(_group_tokens(field, value))) + case (field, value): + match _field_sql(field, value, len(params) + 1): + case UnsupportedTeamFilter() as failure: + return failure + case (fragment, values): + fragments.append(fragment) + params.extend(values) + return "".join(fragments), tuple(params) + + +class _TeamId(BaseModel): + model_config = ConfigDict(frozen=True) + team_id: str + + +_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...]) + + +async def sorted_team_ids( + prisma_client: PrismaClient, + *, + where: Mapping[str, object], + sort_by: str, + descending: bool, + skip: int, + take: int, +) -> Sequence[str] | UnsupportedTeamFilter: + result: Final = _where_sql(_WHERE.validate_python(where)) + if isinstance(result, UnsupportedTeamFilter): + return result + predicate, params = result + expression: Final = COMPUTED_TEAM_SORTS[sort_by] + direction: Final = "DESC" if descending else "ASC" + sql: Final = ( + f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} ' + f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC " + f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}" + ) + rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip)) + return tuple(row.team_id for row in rows) diff --git a/litellm/proxy/management_helpers/user_list_sorting.py b/litellm/proxy/management_helpers/user_list_sorting.py new file mode 100644 index 00000000000..d3d2e323cb0 --- /dev/null +++ b/litellm/proxy/management_helpers/user_list_sorting.py @@ -0,0 +1,77 @@ +from collections.abc import Sequence +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.proxy.utils import PrismaClient + + +class _UserId(BaseModel): + model_config = ConfigDict(frozen=True) + user_id: str + + +_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...]) + + +async def sorted_user_ids( + prisma_client: PrismaClient, + *, + sort_by: Literal["status", "key_count"], + descending: bool, + skip: int, + take: int, + role: str | None, + user_ids: str | None, + user_email: str | None, + team: str | None, + sso_user_ids: str | None, + organization_ids: str | None, + search: str | None, +) -> Sequence[str]: + ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip()) + sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip()) + org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip()) + expression: Final = ( + "CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END" + if sort_by == "status" + else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k ' + "WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)" + ) + direction: Final = "DESC" if descending else "ASC" + sql: Final = f""" + SELECT u.user_id FROM "LiteLLM_UserTable" u + WHERE ($1::text IS NULL OR u.user_role = $1) + AND ($2::text[] IS NULL + OR CASE WHEN cardinality($2::text[]) = 1 + THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%' + ELSE u.user_id = ANY($2::text[]) END) + AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%') + AND ($4::text IS NULL OR $4 = ANY(u.teams)) + AND ($5::boolean OR u.sso_user_id = ANY($6::text[])) + AND (cardinality($7::text[]) = 0 OR EXISTS ( + SELECT 1 FROM "LiteLLM_OrganizationMembership" m + WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[]))) + AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%' + OR u.user_email ILIKE '%' || $8 || '%') + ORDER BY {expression} {direction}, u.user_id ASC + LIMIT $9 OFFSET $10 + """ + rows: Final = _USER_IDS.validate_python( + await prisma_client.db.query_raw( + sql, + role or None, + ids if user_ids else None, + user_email, + team, + sso_user_ids is None, + sso_ids, + org_ids, + search or None, + take, + skip, + *((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()), + ) + ) + return tuple(row.user_id for row in rows) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f632317d9a8..ade719ae0ed 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12210,7 +12210,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a async def model_list( request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - return_wildcard_routes: bool | None = False, + return_wildcard_routes: bool | None = None, team_id: str | None = None, include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, @@ -12226,6 +12226,9 @@ async def model_list( This is just for compatibility with openai projects like aider. Query Parameters: + - return_wildcard_routes: When true, also list wildcard routes such as `openai/*`. + Defaults to `general_settings.return_wildcard_routes`, and an explicit + value on the request wins over it - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a provider's model catalog. When present, the response is Codex's own catalog shape (`{"models": [...]}`) built from the same listing, with @@ -12254,6 +12257,9 @@ async def model_list( global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj settings: Final = cast(dict[str, object], general_settings) # any-ok: legacy settings + include_wildcard_routes: Final = ( + return_wildcard_routes if return_wildcard_routes is not None else settings.get("return_wildcard_routes") is True + ) from litellm.llms.anthropic.common_utils import ( create_anthropic_model_list_response, @@ -12343,7 +12349,7 @@ async def model_list( proxy_model_list=proxy_model_list, user_model=None, infer_model_from_keys=False, - return_wildcard_routes=return_wildcard_routes or False, + return_wildcard_routes=include_wildcard_routes, llm_router=llm_router, model_access_groups=model_access_groups, include_model_access_groups=include_model_access_groups or False, @@ -12411,7 +12417,7 @@ async def model_list( team_id=team_id, include_model_access_groups=include_model_access_groups or False, only_model_access_groups=only_model_access_groups or False, - return_wildcard_routes=return_wildcard_routes or False, + return_wildcard_routes=include_wildcard_routes, user_api_key_cache=user_api_key_cache, ) @@ -19657,6 +19663,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro "proxy_config_reload_interval_seconds": "Integer", "pass_through_endpoints": "PydanticModel", "store_model_in_db": "Boolean", + "return_wildcard_routes": "Boolean", "store_prompts_in_spend_logs": "Boolean", "disable_requester_ip_address_logging": "Boolean", "disable_fallbacks_on_per_model_rate_limits": "Boolean", diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 6ced7827846..351e4c10d87 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import ( get_blog_posts, ) from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.proxy._types import ( CommonProxyErrors, ) @@ -564,7 +565,7 @@ async def _resolve_autorouter_presets( url: str, fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]], ) -> Mapping[str, AutoRouterPresetRecord]: - if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": return _load_bundled_autorouter_presets() try: return await fetch(url) diff --git a/litellm/proxy/spend_tracking/vantage_endpoints.py b/litellm/proxy/spend_tracking/vantage_endpoints.py index d3858f95243..6938588444c 100644 --- a/litellm/proxy/spend_tracking/vantage_endpoints.py +++ b/litellm/proxy/spend_tracking/vantage_endpoints.py @@ -122,7 +122,9 @@ async def _get_vantage_settings(): encrypted_api_key: Final = settings.get("api_key") if encrypted_api_key: - decrypted_api_key = decrypt_value_helper(encrypted_api_key, key="vantage_api_key", exception_type="error") + decrypted_api_key: Final = decrypt_value_helper( + encrypted_api_key, key="vantage_api_key", exception_type="error" + ) if decrypted_api_key is None: raise HTTPException( status_code=500, diff --git a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py index 7411f1b4947..4bb17fd290e 100644 --- a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py @@ -1,5 +1,7 @@ import asyncio +import os import re +import xml.etree.ElementTree as ET from collections import Counter from collections.abc import Awaitable, Mapping from dataclasses import dataclass @@ -12,12 +14,14 @@ from pydantic import ValidationError from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.llms.base import LiteLLMBaseModel router: Final = APIRouter() -LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest" +CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/" +CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml" RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}" LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5 LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60 @@ -26,7 +30,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info" _RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S") _NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b") -_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$") +_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/") _Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"] _PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"}) @@ -45,6 +49,16 @@ class LatestReleaseUnavailable: reason: str +@dataclass(frozen=True, slots=True) +class ChangelogEntry: + version: tuple[int, int, int] + url: str + + @property + def tag(self) -> str: + return "v" + ".".join(str(part) for part in self.version) + + class _GitHubRelease(LiteLLMBaseModel): tag_name: str html_url: str @@ -76,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock: return _latest_release_fetch_lock +def _default_release_check_enabled() -> bool: + return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true" + + def _bucket_for(line: str) -> _Bucket | None: if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None: return None @@ -91,9 +109,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]: return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None)) -def _stable_version(tag_name: str) -> tuple[int, int, int] | None: - match: Final = _STABLE_TAG_PATTERN.match(tag_name) - return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3))) +def _changelog_entry(link: str) -> ChangelogEntry | None: + match: Final = _CHANGELOG_PAGE_PATTERN.match(link) + if match is None: + return None + return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link) + + +def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None: + """The highest stable release with a page in the changelog feed. Release candidate pages don't count.""" + links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link")) + entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None) + return max(entries, key=lambda entry: entry.version, default=None) def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]: @@ -102,7 +129,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]: return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch)) -def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo: +def summarize_release_line( + entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...] +) -> LatestReleaseInfo: """A patch release's notes only list its backports, so count the whole minor line it belongs to.""" counts: Final = sum( (Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)), @@ -113,7 +142,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease new_features=counts.get("new_features", 0), bug_fixes=counts.get("bug_fixes", 0), other_updates=counts.get("other_updates", 0), - release_url=latest.html_url, + release_url=entry.url, ) @@ -137,25 +166,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe return release +async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable: + response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS) + if response.status_code != 200: + return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}") + try: + entry: Final = latest_changelog_entry(response.content) + except ET.ParseError as e: + return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}") + return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry + + async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable: try: + entry: Final = await _fetch_changelog_entry(client) + if isinstance(entry, LatestReleaseUnavailable): + return entry latest: Final = parse_github_release( - await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS) + await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS) ) if isinstance(latest, LatestReleaseUnavailable): return latest - version: Final = _stable_version(latest.tag_name) - if version is None: - return summarize_release_line(latest, ()) earlier: Final = await asyncio.gather( - *(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version)) + *(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version)) ) except httpx.HTTPError as e: return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}") unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None) if unavailable is not None: return unavailable - return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))) + return summarize_release_line( + entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)) + ) async def get_latest_release_info( @@ -188,12 +230,16 @@ async def latest_release_info( client: Annotated[_AsyncGetClient, Depends(_default_client)], cache: Annotated[InMemoryCache, Depends(_default_cache)], fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)], + release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)], ) -> LatestReleaseInfo | None: """ - Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1) - split into new features, bug fixes and other updates. - Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render. + Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole + minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates. + Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render. + Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request. """ + if not release_check_enabled: + return None result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) if isinstance(result, LatestReleaseUnavailable): verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 52b3866d04d..efc83132568 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1294,6 +1294,7 @@ def _failure_usage_to_lift( provider_billed: bool = False, deployment_id: str | None = None, base_model: str | None = None, + dispatched_call_type: str | None = None, ) -> tuple[object, object] | None: current_attempt: Final = _current_failure_usage_to_lift( model_call_details, @@ -1302,6 +1303,7 @@ def _failure_usage_to_lift( provider_billed, deployment_id, base_model, + dispatched_call_type=dispatched_call_type, ) failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) if not failed_attempts: @@ -6030,13 +6032,12 @@ class PrismaClient: elif table_name == "user": db_data = self.jsonify_object(data=data) try: - new_user_row: Final = await UserRepository(self).table.upsert( - where={"user_id": data["user_id"]}, - data={ - "create": {**db_data}, - "update": {}, # don't do anything if it already exists - }, - ) + new_user_row: Final = await UserRepository(self).upsert_user(db_data) + if new_user_row is None: + raise HTTPException( + status_code=403, + detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.", + ) except Exception as e: if ( "Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`" diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 5ed1574c6bc..1fce0717ea3 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -2,8 +2,11 @@ User repository for database operations on LiteLLM_UserTable. """ +import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, contextmanager +from contextvars import ContextVar from itertools import chain from typing import TYPE_CHECKING, Final, Protocol @@ -35,6 +38,16 @@ ORDER BY p.user_id """ _PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...]) +_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None) + + +@contextmanager +def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]: + token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None) + try: + yield + finally: + _FREE_SSO_CREATION.reset(token) class _UserDb(Protocol): @@ -42,6 +55,12 @@ class _UserDb(Protocol): def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ... +class _UserTransaction(_UserDb, Protocol): + async def execute_raw(self, query: str) -> int: ... + + async def query_raw(self, query: str) -> object: ... + + class _PrismaClientView(Protocol): @property def db(self) -> _UserDb: ... @@ -49,6 +68,16 @@ class _PrismaClientView(Protocol): @property def writer_db(self) -> _UserDb: ... + def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ... + + +async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int: + from prisma import Json + + total: Final = await table.count() + deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}}) + return max(0, total - deactivated) + class UserRepository(BaseRepository[LiteLLM_UserTable]): """Repository for user database operations.""" @@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): null, or true all count, so seats are counted as total users minus the deactivated ones. """ - from prisma import Json # pyright: ignore[reportUnknownVariableType] + return await _count_billable_users(self.table) - total: Final = await self.count() - deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}}) - return max(0, total - deactivated) + async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None": + """Return the user row, or None when a new free SSO user would exceed capacity.""" + scope: Final = _FREE_SSO_CREATION.get() + if ( + scope is None + or (scope[0] is not None and scope[0] != data["user_id"]) + or scope[1] is not asyncio.current_task() + ): + return await self.table.upsert( + where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} + ) + + client: Final[_PrismaClientView] = self.prisma_client + async with client.tx() as transaction: + await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED") + await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)") + existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]}) + if existing is not None: + return existing + if await _count_billable_users(transaction.litellm_usertable) >= 5: + return None + return await transaction.litellm_usertable.upsert( + where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} + ) async def create_user( self, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 09eea7cb323..e7380c26529 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,10 +6,10 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from typing_extensions import ( TypeIs, # noqa: TID251 # narrows a client input item before its mark is read assert_never, @@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig from litellm.responses.litellm_completion_transformation.compaction_items import ( with_compaction_items_as_summaries, @@ -85,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.types.responses.main import * from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params +from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, + CallTypes, + CostPerToken, + CustomPricingLiteLLMParams, + all_litellm_params, +) from litellm.utils import ( ProviderConfigManager, client, @@ -544,6 +552,47 @@ async def aresponses_api_with_mcp( return response +def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: + return MappingProxyType( + litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + +class _DeploymentModelInfo(BaseModel): + model_config = ConfigDict(frozen=True) + id: object = None + + +_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter( + Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")] +) + + +def _is_router_deployment(deployment_model_info: object) -> bool: + """A Router stamps the deployment's model_info (with its id) onto every call it routes.""" + parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info) + return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None + + +_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) + + +def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None: + if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None: + return None + return _COST_PER_TOKEN.validate_python( + litellm_params.model_dump( + include={ + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + }, + exclude_none=True, + ) + ) + + def _bridges_to_chat_completions( responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool ) -> bool: @@ -1466,6 +1515,12 @@ def responses( # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) + affinity_api_base: Final = ( + get_openai_credentials(api_base=litellm_params.api_base).api_base + if custom_llm_provider == "openai" + else litellm_params.api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) try: effective_extra_headers: Final = ( add_provider_affinity_header( @@ -1479,8 +1534,9 @@ def responses( "litellm_metadata": kwargs.get("litellm_metadata"), } ), + default_header=default_affinity_header, ) - if litellm_params.provider_affinity_header is not None + if litellm_params.provider_affinity_header is not None or default_affinity_header is not None else extra_headers ) except ValueError as affinity_error: @@ -1664,6 +1720,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, + **_per_call_custom_pricing(litellm_params), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -1674,6 +1731,9 @@ def responses( custom_llm_provider=custom_llm_provider, ) + if not _is_router_deployment(deployment_model_info): + litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) + litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second # Decode any litellm-encoded encrypted-content item IDs back to their original IDs input = with_compaction_items_as_summaries( with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) diff --git a/litellm/router.py b/litellm/router.py index da04e59f18a..705accbcf8d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14186,6 +14186,43 @@ class Router: deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params) ) + @staticmethod + def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool: + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + declared: Final = model_info.get("supports_vision") + if declared is not None: + return declared is False + raw_params: Final = plain_deployment.get("litellm_params") or {} + params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts + if not params: + return False + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return litellm.utils.is_vision_explicitly_disabled( + base_model if isinstance(base_model, str) else model, custom_llm_provider + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + @staticmethod + def _request_carries_image( + messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is + request_kwargs: Mapping[str, object], + ) -> bool: + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + structured: Final = resolve_structured_messages( + messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts + request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping + ) + return bool(structured) and request_contains_image_content(structured) + @staticmethod def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]: requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs)) @@ -14212,6 +14249,7 @@ class Router: - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models - filter models above rpm limits - if region given, filter out models not in that region / unknown region + - request carries an image and the deployment declares supports_vision false, unless every candidate does - [TODO] function call and model doesn't support function calling """ @@ -14221,7 +14259,7 @@ class Router: # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request) _returned_deployments = list(healthy_deployments) - invalid_model_indices: Final = set() # Use set for O(1) membership checks + invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks _context_window_error = False _potential_error_str = "" @@ -14371,9 +14409,20 @@ class Router: model=model, llm_provider="", ) - if len(excluded_indices) > 0: + remaining: Final = tuple( + (idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices + ) + text_only_indices: Final[frozenset[int]] = ( + frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model)) + if remaining and self._request_carries_image(messages, request_settings) + else frozenset() + ) + dropped_indices: Final[frozenset[int]] = excluded_indices | ( + text_only_indices if len(text_only_indices) < len(remaining) else frozenset() + ) + if len(dropped_indices) > 0: # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops) - _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices] + _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices] return _returned_deployments diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index ebb221e98be..74cd28161a4 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { } MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { + Feature.PROVIDER_AFFINITY, Feature.SAFEGUARDS_CLASSIFIER, Feature.PRE_REQUEST_HOOK, Feature.WEBSEARCH_INTERCEPTION, diff --git a/litellm/rust_bridge/detectors.py b/litellm/rust_bridge/detectors.py index 0ad1b787e38..0af3a2e6fa6 100644 --- a/litellm/rust_bridge/detectors.py +++ b/litellm/rust_bridge/detectors.py @@ -128,7 +128,7 @@ def _custom_loggers(callbacks: Iterable[object]) -> tuple[CustomLogger, ...]: def overridden(method: str, callbacks: Iterable[object]) -> bool: - base: Final = getattr(CustomLogger, method) + base: Final[object] = getattr(CustomLogger, method) return any(getattr(type(callback), method, base) is not base for callback in _custom_loggers(callbacks)) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 691c04f2daa..b53e80805a5 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -5,10 +5,15 @@ from __future__ import annotations from typing import Final import litellm -from litellm.rust_bridge.detectors import overridden, present, streaming +from litellm.llms.opencode.common_utils import is_opencode_api_base +from litellm.rust_bridge.detectors import api_base, overridden, present, streaming from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str +def provider_affinity(request: NativeCall) -> bool: + return is_opencode_api_base(api_base(request)) + + def anthropic_workload_identity(request: NativeCall) -> bool: """Python mints an Anthropic federation token when no static credential is configured.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo diff --git a/litellm/rust_bridge/responses/detectors.py b/litellm/rust_bridge/responses/detectors.py index 392c11f48a8..763b63d6742 100644 --- a/litellm/rust_bridge/responses/detectors.py +++ b/litellm/rust_bridge/responses/detectors.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay from pydantic import TypeAdapter import litellm +from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str @@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool: def provider_affinity(request: NativeCall) -> bool: """Python validates and fills the affinity header from the session ID.""" - return present(request, "provider_affinity_header") + return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request)) def text_format(request: NativeCall) -> bool: diff --git a/litellm/text_completions/__init__.py b/litellm/text_completions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py new file mode 100644 index 00000000000..ec1b96fd432 --- /dev/null +++ b/litellm/text_completions/streaming_iterator.py @@ -0,0 +1,88 @@ +from typing import Final + +from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse + + +class TextCompletionStreamWrapper: + def __init__( + self, + completion_stream, + model, + stream_options: dict | None = None, + custom_llm_provider: str | None = None, + ): + self.completion_stream = completion_stream + self.model = model + self.stream_options = stream_options + self.custom_llm_provider = custom_llm_provider + + def __iter__(self): + return self + + def __aiter__(self): + return self + + def convert_to_text_completion_object(self, chunk: ModelResponse): + try: + response: Final = TextCompletionResponse() + response["id"] = chunk.get("id", None) + response["object"] = "text_completion" + response["created"] = chunk.get("created", None) + response["model"] = chunk.get("model", None) + text_choices: Final = TextChoices() + if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices + raise Exception + + if chunk["choices"]: + delta: Final = chunk["choices"][0]["delta"] + text_choices["text"] = delta["content"] + text_choices["reasoning_content"] = delta.get("reasoning_content") + text_choices["index"] = chunk["choices"][0]["index"] + text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] + response["choices"] = [text_choices] + else: + response["choices"] = [] + + # only pass usage when stream_options["include_usage"] is True + if self.stream_options and self.stream_options.get("include_usage", False) is True: + response.usage = chunk.get("usage", None) + else: + response.usage = None + + return response + except Exception as e: + raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") + + def __next__(self): + # model_response = ModelResponse(stream=True, model=self.model) + TextCompletionResponse() + try: + for chunk in self.completion_stream: + if chunk == "None" or chunk is None: + raise Exception + processed_chunk = self.convert_to_text_completion_object(chunk=chunk) + return processed_chunk + raise StopIteration + except StopIteration: + raise StopIteration + except Exception as e: + from litellm.litellm_core_utils.exception_mapping_utils import exception_type + + raise exception_type( + model=self.model, + custom_llm_provider=self.custom_llm_provider or "", + original_exception=e, + completion_kwargs={}, + extra_kwargs={}, + ) + + async def __anext__(self): + try: + async for chunk in self.completion_stream: + if chunk == "None" or chunk is None: + raise Exception + processed_chunk = self.convert_to_text_completion_object(chunk=chunk) + return processed_chunk + raise StopIteration + except StopIteration: + raise StopAsyncIteration diff --git a/litellm/types/caching.py b/litellm/types/caching.py index b9bcbbb93dd..ced2065a674 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -75,7 +75,7 @@ class RedisPipelineSetOperation(TypedDict): """ key: str - value: Any + value: object ttl: int | None @@ -133,7 +133,7 @@ class HealthCheckCacheParams(LiteLLMBaseModel): host: str | None = None port: str | int | None = None - redis_kwargs: dict[str, Any] | None = None + redis_kwargs: dict[str, object] | None = None namespace: str | None = None redis_version: str | int | float | None = None diff --git a/litellm/types/completion.py b/litellm/types/completion.py index a6693030e89..0f4bdbd75ce 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -246,7 +246,7 @@ _CompletionDispatchContext = CompletionDispatchContext _CompletionDispatchResult = Union[ - Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]], + Coroutine[object, object, Union["ModelResponse", "CustomStreamWrapper"]], "ModelResponse", "CustomStreamWrapper", ] diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index a1d48f730c0..5ad6e45e6d2 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[ DecisionQuestionMap: TypeAlias = Annotated[ Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], - Field(min_length=1, max_length=MAX_DECISION_QUESTIONS), + Field(min_length=1), ] diff --git a/litellm/types/extraction.py b/litellm/types/extraction.py index 4311e549b9a..562bc41f9de 100644 --- a/litellm/types/extraction.py +++ b/litellm/types/extraction.py @@ -13,6 +13,7 @@ from pydantic import ( model_validator, with_config, ) +from pydantic.fields import FieldInfo from typing_extensions import ReadOnly, Required, Self, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -30,7 +31,7 @@ def _is_none(value: object) -> bool: return value is None -_OMITTED_WHEN_NONE: Final = Field(exclude_if=_is_none) +_OMITTED_WHEN_NONE: Final = FieldInfo.from_field(exclude_if=_is_none) OptionalJson: TypeAlias = Annotated[JsonValue, _OMITTED_WHEN_NONE] OptionalFloat: TypeAlias = Annotated[float | None, _OMITTED_WHEN_NONE] OptionalInt: TypeAlias = Annotated[int | None, _OMITTED_WHEN_NONE] diff --git a/litellm/types/google_genai/main.py b/litellm/types/google_genai/main.py index 0241c9fa809..dcde0a031e3 100644 --- a/litellm/types/google_genai/main.py +++ b/litellm/types/google_genai/main.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: ToolConfigDict = _genai_types.ToolConfigDict class GenerateContentRequestDict(GenerateContentRequestParametersDict): - generationConfig: Any | None + generationConfig: object | None tools: ToolConfigDict | None class GenerateContentResponse(GoogleGenAIGenerateContentResponse, BaseLiteLLMOpenAIResponseObject): diff --git a/litellm/types/images/main.py b/litellm/types/images/main.py index 5d80135a8a1..963c980a4b8 100644 --- a/litellm/types/images/main.py +++ b/litellm/types/images/main.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from typing_extensions import TypedDict @@ -20,7 +20,7 @@ class ImageEditOptionalRequestParams(TypedDict, total=False): response_format: Literal["url", "b64_json"] | None size: str | None user: str | None - imageConfig: dict[str, Any] | None + imageConfig: dict[str, object] | None class ImageEditRequestParams(ImageEditOptionalRequestParams, total=False): diff --git a/litellm/types/integrations/argilla.py b/litellm/types/integrations/argilla.py index 2c98486c7a7..754916c2f3a 100644 --- a/litellm/types/integrations/argilla.py +++ b/litellm/types/integrations/argilla.py @@ -1,10 +1,10 @@ -from typing import Any, Final +from typing import Final from typing_extensions import TypedDict class ArgillaItem(TypedDict): - fields: dict[str, Any] + fields: dict[str, object] class ArgillaPayload(TypedDict): diff --git a/litellm/types/integrations/cloudzero.py b/litellm/types/integrations/cloudzero.py index 988d501600f..dcc5d03ccca 100644 --- a/litellm/types/integrations/cloudzero.py +++ b/litellm/types/integrations/cloudzero.py @@ -1,7 +1,4 @@ -from typing import Any - - -class CBFRecord(dict[str, Any]): +class CBFRecord(dict[str, object]): """CloudZero Billing Format (CBF) record structure. This class represents a CBF record that is created from LiteLLM usage data @@ -31,4 +28,4 @@ class CBFRecord(dict[str, Any]): # Type alias for better readability in function signatures -CBFRecordDict = dict[str, Any] +CBFRecordDict = dict[str, object] diff --git a/litellm/types/integrations/compression_interception.py b/litellm/types/integrations/compression_interception.py index a1e66e5057e..e9c40fbf838 100644 --- a/litellm/types/integrations/compression_interception.py +++ b/litellm/types/integrations/compression_interception.py @@ -2,7 +2,7 @@ Type definitions for Compression Interception integration. """ -from typing import Any, Literal, TypedDict +from typing import Literal, TypedDict class CompressionInterceptionConfig(TypedDict, total=False): @@ -24,7 +24,7 @@ class CompressionInterceptionConfig(TypedDict, total=False): compression_trigger: int compression_target: int | None embedding_model: str | None - embedding_model_params: dict[str, Any] | None + embedding_model_params: dict[str, object] | None class CompressionSavingsMetadata(TypedDict): diff --git a/litellm/types/integrations/gcs_bucket.py b/litellm/types/integrations/gcs_bucket.py index 3840d7681a5..79563a0a6e4 100644 --- a/litellm/types/integrations/gcs_bucket.py +++ b/litellm/types/integrations/gcs_bucket.py @@ -31,5 +31,5 @@ class GCSLogQueueItem(TypedDict): """ payload: StandardLoggingPayload - kwargs: dict[str, Any] - response_obj: Any | None + kwargs: dict[str, object] + response_obj: object | None diff --git a/litellm/types/integrations/langsmith.py b/litellm/types/integrations/langsmith.py index 594190ddb5c..f053a5864c1 100644 --- a/litellm/types/integrations/langsmith.py +++ b/litellm/types/integrations/langsmith.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from datetime import datetime -from typing import Any, NamedTuple +from typing import NamedTuple from typing_extensions import TypedDict @@ -9,7 +9,7 @@ from litellm.types.llms.base import LiteLLMBaseModel class LangsmithInputs(LiteLLMBaseModel): model: str | None = None - messages: list[Any] | None = None + messages: list[object] | None = None stream: bool | None = None call_type: str | None = None litellm_call_id: str | None = None @@ -17,9 +17,9 @@ class LangsmithInputs(LiteLLMBaseModel): temperature: float | None = None max_tokens: int | None = None custom_llm_provider: str | None = None - input: list[Any] | None = None + input: list[object] | None = None log_event_type: str | None = None - original_response: Any | None = None + original_response: object | None = None response_cost: float | None = None # LiteLLM Virtual Key specific fields diff --git a/litellm/types/integrations/posthog.py b/litellm/types/integrations/posthog.py index ac04d08107d..d9739df7df7 100644 --- a/litellm/types/integrations/posthog.py +++ b/litellm/types/integrations/posthog.py @@ -1,4 +1,4 @@ -from typing import Any, Final, TypedDict +from typing import Final, TypedDict POSTHOG_MAX_BATCH_SIZE: Final = 100 @@ -7,7 +7,7 @@ class PostHogEventPayload(TypedDict): """PostHog event payload structure""" event: str # "$ai_generation" or "$ai_embedding" - properties: dict[str, Any] + properties: dict[str, object] distinct_id: str diff --git a/litellm/types/interactions/README.md b/litellm/types/interactions/README.md index d890991d99d..45783d604db 100644 --- a/litellm/types/interactions/README.md +++ b/litellm/types/interactions/README.md @@ -9,16 +9,38 @@ https://ai.google.dev/static/api/interactions.openapi.json ### How to Regenerate -When the API spec changes, regenerate the types with: +The compliance tests in `tests/unit/interactions/test_openapi_compliance.py` read the snapshot at +`tests/unit/interactions/interactions.openapi.json`, so a PR never depends on Google's live spec. The scheduled +`interactions-spec-canary` workflow runs those tests against the live spec once a day (`--interactions-spec-url`), +files the issue "Interactions spec canary is red" on a red run and closes it once a run is green again. A red run +means the spec moved in a way these types no longer cover + +When the red run names one new status, the small fix is a hand edit adding that member to the enum the failing +schema's model carries, the way `queued` was added in #34135. When the spec moved further than that, refresh the +snapshot and regenerate the types from it: ```bash -uv tool run --from datamodel-code-generator datamodel-codegen \ - --url "https://ai.google.dev/static/api/interactions.openapi.json" \ +curl -fsSL https://ai.google.dev/static/api/interactions.openapi.json \ + -o tests/unit/interactions/interactions.openapi.json +uv tool run --from datamodel-code-generator==0.66.0 datamodel-codegen \ + --input tests/unit/interactions/interactions.openapi.json \ + --input-file-type openapi \ --output litellm/types/interactions/generated.py \ --output-model-type pydantic_v2.BaseModel \ - --target-python-version 3.9 + --target-python-version 3.10 \ + --base-class litellm.types.llms.base.LiteLLMBaseModel ``` +The pin is the same `datamodel-code-generator` version `scripts/generate_trace_types.py` uses. The generator names +inline enums by order of appearance (`Status`, `Status1`, `Status2`, ...), so a regeneration can rename them. The +compliance tests resolve each status enum through its model's `status` field, but +`tests/unit/interactions/test_background_cost_polling.py` and +`tests/unit/proxy/hooks/test_proxy_track_cost_callback.py` import `Status1` by name and need the new name, and +`litellm/types/interactions/__init__.py` exports the interaction status enum as `InteractionStatus`, which has to +point at the enum the regenerated `Interaction.status` carries (a compliance test checks their members match). The +spec has grown a lot since the 2025-12-16 generation (agents, triggers, credentials, environments), so a +regeneration today is a review-sized change rather than a refresh + Then add the LiteLLM-specific types at the bottom of the generated file: - `InteractionsAPIResponse` - `InteractionsAPIStreamingResponse` diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 03956ae7a54..9ad13243451 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import Any, Literal, TypeAlias +from typing import Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -20,7 +20,7 @@ class AnthropicResponseTextBlock(TypedDict, total=False): Anthropic Response Text Block: https://docs.anthropic.com/en/api/messages """ - citations: list[dict[str, Any]] | None + citations: list[dict[str, object]] | None text: str type: Literal["text"] diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 47fec914f9c..5a2da071cec 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from openai._models import BaseModel as OpenAIObject from pydantic import ConfigDict @@ -45,7 +45,7 @@ class BaseLiteLLMOpenAIResponseObject(LiteLLMBaseModel): class HiddenParams(OpenAIObject): - original_response: str | Any | None = None + original_response: str | object | None = None model_id: str | None = None # used in Router for individual deployments api_base: str | None = None # returns api base used for making completion call _response_ms: float | None = None diff --git a/litellm/types/llms/databricks.py b/litellm/types/llms/databricks.py index c7023e35777..bc229421d91 100644 --- a/litellm/types/llms/databricks.py +++ b/litellm/types/llms/databricks.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from pydantic import BaseModel from typing_extensions import ( @@ -23,7 +23,7 @@ class GenericStreamingChunk(TypedDict, total=False): class DatabricksTextContent(TypedDict, total=False): type: Literal["text"] text: Required[str] - citations: list[dict[str, Any]] | None + citations: list[dict[str, object]] | None class DatabricksReasoningSummary(TypedDict): @@ -35,7 +35,7 @@ class DatabricksReasoningSummary(TypedDict): class DatabricksReasoningContent(TypedDict, total=False): type: Literal["reasoning"] summary: Required[list[DatabricksReasoningSummary]] - citations: list[dict[str, Any]] | None + citations: list[dict[str, object]] | None AllDatabricksContentListValues = DatabricksTextContent | DatabricksReasoningContent diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index f6394f52c99..3584f8ce1da 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -2,7 +2,7 @@ Types and field definitions for cache settings management endpoints """ -from typing import Any, Final +from typing import Final from litellm.types.llms.base import LiteLLMBaseModel @@ -10,9 +10,9 @@ from litellm.types.llms.base import LiteLLMBaseModel class CacheSettingsField(LiteLLMBaseModel): field_name: str field_type: str - field_value: Any + field_value: object field_description: str - field_default: Any = None + field_default: object = None options: list[str] | None = None # For fields with predefined options/enum values ui_field_name: str # User-friendly display name link: str | None = None # Documentation link for the field diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index 9c8fe446d72..64068f30a16 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -2,7 +2,7 @@ Types and field definitions for router settings management endpoints """ -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import Field, field_validator @@ -72,9 +72,9 @@ class FallbackDeleteResponse(LiteLLMBaseModel): class RouterSettingsField(LiteLLMBaseModel): field_name: str field_type: str - field_value: Any + field_value: object field_description: str - field_default: Any = None + field_default: object = None options: list[str] | None = None # For fields with predefined options/enum values ui_field_name: str # User-friendly display name link: str | None = None # Documentation link for the field diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index f0be66f055d..3fa9adef913 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -113,7 +113,7 @@ class MCPPublicServer(LiteLLMBaseModel): transport: MCPTransportType spec_path: str | None = None auth_type: MCPAuthType | None = None - mcp_info: dict[str, Any] | None = None + mcp_info: dict[str, object] | None = None class MCPAllowedClient(LiteLLMBaseModel): @@ -448,11 +448,11 @@ class MCPPreCallRequestObject(LiteLLMBaseModel): """ tool_name: str - arguments: dict[str, Any] + arguments: dict[str, object] server_name: str | None = None tool_description: str | None = None tool_input_schema: Mapping[str, object] | None = None - user_api_key_auth: dict[str, Any] | None = None + user_api_key_auth: dict[str, object] | None = None hidden_params: HiddenParams = HiddenParams() @@ -462,7 +462,7 @@ class MCPPreCallResponseObject(LiteLLMBaseModel): """ should_proceed: bool = True - modified_arguments: dict[str, Any] | None = None + modified_arguments: dict[str, object] | None = None error_message: str | None = None hidden_params: HiddenParams = HiddenParams() @@ -473,7 +473,7 @@ class MCPDuringCallRequestObject(LiteLLMBaseModel): """ tool_name: str - arguments: dict[str, Any] + arguments: dict[str, object] server_name: str | None = None tool_description: str | None = None tool_input_schema: Mapping[str, object] | None = None diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 219f23200d2..d29e4db55a4 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -117,7 +117,7 @@ class MCPServer(LiteLLMBaseModel): # Admin-configured env vars. Each entry is {name, value, scope, description}. # scope=="global" values are interpolated into static_headers using ${NAME}. # scope=="user" values must be supplied per-user. - env_vars: list[dict[str, Any]] | None = None + env_vars: list[dict[str, object]] | None = None # OAuth-specific fields authorization_server_metadata: Mapping[str, object] | None = Field(default=None, exclude=True) client_id: str | None = None @@ -240,7 +240,7 @@ class MCPServer(LiteLLMBaseModel): # token_validation: key-value pairs that must match fields in the OAuth token # response (supports dot-notation for nested fields, e.g. "team.enterprise_id"). # Tokens that fail validation are rejected before storage. - token_validation: dict[str, Any] | None = None + token_validation: dict[str, object] | None = None oauth_identity_binding: MCPOAuthIdentityBinding | None = None # Optional TTL override (seconds) for the Redis per-user token cache, capped # at the token's expires_in minus the expiry buffer so a cached entry never diff --git a/litellm/types/prompts/init_prompts.py b/litellm/types/prompts/init_prompts.py index fc4c36e760a..c1a3ab56311 100644 --- a/litellm/types/prompts/init_prompts.py +++ b/litellm/types/prompts/init_prompts.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Any, Literal +from typing import Literal from pydantic import ConfigDict @@ -31,7 +31,7 @@ class PromptLiteLLMParams(LiteLLMBaseModel): api_base: str | None = None api_key: str | None = None - provider_specific_query_params: dict[str, Any] | None = None + provider_specific_query_params: dict[str, object] | None = None ignore_prompt_manager_model: bool | None = False ignore_prompt_manager_optional_params: bool | None = False @@ -66,7 +66,7 @@ class PromptSpec(LiteLLMBaseModel): class PromptTemplateBase(LiteLLMBaseModel): litellm_prompt_id: str content: str - metadata: dict[str, Any] | None = None + metadata: dict[str, object] | None = None class PromptInfoResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/callback_logs_endpoints.py b/litellm/types/proxy/callback_logs_endpoints.py index b2519d97122..0291cc902bc 100644 --- a/litellm/types/proxy/callback_logs_endpoints.py +++ b/litellm/types/proxy/callback_logs_endpoints.py @@ -5,7 +5,7 @@ External producers (e.g. the litellm-rust gateway) POST finished logging payloads here; the proxy replays them through the standard callback fan-out. """ -from typing import Any, Literal +from typing import Literal from pydantic import Field @@ -17,7 +17,7 @@ class CallbackLogRecord(LiteLLMBaseModel): """A single finished logging event to replay through the callbacks.""" status: Literal["success", "failure"] - standard_logging_payload: dict[str, Any] + standard_logging_payload: dict[str, object] error: str | None = None diff --git a/litellm/types/proxy/cloudzero_endpoints.py b/litellm/types/proxy/cloudzero_endpoints.py index ece294674b9..f87537494bd 100644 --- a/litellm/types/proxy/cloudzero_endpoints.py +++ b/litellm/types/proxy/cloudzero_endpoints.py @@ -46,7 +46,7 @@ class CloudZeroExportResponse(LiteLLMBaseModel): dry_run_data: dict[str, Any] | None = Field( None, description="Dry run data including usage data and CBF transformed data" ) - summary: dict[str, Any] | None = Field(None, description="Summary statistics for dry run") + summary: dict[str, object] | None = Field(None, description="Summary statistics for dry run") class CloudZeroSettingsView(LiteLLMBaseModel): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py index 60846b2a1bd..4f4bebf9704 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py @@ -1,5 +1,3 @@ -from typing import Any - from pydantic import Field from typing_extensions import TypedDict @@ -23,7 +21,7 @@ class AzurePromptShieldGuardrailResponse(TypedDict): """Configuration parameters for the Azure Prompt Shield guardrail""" userPromptAnalysis: UserPromptAnalysis - documentsAnalysis: list[dict[str, Any]] + documentsAnalysis: list[dict[str, object]] class AzurePromptShieldGuardrailConfigModel( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py index 90662e84ed5..6ee5741baf5 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py @@ -1,4 +1,4 @@ -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import Field from typing_extensions import Required, TypedDict @@ -37,7 +37,7 @@ class AzureTextModerationGuardrailResponseCategoriesAnalysis(TypedDict): class AzureTextModerationGuardrailResponse(TypedDict): """Response from the Azure Text Moderation guardrail""" - blocklistsMatch: list[dict[str, Any]] + blocklistsMatch: list[dict[str, object]] categoriesAnalysis: list[AzureTextModerationGuardrailResponseCategoriesAnalysis] diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py b/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py index 6598f1de62b..c0f5ea5346e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from pydantic import Field @@ -93,7 +93,7 @@ class CompresrGuardrailOptionalParams(LiteLLMBaseModel): "Unset lets the server default apply (~10.0)." ), ) - compression_params: dict[str, Any] | None = Field( + compression_params: dict[str, object] | None = Field( default=None, description=( "Passthrough of extra parameters forwarded verbatim in the Compresr " diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py b/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py index 78800a5e9a2..d49a3afae04 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Literal, TypedDict +from typing import Literal, TypedDict from pydantic import Field @@ -77,7 +77,7 @@ class CompetitorIntentDetection(TypedDict): action_hint: str entities: dict[str, list[str]] signals: list[str] - evidence: list[dict[str, Any]] + evidence: list[dict[str, object]] ContentFilterDetection = PatternDetection | BlockedWordDetection | CategoryKeywordDetection | CompetitorIntentDetection @@ -157,7 +157,7 @@ class LitellmContentFilterGuardrailConfigModel(GuardrailConfigModel): ) # Competitor intent blocker (generic; industry presets add domain_words, etc.) - competitor_intent_config: dict[str, Any] | None = Field( + competitor_intent_config: dict[str, object] | None = Field( default=None, description="Optional config for intent-based competitor comparison detection. " "Keys: brand_self (list), competitors (list), competitor_aliases (dict), " diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py index 9d1e375af72..6fe324ce6b0 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py @@ -1,6 +1,6 @@ -from typing import Any +from collections.abc import Mapping -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.guardrails import PiiEntityType @@ -17,5 +17,5 @@ class PresidioAnalyzeResponseItem(TypedDict, total=False): start: int | None end: int | None score: float | None - analysis_explanation: dict[str, Any] | None - recognition_metadata: dict[str, Any] | None + analysis_explanation: ReadOnly[Mapping[str, object] | None] + recognition_metadata: ReadOnly[Mapping[str, object] | None] diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 66702abfb34..213c911cfdc 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -1,7 +1,6 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import date from enum import Enum -from typing import Any from pydantic import ConfigDict, Field from typing_extensions import TypedDict @@ -61,7 +60,7 @@ class KeyMetricWithMetadata(MetricBase): class MetricWithMetadata(MetricBase): - metadata: dict[str, Any] = Field(default_factory=dict) + metadata: Mapping[str, object] = Field(default_factory=dict) # API key breakdown for this metric (e.g., which API keys are using this MCP server) api_key_breakdown: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata} diff --git a/litellm/types/proxy/management_endpoints/config_overrides.py b/litellm/types/proxy/management_endpoints/config_overrides.py index c82e616cf9a..b807b5537d5 100644 --- a/litellm/types/proxy/management_endpoints/config_overrides.py +++ b/litellm/types/proxy/management_endpoints/config_overrides.py @@ -1,5 +1,3 @@ -from typing import Any - from pydantic import Field from litellm.types.llms.base import LiteLLMBaseModel @@ -103,5 +101,5 @@ class ConfigOverrideSettingsResponse(LiteLLMBaseModel): """Response model for config override settings GET endpoints.""" config_type: str = Field(description="The type of config override") - values: dict[str, Any] = Field(description="Current configuration values (sensitive fields decrypted)") - field_schema: dict[str, Any] = Field(description="Schema information for UI rendering") + values: dict[str, object] = Field(description="Current configuration values (sensitive fields decrypted)") + field_schema: dict[str, object] = Field(description="Schema information for UI rendering") diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index febd4485d10..fc0b73f77b6 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import ConfigDict, Field, field_validator from typing_extensions import ReadOnly, TypedDict @@ -80,7 +80,7 @@ class UserUpdateResult(LiteLLMBaseModel): user_email: str | None = None success: bool error: str | None = None - updated_user: dict[str, Any] | None = None + updated_user: dict[str, object] | None = None class BulkUpdateUserResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index 6a4160c3c50..c012f872280 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -1,6 +1,5 @@ from collections.abc import Mapping from datetime import datetime -from typing import Any from pydantic import ConfigDict, Field @@ -22,7 +21,7 @@ class UpdateUsefulLinksRequest(LiteLLMBaseModel): # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) - useful_links: dict[str, str | dict[str, Any]] + useful_links: dict[str, str | dict[str, object]] class AutoRouterClassifierDefaultPromptResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 743b2d7c963..51c48e6ebf9 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import ConfigDict, Field, field_validator, model_validator @@ -121,8 +121,8 @@ class TeamMemberAddResult(LiteLLMBaseModel): user_email: str | None = None success: bool error: str | None = None - updated_user: dict[str, Any] | None = None - updated_team_membership: dict[str, Any] | None = None + updated_user: dict[str, object] | None = None + updated_team_membership: dict[str, object] | None = None class BulkTeamMemberAddResponse(LiteLLMBaseModel): @@ -133,7 +133,7 @@ class BulkTeamMemberAddResponse(LiteLLMBaseModel): total_requested: int successful_additions: int failed_additions: int - updated_team: dict[str, Any] | None = None + updated_team: dict[str, object] | None = None class TeamMemberRef(MemberDeleteRequest): diff --git a/litellm/types/proxy/prompt_endpoints.py b/litellm/types/proxy/prompt_endpoints.py index 930713663a1..cded9f5fb87 100644 --- a/litellm/types/proxy/prompt_endpoints.py +++ b/litellm/types/proxy/prompt_endpoints.py @@ -1,9 +1,7 @@ -from typing import Any - from litellm.types.llms.base import LiteLLMBaseModel class TestPromptRequest(LiteLLMBaseModel): dotprompt_content: str - prompt_variables: dict[str, Any] | None = None + prompt_variables: dict[str, object] | None = None conversation_history: list[dict[str, str]] | None = None diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index c487ce8b6c5..0e8a2bb3401 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Any, Literal +from typing import Literal from pydantic import ConfigDict @@ -13,7 +13,7 @@ class PublicModelHubInfo(LiteLLMBaseModel): # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) - useful_links: dict[str, str | dict[str, Any]] | None + useful_links: dict[str, str | dict[str, object]] | None class ProviderCredentialField(LiteLLMBaseModel): diff --git a/litellm/types/proxy/vantage_endpoints.py b/litellm/types/proxy/vantage_endpoints.py index 1833d8b8386..ccf050a8921 100644 --- a/litellm/types/proxy/vantage_endpoints.py +++ b/litellm/types/proxy/vantage_endpoints.py @@ -2,8 +2,8 @@ Vantage endpoint types for LiteLLM Proxy """ +from collections.abc import Mapping from datetime import datetime -from typing import Any from pydantic import Field, field_validator @@ -57,10 +57,10 @@ class VantageExportResponse(LiteLLMBaseModel): message: str status: str - dry_run_data: dict[str, Any] | None = Field( + dry_run_data: Mapping[str, object] | None = Field( None, description="Dry run data including usage data and FOCUS transformed data" ) - summary: dict[str, Any] | None = Field(None, description="Summary statistics for dry run") + summary: Mapping[str, object] | None = Field(None, description="Summary statistics for dry run") class VantageSettingsView(LiteLLMBaseModel): diff --git a/litellm/types/rag.py b/litellm/types/rag.py index 2f912d21587..02c06f242d1 100644 --- a/litellm/types/rag.py +++ b/litellm/types/rag.py @@ -3,7 +3,7 @@ Type definitions for RAG (Retrieval Augmented Generation) Ingest API. """ from collections.abc import Mapping -from typing import Any, Literal +from typing import Literal from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict @@ -232,7 +232,7 @@ class RAGIngestRequest(LiteLLMBaseModel): file_url: str | None = None # URL to fetch file from file_id: str | None = None # Existing file ID - ingest_options: dict[str, Any] # RAGIngestOptions as dict for flexibility + ingest_options: dict[str, object] # RAGIngestOptions as dict for flexibility model_config = ConfigDict(extra="allow") # Allow additional fields @@ -260,7 +260,7 @@ class RAGQueryRequest(LiteLLMBaseModel): """Request body for RAG query API.""" model: str - messages: list[Any] + messages: list[object] retrieval_config: RAGRetrievalConfig rerank: RAGRerankConfig | None = None stream: bool | None = False diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 5588527c141..72dc63a581f 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -5,7 +5,7 @@ from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch from pydantic import ConfigDict, PrivateAttr -from typing_extensions import Any, TypedDict +from typing_extensions import TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -100,7 +100,7 @@ class OutputCodeInterpreterCall(BaseLiteLLMOpenAIResponseObject): def build_code_interpreter_log_outputs( - content: Any, + content: object, ) -> list[OutputCodeInterpreterCallLog] | None: """Convert Anthropic bash_code_execution stdout/stderr to log outputs. diff --git a/litellm/types/vector_store_files.py b/litellm/types/vector_store_files.py index 6b4953965d5..3623746216a 100644 --- a/litellm/types/vector_store_files.py +++ b/litellm/types/vector_store_files.py @@ -1,7 +1,9 @@ +import builtins +from collections.abc import Mapping from enum import Enum -from typing import Any, Literal +from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class VectorStoreFileStatus(str, Enum): @@ -33,7 +35,7 @@ class VectorStoreFileObject(TypedDict, total=False): usage_bytes: int | None vector_store_id: str status: VectorStoreFileStatus - last_error: dict[str, Any] | None + last_error: ReadOnly[Mapping[str, builtins.object] | None] chunking_strategy: VectorStoreFileChunkingStrategy | None attributes: dict[str, str | int | float | bool] | None @@ -83,5 +85,5 @@ class VectorStoreFileContentResponse(TypedDict, total=False): class VectorStoreFileAuthCredentials(TypedDict, total=False): - headers: dict[str, Any] - query_params: dict[str, Any] + headers: ReadOnly[Mapping[str, object]] + query_params: ReadOnly[Mapping[str, object]] diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index 5e7d517ece2..2259339e479 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -31,7 +31,7 @@ class LiteLLM_ManagedVectorStore(TypedDict, total=False): vector_store_name: str | None vector_store_description: str | None - vector_store_metadata: dict[str, Any] | str | None + vector_store_metadata: dict[str, object] | str | None created_at: datetime | None updated_at: datetime | None diff --git a/litellm/utils.py b/litellm/utils.py index 7de7ce6188d..809a94cd93e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields @@ -218,6 +219,7 @@ def _get_cached_audio_utils(): return _audio_utils_module +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -262,13 +264,17 @@ from litellm.types.utils import ( SearchProviders, SelectTokenizerResponse, StreamingChoices, - TextChoices, - TextCompletionResponse, TranscriptionResponse, Usage, credential_api_key_secret_reference, is_litellm_owned_kwarg, ) +from litellm.types.utils import ( + TextChoices as _TextChoices, +) +from litellm.types.utils import ( + TextCompletionResponse as _TextCompletionResponse, +) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} _BACKFILL_MODES: Final = frozenset({"chat", "responses"}) @@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError): claude_json_str = json.dumps(json_data) import importlib.metadata from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never +TextChoices: TypeAlias = _TextChoices +TextCompletionResponse: TypeAlias = _TextCompletionResponse +TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper + from litellm import utils as litellm_utils # These are lazy loaded via __getattr__ @@ -340,7 +350,8 @@ if TYPE_CHECKING: # Type stubs for lazy-loaded functions to help mypy understand their types # These imports allow mypy to understand the types when these are accessed via __getattr__ - from litellm.litellm_core_utils.exception_mapping_utils import exception_type + exception_type: Callable[..., Exception] + from litellm.litellm_core_utils.get_litellm_params import ( get_base_model_from_litellm_call_metadata, get_litellm_params, @@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) - if kind is None: + if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): return None backend: Final = _huggingface_tokenizer_backend() key: Final = _huggingface_tokenizer_failure_key(kind, backend) @@ -7945,88 +7956,6 @@ def register_prompt_template( return litellm.custom_prompt_dict -class TextCompletionStreamWrapper: - def __init__( - self, - completion_stream, - model, - stream_options: dict | None = None, - custom_llm_provider: str | None = None, - ): - self.completion_stream = completion_stream - self.model = model - self.stream_options = stream_options - self.custom_llm_provider = custom_llm_provider - - def __iter__(self): - return self - - def __aiter__(self): - return self - - def convert_to_text_completion_object(self, chunk: ModelResponse): - try: - response: Final = TextCompletionResponse() - response["id"] = chunk.get("id", None) - response["object"] = "text_completion" - response["created"] = chunk.get("created", None) - response["model"] = chunk.get("model", None) - text_choices: Final = TextChoices() - if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices - raise Exception - - if chunk["choices"]: - delta: Final = chunk["choices"][0]["delta"] - text_choices["text"] = delta["content"] - text_choices["reasoning_content"] = delta.get("reasoning_content") - text_choices["index"] = chunk["choices"][0]["index"] - text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] - response["choices"] = [text_choices] - else: - response["choices"] = [] - - # only pass usage when stream_options["include_usage"] is True - if self.stream_options and self.stream_options.get("include_usage", False) is True: - response["usage"] = chunk.get("usage", None) - - return response - except Exception as e: - raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") - - def __next__(self): - # model_response = ModelResponse(stream=True, model=self.model) - TextCompletionResponse() - try: - for chunk in self.completion_stream: - if chunk == "None" or chunk is None: - raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk - raise StopIteration - except StopIteration: - raise StopIteration - except Exception as e: - exception_type: Final = litellm_utils.exception_type - raise exception_type( - model=self.model, - custom_llm_provider=self.custom_llm_provider or "", - original_exception=e, - completion_kwargs={}, - extra_kwargs={}, - ) - - async def __anext__(self): - try: - async for chunk in self.completion_stream: - if chunk == "None" or chunk is None: - raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk - raise StopIteration - except StopIteration: - raise StopAsyncIteration - - def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream: return ModelResponseStream( id=model_response.id, diff --git a/migrations/Dockerfile b/migrations/Dockerfile index b39a9bfa9bf..84ea67fcc5f 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -61,7 +61,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -118,7 +118,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \ + apk add --no-cache bash openssl libssl3 tzdata python-3.13 nodejs libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb9bfa6ffe4..14fe355a805 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12827,6 +12827,7 @@ "supports_vision": true }, "azure_ai/Phi-4": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -12840,6 +12841,7 @@ "supports_vision": false }, "azure_ai/Phi-4-mini-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12851,6 +12853,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -12865,6 +12868,7 @@ "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12876,6 +12880,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -24892,27 +24897,118 @@ ] }, "elevenlabs/eleven_v3": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] }, "elevenlabs/eleven_multilingual_v2": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4": { + "input_cost_per_character": 8e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4_turbo": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v3_conversational": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] @@ -36861,17 +36957,17 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, - "cache_creation_input_token_cost_decisions": 1e-07, + "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, - "cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07, + "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-08, "cache_creation_input_token_cost_priority": 2.5e-07, "cache_read_input_token_cost": 1e-08, - "cache_read_input_token_cost_decisions": 1e-07, + "cache_read_input_token_cost_decisions": 1e-08, "cache_read_input_token_cost_above_272k_tokens": 2e-08, - "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, "cache_read_input_token_cost_batches": 5e-09, @@ -42337,6 +42433,134 @@ ], "supports_audio_output": true }, + "mistral/voxtral-mini-transcribe-3": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-3-0": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3-0": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3-0": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-3": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-mini-tts-3-0": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "mistral/voxtral-small-2507": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -45623,6 +45847,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -45648,6 +45873,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -48499,6 +48725,17 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.together.ai/pricing", + "supports_prompt_caching": true + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -49352,6 +49589,7 @@ "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49371,6 +49609,7 @@ "supports_tool_choice": true }, "us-gov.openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -52906,12 +53145,14 @@ }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, + "output_cost_per_token_batches": 5.75e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -52926,12 +53167,14 @@ }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, + "output_cost_per_token_batches": 3.5e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -59917,6 +60160,7 @@ } }, "bedrock_mantle/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -59942,6 +60186,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -59967,6 +60212,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7e-08, "input_cost_per_token_flex": 7e-08, @@ -59990,6 +60236,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -67841,6 +68088,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -67860,6 +68108,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68138,6 +68387,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68157,6 +68407,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68545,6 +68796,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68564,6 +68816,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -68704,6 +68957,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68723,6 +68977,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -73330,6 +73585,7 @@ ] }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/osv-scanner.toml b/osv-scanner.toml index bb802137f6e..33e2f614edc 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v" ignoreUntil = 2026-11-01 reason = "diskcache has no fixed release published; remove this entry once one exists" +[[IgnoredVulns]] +id = "GHSA-v4x8-3wr5-3wq8" +ignoreUntil = 2026-11-20 +reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" + [[IgnoredVulns]] id = "GHSA-vfj7-8cjw-p6xm" ignoreUntil = 2026-11-03 diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 2add6868bd9..890fa0fd773 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", + "certifi>=2026.10.19", "typing-extensions>=4.13.0,<5.0", ] diff --git a/pyproject.toml b/pyproject.toml index 9076760153a..817c50df90b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ dependencies = [ "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", - "boto3>=1.43.1,<2.0", + "boto3>=1.43.117,<2.0", + "certifi>=2026.10.19", "typing-extensions>=4.13.0,<5.0", ] @@ -72,7 +73,7 @@ proxy = [ "cryptography>=49.0.0,<51.0", "pynacl>=1.6.2,<2.0", "websockets>=15.0.1,<16.0", - "boto3>=1.43.1,<2.0", + "boto3>=1.43.117,<2.0", "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=2.3.0,<3", @@ -136,10 +137,10 @@ semantic-router = [ "semantic-router>=0.1.15,<1.0; python_version < '3.14'", "aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'", ] -mlflow = ["mlflow>=3.11.1,<4.0"] +mlflow = ["mlflow>=3.17.2,<4.0"] grpc = [ - # Newest non-yanked release older than the 30-day cutoff. - "grpcio==1.78.0", + # First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff + "grpcio==1.84.1", ] stt-vertex-chirp = [ # Google Cloud Speech-to-Text v2 streaming (gRPC) for Chirp models on @@ -173,19 +174,19 @@ proxy-runtime = [ "google-cloud-speech>=2.40.0,<3.0", "google-genai>=1.37.0,<2.0", "anthropic[vertex]>=0.84.0,<1.0", - "grpcio==1.78.0", + "grpcio==1.84.1", "prometheus-client>=0.20.0,<1.0", "langfuse>=4.7,<5.0", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "ddtrace>=4.8.2,<5.0", "sentry-sdk>=2.21.0,<3.0", "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.16.1,<7.0", + "pypdf>=6.21.2,<7.0", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] @@ -222,10 +223,10 @@ dev = [ "types-PyYAML==6.0.12.20250915", "botocore-stubs==1.43.14", "types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "langfuse>=4.7,<5.0", "fastapi-offline==1.7.6", "fakeredis==2.34.1", @@ -254,10 +255,10 @@ proxy-dev = [ "hypercorn==0.17.3", "prometheus-client==0.20.0", "sentry-sdk==2.21.0", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "azure-identity==1.25.2", "a2a-sdk==1.1.0", "backports-zstd==1.8.0; python_version < '3.14'", @@ -279,7 +280,7 @@ ci = [ "langchain==1.3.9", "lunary==1.4.36; python_version == '3.10'", "lunary==1.4.37; python_version >= '3.11'", - "logfire==4.6.0", + "logfire==5.2.0", "traceloop-sdk==0.34.0", "detect-secrets==1.5.0", "PyGithub==2.8.1", @@ -358,7 +359,7 @@ constraint-dependencies = [ "packaging>=24.0", "soupsieve>=2.8.4", "httplib2>=0.32.0", - "setuptools>=83.0.0", + "setuptools>=84.1.0", ] override-dependencies = [ # a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0. @@ -370,6 +371,23 @@ default-groups = ["dev"] required-version = ">=0.10.9" exclude-newer = "3 days" +[tool.uv.exclude-newer-package] +# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window +certifi = "2026-10-20T03:00:00Z" +# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window +grpcio = "2026-10-22T15:00:00Z" +# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window +boto3 = "2026-10-21T19:31:00Z" +botocore = "2026-10-21T19:31:00Z" +# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window +brotli = "2026-10-22T14:01:00Z" +# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window +mlflow = "2026-10-21T03:00:00Z" +mlflow-skinny = "2026-10-21T03:00:00Z" +mlflow-tracing = "2026-10-21T03:00:00Z" +# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window +pypdf = "2026-10-21T17:00:00Z" + [tool.uv.sources] litellm-proxy-extras = { workspace = true } litellm-enterprise = { workspace = true } diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index e533907659b..cd951c6cd28 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -176,6 +176,7 @@ langgraph: >=1.0.10 # MIT License langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE hypothesis: >=6.165.10 # MPL 2.0 license pytest-rerunfailures: >=15.1 # MPL 2.0 license +certifi: >=2026.10.19 # MPL 2.0 license - https://github.com/certifi/python-certifi/blob/master/LICENSE pytest-recording: >=0.13.4 # MIT license expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE typing-extensions: >=4.13.0 # PSF-2.0 license - https://github.com/python/typing_extensions/blob/main/LICENSE diff --git a/tests/integration/compatibility/test_status_code_followups_usage_chat.py b/tests/integration/compatibility/test_status_code_followups_usage_chat.py deleted file mode 100644 index de9ec16cf30..00000000000 --- a/tests/integration/compatibility/test_status_code_followups_usage_chat.py +++ /dev/null @@ -1,125 +0,0 @@ -"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check -the daily activity routes answer 400 with, instead of a generic fetch error.""" - -from __future__ import annotations - -import json -import uuid -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Final - -import pytest -from integration._support.client import JSON_OBJECT, Gateway, object_value -from integration._support.upstream import delete_scenario, register_scenario -from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway -from pydantic import JsonValue - -pytestmark: Final = pytest.mark.timeout(180) - -_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data") - - -@dataclass(frozen=True, slots=True) -class _UsageChat: - gateway: Gateway - identity: str - - -@pytest.fixture(scope="module") -def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]: - """A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream.""" - identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}" - handle: Final = register_scenario(identity, json_response({})) - try: - with owned_gateway( - tmp_path_factory.mktemp("audit-usage-chat"), - {"model_list": [], "router_settings": {"num_retries": 0}}, - environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity}, - ) as owned: - yield _UsageChat(owned, identity) - finally: - delete_scenario(handle) - - -def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]: - arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date}) - return { - "id": "chatcmpl-usage", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}} - for tool in _TOOLS - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - - -def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]: - """Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up - model call carried, keyed by tool, and the tool statuses the SSE stream reported.""" - register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date))) - Upstream(usage_chat.gateway.upstream_url).reset() - with usage_chat.gateway.client.stream( - "POST", - "/usage/ai/chat", - json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]}, - headers={"Authorization": f"Bearer {usage_chat.gateway.key}"}, - timeout=60, - ) as response: - assert response.status_code == 200 - events: Final = tuple( - object_value(JSON_OBJECT.validate_json(line.removeprefix("data: "))) - for line in response.iter_lines() - if line.startswith("data: ") - ) - calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity) - assert len(calls) == 2, calls - messages: Final = object_value(calls[1]["body"]).get("messages") - assert isinstance(messages, list), calls[1] - tool_messages: Final = { - str(message["tool_call_id"]).removeprefix("call-"): str(message["content"]) - for message in messages - if isinstance(message, dict) and message.get("role") == "tool" - } - statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call") - return tool_messages, statuses - - -_REVERSED: Final = "Invalid date range: end_date must be on or after start_date" -_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates" -_NO_TEAM_DATA: Final = "No Team usage data found for the given date range." -_NO_TAG_DATA: Final = "No Tag usage data found for the given date range." - - -@pytest.mark.parametrize( - ("start_date", "end_date", "team_message", "tag_message"), - ( - pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"), - pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"), - pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"), - pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"), - ), -) -def test_usage_chat_tools_answer_with_the_query_layers_date_check( - usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str -) -> None: - """The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query - takes these dates without that check, still answers with its usage summary.""" - tool_messages, statuses = _tool_results(usage_chat, start_date, end_date) - assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages - assert tool_messages["get_team_usage_data"] == team_message, tool_messages - assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages - assert statuses == ("running", "complete") * len(_TOOLS), statuses diff --git a/tests/integration/management/test_budget_list_sorting.py b/tests/integration/management/test_budget_list_sorting.py new file mode 100644 index 00000000000..aae0bd6e274 --- /dev/null +++ b/tests/integration/management/test_budget_list_sorting.py @@ -0,0 +1,94 @@ +from datetime import datetime, timezone +from itertools import chain +from typing import Final +from uuid import uuid4 + +import pytest + +from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from +from tests.integration._support.client import Gateway, Scenario + + +@pytest.mark.parametrize("descending", (False, True)) +def test_budget_reset_sort_orders_durations_before_pagination( + descending: bool, + gateway: Gateway, + scenario: Scenario, +) -> None: + prefix: Final = f"sort-{uuid4().hex}" + durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "５d", "𝟝d", "5d", None) + ids: Final = tuple( + scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10) + for index, duration in enumerate(durations) + ) + now: Final = datetime.now(timezone.utc) + expected: Final = tuple( + budget_id + for _, budget_id in sorted( + zip(durations, ids, strict=True), + key=lambda item: ( + item[0] is None, + duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0, + item[1], + ), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/management/v1/budgets", + params={ + "q": prefix, + "sort": "-budget_duration" if descending else "budget_duration", + "page_size": "3", + "page": str(page), + }, + ) + for page in range(1, -(-len(ids) // 3) + 1) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected + filtered: Final = gateway.request( + "GET", + "/management/v1/budgets", + params={ + "q": prefix, + "sort": "budget_duration", + "filter[budget_duration][in]": "7d,30d", + }, + ) + assert filtered.status_code == 200, filtered.text + assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"] + + +@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")) +@pytest.mark.parametrize("descending", (False, True)) +def test_budget_limits_sort_unlimited_as_infinity( + field: str, descending: bool, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-unlimited-{uuid4().hex}" + values: Final = (20, None, 0, 3) + ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values)) + expected: Final = tuple( + identity + for value, identity in sorted( + zip(values, ids, strict=True), + key=lambda item: float("inf") if item[0] is None else item[0], + reverse=descending, + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/management/v1/budgets", + params={ + "q": prefix, + "sort": ("-" if descending else "") + field, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected diff --git a/tests/integration/management/test_key_list.py b/tests/integration/management/test_key_list.py index b70a3782eda..4634314817b 100644 --- a/tests/integration/management/test_key_list.py +++ b/tests/integration/management/test_key_list.py @@ -1,11 +1,16 @@ -from typing import FrozenSet +from datetime import datetime, timedelta, timezone +from itertools import chain +from typing import Final, FrozenSet import pytest from litellm.proxy.utils import hash_token -from tests.integration._support.actors import TEAM_ALPHA, Actor +from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources from tests.integration._support.actors import create_scratch_key +from tests.integration._support.client import Gateway, Scenario +from litellm.repositories.table_repositories import DeletedVerificationTokenRepository +from litellm.repositories.verification_token_repository import VerificationTokenRepository pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor }, ) assert resp.status_code == 403, resp.text + + +@pytest.mark.parametrize( + "field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit") +) +@pytest.mark.parametrize("direction", ("asc", "desc")) +@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted")) +async def test_key_list_sorts_across_pages_with_nulls_last( + field: str, + direction: str, + deleted: bool, + gateway: Gateway, + scenario: Scenario, + db: ManagementDatabase, + resources: ScratchResources, +) -> None: + team_admin: Final = scenario.user(user_role="internal_user") + owner: Final = scenario.user(user_role="internal_user") + team_id: Final = scenario.team( + members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}] + ) + team_admin_key: Final = scenario.key(user_id=team_admin) + table: Final = ( + DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table + ) + values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None) + tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values))) + rows: Final = [ + { + "token": token, + "key_alias": resources.prefix, + "team_id": team_id, + "user_id": owner, + field: ( + datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value) + if field in ("expires", "last_active", "budget_reset_at") and value is not None + else value + ), + } + for token, value in zip(tokens, values, strict=True) + ] + await table.create_many(rows) + try: + valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None) + expected: Final = [ + token + for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1])) + ] + [token for value, token in zip(values, tokens, strict=True) if value is None] + for caller in (gateway.key, team_admin_key): + pages: Final = [ + gateway.client.get( + "/key/list", + headers={"Authorization": f"Bearer {caller}"}, + params={ + "key_alias": resources.prefix, + "team_id": team_id, + "include_team_keys": "true", + "sort_by": field, + "sort_order": direction, + "size": "2", + "page": str(page), + **({"status": "deleted"} if deleted else {}), + }, + ) + for page in (1, 2, 3) + ] + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total_count"] == len(tokens) for page in pages) + assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected + finally: + await table.delete_many(where={"token": {"in": tokens}}) diff --git a/tests/integration/management/test_sso_sign_in.py b/tests/integration/management/test_sso_sign_in.py index c0c03571a13..d0b6e2579c8 100644 --- a/tests/integration/management/test_sso_sign_in.py +++ b/tests/integration/management/test_sso_sign_in.py @@ -3,7 +3,9 @@ import contextlib import json import uuid from collections.abc import Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from functools import partial from pathlib import Path from typing import Final from urllib.parse import parse_qs, urlencode, urlparse @@ -15,7 +17,7 @@ from pydantic import JsonValue from tests.integration._support.client import Gateway, gateway_from_environment from tests.integration._support.database import read_rows -from tests.integration._support.process import owned_proxy +from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy from tests.integration._support.wire import Reply, Request, wire_server pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state] @@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role( "metadata": {"auth_provider": IDP_PROVIDER}, } ] + + +def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value) + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + with _sso_gateway(tmp_path, {}) as gateway: + for index in range(4): + signed_in: Final = _sign_in(gateway, f"free-sso-{index}") + assert signed_in.status_code == 303, signed_in.text + + contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8)) + with ThreadPoolExecutor(max_workers=len(contenders)) as pool: + responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders)) + + assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403] + assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}] + for subject, response in zip(contenders, responses): + if response.status_code == 403: + assert "requires a LiteLLM Enterprise license" in response.text + assert not response.cookies + assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == [] + + existing: Final = _sign_in(gateway, "free-sso-0") + assert existing.status_code == 303, existing.text + + admin_created: Final = gateway.request( + "POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False} + ) + assert admin_created.status_code == 200, admin_created.text + above_capacity: Final = _sign_in(gateway, "free-sso-0") + refused: Final = _sign_in(gateway, "free-sso-extra") + assert (above_capacity.status_code, refused.status_code) == (303, 403) + assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}] diff --git a/tests/integration/management/test_team_list_sorting.py b/tests/integration/management/test_team_list_sorting.py new file mode 100644 index 00000000000..c09e51f0dfe --- /dev/null +++ b/tests/integration/management/test_team_list_sorting.py @@ -0,0 +1,244 @@ +import json +from itertools import chain +from typing import Final +from uuid import uuid4 + +import pytest + +from tests.integration._support.client import Gateway, Scenario +from tests.integration._support.database import write_rows + + +@pytest.mark.parametrize( + "field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count") +) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_team_list_sorts_across_pages_and_respects_membership( + field: str, + direction: str, + gateway: Gateway, + scenario: Scenario, +) -> None: + prefix: Final = f"sort-{uuid4().hex}" + owner: Final = scenario.user(user_role="internal_user") + caller: Final = scenario.key(user_id=owner) + values: Final = (20, 3, 3, 100, 0) + teams: Final = tuple( + scenario.team( + team_id=f"{prefix}-{index}", + team_alias=f"{prefix}-{index}", + members_with_roles=[{"user_id": owner, "role": "admin"}], + ) + for index in range(len(values)) + ) + outsider: Final = scenario.team(team_alias=f"{prefix}-outside") + for team_id, value in zip(teams, values, strict=True): + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, ' + "tpm_limit = %s::bigint, rpm_limit = %s::bigint, " + "updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', " + "members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) " + "WHERE team_id = %s", + ( + str(value), + str(value), + str(value), + str(value), + str(value), + json.dumps( + [{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)] + ), + str(value), + team_id, + ), + ) + write_rows( + 'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) ' + "SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n", + (team_id, team_id, str(value)), + ) + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s', + (teams[-1],), + ) + try: + expected: Final = tuple( + team_id + for value, team_id in sorted( + zip(values, teams, strict=True), + key=lambda item: ( + (item[0] != 0 if direction == "desc" else item[0] == 0) + if field in ("max_budget", "tpm_limit", "rpm_limit") + else False, + item[0] * (1 if direction == "asc" else -1), + item[1], + ), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/v2/team/list", + key=caller, + params={ + "search": prefix, + "user_id": owner, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2, 3) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) + assert returned == expected + assert outsider not in returned + assert all(page.json()["total"] == len(teams) for page in pages) + finally: + for team_id in teams: + write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,)) + + +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_team_list_sorts_organization_names_and_missing_values( + direction: str, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-org-{uuid4().hex}" + budget: Final = scenario.budget(max_budget=10) + organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", "")) + teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4)) + try: + for (organization_id, alias), team_id in zip(organizations, teams): + write_rows( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) " + "VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())", + (organization_id, alias, budget), + ) + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s', + (organization_id, team_id), + ) + ascending: Final = (teams[0], teams[2], teams[1]) + ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending)) + pages: Final = tuple( + gateway.request( + "GET", + "/v2/team/list", + params={ + "search": prefix, + "sort_by": "organization_alias", + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1]) + finally: + for team_id in teams: + write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,)) + for organization_id, _ in organizations: + write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,)) + + +@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count")) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_computed_team_sorts_preserve_org_and_membership_union( + field: str, direction: str, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-org-scope-{uuid4().hex}" + owner: Final = scenario.user(user_role="internal_user") + budget: Final = scenario.budget(max_budget=10) + orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside") + teams: Final = tuple( + scenario.team( + team_id=f"{prefix}-{i}", + team_alias=f"{prefix}-{i}", + members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [], + ) + for i in range(4) + ) + try: + for org, alias in zip(orgs, ("alpha", "zulu"), strict=True): + write_rows( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) " + "VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())", + (org, alias, budget), + ) + write_rows( + 'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) ' + "VALUES (%s, %s, 'org_admin')", + (owner, orgs[0]), + ) + for index, team in enumerate(teams): + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, ' + "models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s", + ( + orgs[0] if index < 2 else orgs[1], + json.dumps( + [ + {"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} + for i in range(index + 1) + ] + ), + str(index + 1), + team, + ), + ) + write_rows( + 'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) ' + "SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n", + (team, team, str(index + 1)), + ) + caller: Final = scenario.key(user_id=owner) + values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3) + expected: Final = tuple( + team + for value, team in sorted( + zip(values, teams[:3], strict=True), + key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/v2/team/list", + key=caller, + params={ + "user_id": owner, + "search": prefix, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total"] == 3 for page in pages) + assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected + denied: Final = gateway.request( + "GET", + "/v2/team/list", + key=caller, + params={ + "organization_id": orgs[1], + "sort_by": field, + }, + ) + assert denied.status_code == 403, denied.text + finally: + for team in teams: + write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) + write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,)) + for org in orgs: + write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,)) + write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,)) diff --git a/tests/integration/management/test_user_list_sorting.py b/tests/integration/management/test_user_list_sorting.py new file mode 100644 index 00000000000..f5578140e1c --- /dev/null +++ b/tests/integration/management/test_user_list_sorting.py @@ -0,0 +1,179 @@ +from itertools import chain +from typing import Final +from uuid import uuid4 + +import pytest + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from tests.integration._support.client import Gateway, Scenario +from tests.integration._support.database import write_rows + + +@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count")) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None: + prefix: Final = f"sort-{uuid4().hex}" + ids: Final = tuple( + scenario.user( + user_id=f"{prefix}-{index}", + user_role="internal_user", + user_alias=f"Alias {value if index != 2 else 4:03}", + user_email=f"{prefix}-{index}@example.com", + max_budget=value, + metadata={"scim_active": value != 3}, + ) + for index, value in enumerate((20, 3, 3, 100, 0)) + ) + for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True): + write_rows( + 'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, ' + "updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s", + (f"{prefix}-{value:03}", str(value), user_id), + ) if value != 3 or user_id == ids[1] else write_rows( + "UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s", + (f"{prefix}-004", user_id), + ) + for index in range(value if value < 4 else 1): + scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}") + write_rows( + 'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s', + (ids[-1],), + ) + scenario.key(user_id=ids[-1]) + write_rows( + 'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1]) + ) + values: Final = ( + (0, 1, 1, 0, 0) + if field == "status" + else (1, 3, 3, 1, 0) + if field == "key_count" + else (20, 3, 4, 100, None) + if field == "sso_user_id" + else (20, 3, 4, 100, 0) + if field == "user_alias" + else (20, 3, 3, 100, None) + ) + expected: Final = tuple( + user_id + for _, user_id in sorted( + ((value, user_id) for value, user_id in zip(values, ids, strict=True)), + key=lambda item: ( + (item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None), + (item[0] or 0) * (1 if direction == "asc" else -1), + item[1], + ), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/user/list", + params={ + "search": prefix, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2, 3) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total"] == len(ids) for page in pages) + assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected + + +@pytest.mark.parametrize("field", ("status", "key_count")) +def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None: + team: Final = scenario.team() + member: Final = scenario.member(team) + other: Final = scenario.user(user_role="internal_user") + for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}): + baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"}) + response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field}) + assert response["total"] == baseline["total"] + assert sorted(user["user_id"] for user in response["users"]) == sorted( + user["user_id"] for user in baseline["users"] + ) + ordinary_key: Final = scenario.key(user_id=other) + denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field}) + assert denied.status_code == 403, denied.text + + +@pytest.mark.parametrize("field", ("status", "key_count")) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_computed_user_sorts_preserve_org_admin_scope( + field: str, direction: str, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-org-users-{uuid4().hex}" + owner: Final = scenario.user(user_role="internal_user") + members: Final = tuple( + scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1}) + for index in range(4) + ) + budget: Final = scenario.budget(max_budget=10) + orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside") + try: + for org in orgs: + write_rows( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) " + "VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())", + (org, org, budget), + ) + write_rows( + 'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) ' + "VALUES (%s, %s, 'org_admin')", + (owner, orgs[0]), + ) + for index, member in enumerate(members): + write_rows( + 'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) ' + "VALUES (%s, %s, 'internal_user')", + (member, orgs[0] if index < 3 else orgs[1]), + ) + for _ in range(index): + scenario.key(user_id=member) + caller: Final = scenario.key(user_id=owner) + values: Final = (0, 1, 0) if field == "status" else (0, 1, 2) + expected: Final = tuple( + member + for value, member in sorted( + zip(values, members[:3], strict=True), + key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/user/list", + key=caller, + params={ + "search": prefix, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total"] == 3 for page in pages) + returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages)) + assert returned == expected + denied: Final = gateway.request( + "GET", + "/user/list", + key=caller, + params={ + "organization_ids": orgs[1], + "sort_by": field, + }, + ) + assert denied.status_code == 403, denied.text + finally: + for org in orgs: + write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,)) + write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,)) diff --git a/tests/integration/security/test_offline_mode.py b/tests/integration/security/test_offline_mode.py new file mode 100644 index 00000000000..11012eb14bc --- /dev/null +++ b/tests/integration/security/test_offline_mode.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, object_value +from integration._support.forward_proxy import refusing_forward_proxy +from integration._support.process import owned_proxy +from pydantic import JsonValue + +_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm" +_CONTROL_HOST: Final = "provider.offline-mode.example" +_REMOTE_URLS: Final = { + "LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json", + "LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json", + "LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json", + "LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml", +} +_SELF_INITIATED_HOSTS: Final = frozenset( + { + "cost-map.offline-mode.example:443", + "beta-headers.offline-mode.example:443", + "autorouter.offline-mode.example:443", + "blog.offline-mode.example:443", + "raw.githubusercontent.com:443", + "docs.litellm.ai:443", + "api.pwnedpasswords.com:443", + "huggingface.co:443", + } +) +_PER_FEATURE_FLAGS: Final = ( + "LITELLM_OFFLINE_MODE", + "LITELLM_LOCAL_MODEL_COST_MAP", + "LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", + "LITELLM_LOCAL_AUTOROUTER_PRESETS", + "LITELLM_LOCAL_POLICY_TEMPLATES", + "LITELLM_LOCAL_BLOG_POSTS", + "LITELLM_DISABLE_LATEST_RELEASE_CHECK", + "LITELLM_LICENSE", +) + + +def _bundled(name: str) -> JsonValue: + return json.loads((_PACKAGE / name).read_text(encoding="utf-8")) + + +def _write_config(directory: Path) -> Path: + config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "offline-mode-control", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"https://{_CONTROL_HOST}/v1", + "api_key": "synthetic-provider-key", + }, + }, + { + "model_name": "offline-mode-kimi", + "litellm_params": { + "model": "bedrock/us.moonshotai.kimi-k3", + "aws_region_name": "us-east-1", + "aws_access_key_id": "synthetic-access-key", + "aws_secret_access_key": "synthetic-secret-key", + }, + }, + ], + "router_settings": {"num_retries": 0}, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return config + + +def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue: + response: Final = proxy.request(method, path, body) + assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}" + return json.loads(response.content) + + +@pytest.mark.timeout(300) +@pytest.mark.parametrize( + ("offline_mode", "expected_hosts", "reload_status"), + [ + pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"), + pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"), + ], +) +def test_offline_mode_stops_every_self_initiated_request( + gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int +) -> None: + with refusing_forward_proxy() as egress: + environment: Final = { + "HTTPS_PROXY": egress.url, + "HTTP_PROXY": egress.url, + "ALL_PROXY": egress.url, + "NO_PROXY": "127.0.0.1,localhost", + "AIOHTTP_TRUST_ENV": "True", + **_REMOTE_URLS, + **({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}), + } + with owned_proxy( + gateway, + tmp_path, + environment, + config=_write_config(tmp_path), + remove_environment=_PER_FEATURE_FLAGS, + workers=2, + ) as isolated: + source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source")) + reloaded: Final = isolated.request("POST", "/reload/model_cost_map") + beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers")) + release: Final = _json(isolated, "GET", "/get/latest_release_info") + presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets")) + templates: Final = _json(isolated, "GET", "/policy/templates") + blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts")) + user: Final = object_value( + _json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"}) + ) + password_set: Final = object_value( + _json( + isolated, + "POST", + "/user/update", + {"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"}, + ) + ) + tokens: Final = object_value( + _json( + isolated, + "POST", + "/utils/token_counter", + {"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]}, + ) + ) + kimi_tokens: Final = object_value( + _json( + isolated, + "POST", + "/utils/token_counter", + {"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]}, + ) + ) + control: Final = isolated.request( + "POST", + "/v1/chat/completions", + {"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]}, + ) + reached: Final = frozenset(egress.targets()) + + assert reached == expected_hosts + assert control.status_code != 200, control.text + assert source["source"] == "local", source + assert source["is_env_forced"] is (offline_mode is not None), source + assert reloaded.status_code == reload_status, reloaded.text + assert beta_headers["status"] == "success", beta_headers + assert release is None + assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json"))) + assert templates == _bundled("policy_templates_backup.json") + assert [object_value(post)["url"] for post in _list(blog["posts"])] == [ + object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"]) + ] + assert password_set["user_id"] == user["user_id"], password_set + assert tokens["tokenizer_type"] == "openai_tokenizer", tokens + assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens + + +def _list(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value diff --git a/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py b/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py index 780ee43fb3f..0c18f2a4b6f 100644 --- a/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py +++ b/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py @@ -99,16 +99,9 @@ class _Activity(BaseModel): results: tuple[_Day, ...] -class _UpstreamMessage(BaseModel): - model_config = ConfigDict(frozen=True) - role: str | None = None - - class _UpstreamRequest(BaseModel): model_config = ConfigDict(frozen=True) stream: bool | None = None - tools: tuple[JsonValue, ...] | None = None - messages: tuple[_UpstreamMessage, ...] = () @dataclass(frozen=True, slots=True) @@ -171,17 +164,6 @@ def _stream_frames() -> tuple[bytes, ...]: return (*(f"data: {json.dumps({**envelope, **frame})}\n\n".encode() for frame in frames), b"data: [DONE]\n\n") -def _usage_tool_call() -> Mapping[str, JsonValue]: - arguments: Final = json.dumps({"start_date": _day(-1), "end_date": _day(1)}) - return { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_usage", "type": "function", "function": {"name": "get_usage_data", "arguments": arguments}} - ], - } - - def _response_object() -> bytes: return json.dumps( { @@ -216,9 +198,6 @@ def _respond(request: Request) -> Reply: body: Final = _UpstreamRequest.model_validate_json(request.body or b"{}") if body.stream: return Reply(content_type="text/event-stream", chunks=_stream_frames()) - roles: Final = frozenset(message.role for message in body.messages) - if body.tools and "tool" not in roles: - return Reply(body=_completion(_usage_tool_call(), "tool_calls")) return Reply(body=_completion({"role": "assistant", "content": REPLY_TEXT}, "stop")) @@ -459,12 +438,6 @@ def _restore(database_url: str, digest: str) -> None: ) -def _events(response: httpx.Response) -> tuple[Mapping[str, JsonValue], ...]: - return tuple( - json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ") - ) - - @dataclass(frozen=True, slots=True) class Tenant: label: str @@ -714,39 +687,19 @@ def _rig(gateway: Gateway, directory: Path) -> Generator[tuple[Gateway, str]]: @pytest.mark.timeout(360) -def test_usage_ai_chat_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees( +def test_usage_page_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees( gateway: Gateway, tmp_path: Path ) -> None: with _rig(gateway, tmp_path) as (proxy, database_url): - spender: Final = _spender(proxy, database_url, "ai-chat") + spender: Final = _spender(proxy, database_url, "repeated-timeout") with _pinned(proxy) as pinned: with _locked_spend_logs(database_url): - with _recording(database_url) as during_chat: - chat: Final = pinned.request( - "POST", - "/usage/ai/chat", - body={ - "messages": [{"role": "user", "content": "What did we spend?"}], - "model": "openai/gpt-4o-mini", - }, - ) - assert chat.status_code == 200, chat.text - assert chat.elapsed >= FAILED_LOOKUP_FLOOR, chat.elapsed - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": _day(-1), "end_date": _day(1)}, - } - assert _events(chat) == ( - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": REPLY_TEXT}, - {"type": "done"}, - ), chat.text - assert len(_lookups(during_chat)) == 1 + with _recording(database_url) as during_timeout: + timed_out: Final = _read(pinned, spender.digest) + assert timed_out.status_code == 200, timed_out.text + assert timed_out.elapsed >= FAILED_LOOKUP_FLOOR, timed_out.elapsed + assert _aliases(timed_out, spender.digest) == (None,), timed_out.text + assert len(_lookups(during_timeout)) == 1 cached_miss: Final = _read(pinned, spender.digest) assert cached_miss.elapsed < FAILED_LOOKUP_FLOOR, cached_miss.elapsed assert _aliases(cached_miss, spender.digest) == (None,), cached_miss.text @@ -1087,44 +1040,3 @@ def test_usage_page_rejects_bad_key_filters_and_unrelated_routes_ignore_a_locked assert gateway_activity == 200, gateway_activity chat: Final = proxy.chat(MODEL, text=f"locked {uuid.uuid4().hex}") assert object_value(chat["usage"]) == dict(USAGE), chat - - -@pytest.mark.timeout(300) -def test_usage_ai_chat_survives_a_dropped_database_connection_during_alias_recovery( - gateway: Gateway, tmp_path: Path -) -> None: - with ( - scratch_database() as database_url, - database_relay(database_url, b"AS " + LOOKUP_MARKER.encode()) as (relay, relayed_url), - wire_server(_respond) as wire, - _proxy(gateway, tmp_path, relayed_url, wire.url) as proxy, - ): - spender: Final = _spender(proxy, database_url, "dropped-ai-chat") - with _pinned(proxy) as pinned: - relay.arm() - chat: Final = pinned.request( - "POST", - "/usage/ai/chat", - body={ - "messages": [{"role": "user", "content": "What did we spend?"}], - "model": "openai/gpt-4o-mini", - }, - ) - assert chat.status_code == 200, chat.text - assert relay.tripped.is_set(), chat.text - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": _day(-1), "end_date": _day(1)}, - } - assert _events(chat) == ( - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": REPLY_TEXT}, - {"type": "done"}, - ), chat.text - recovered: Final = _named_once_reconnected(pinned, spender) - assert _metadata(recovered, spender.digest) == _full_metadata(spender), recovered.text diff --git a/tests/integration/spend/test_daily_activity_key_owner_traffic.py b/tests/integration/spend/test_daily_activity_key_owner_traffic.py index b8f113aca49..5284ad631de 100644 --- a/tests/integration/spend/test_daily_activity_key_owner_traffic.py +++ b/tests/integration/spend/test_daily_activity_key_owner_traffic.py @@ -6,7 +6,6 @@ from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from hashlib import sha256 -from pathlib import Path from queue import SimpleQueue from typing import Final @@ -31,8 +30,7 @@ from integration._support.daily_activity import ( user_row, user_with_an_email, ) -from integration._support.database import read_rows, scratch_database -from integration._support.process import owned_proxy +from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue @@ -51,22 +49,6 @@ UNIFIED_ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/respons REQUESTS_OF_A_BURST: Final = 21 READS_DURING_A_BURST: Final = 30 TOKEN_LIMIT_DISCOVERY: Final = ("GET", "/v1/models") -TOOL_CALL: Final = "call_integration_usage" -ANSWER: Final = "One request cost $0.25" -SUMMARY_OF_ONE_SEEDED_ROW: Final = "\n".join( - ( - "Total Spend: $0.2500", - "Total Requests: 1", - "Successful: 1 | Failed: 0", - "Total Tokens: 15", - "", - "Top Models by Spend:", - " - gpt-4o-mini: $0.2500 (1 reqs, 15 tokens)", - "", - "Top Providers by Spend:", - " - openai: $0.2500 (1 reqs)", - ) -) def _chat_completion() -> dict[str, JsonValue]: @@ -105,55 +87,6 @@ def _provider(request: Request) -> Reply: return Reply(body=json.dumps(body).encode()) -def _usage_tool_call() -> dict[str, JsonValue]: - call: Final[dict[str, JsonValue]] = { - "id": TOOL_CALL, - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": json.dumps({"start_date": DAY, "end_date": DAY}), - }, - } - return { - "id": f"chatcmpl-{uuid.uuid4().hex}", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": None, "tool_calls": [call]}, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - } - - -def _streamed_chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> bytes: - chunk: Final = { - "id": "chatcmpl-integration-usage", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], - } - return f"data: {json.dumps(chunk)}\n\n".encode() - - -def _usage_analyst(request: Request) -> Reply: - if json.loads(request.body).get("stream"): - return Reply( - chunks=( - _streamed_chunk({"role": "assistant", "content": ANSWER}, None), - _streamed_chunk({}, "stop"), - b"data: [DONE]\n\n", - ), - content_type="text/event-stream", - ) - return Reply(body=json.dumps(_usage_tool_call()).encode()) - - def _sent_for_callers(requests: Iterable[Request]) -> tuple[Request, ...]: return tuple(request for request in requests if (request.method, request.target) != TOKEN_LIMIT_DISCOVERY) @@ -322,65 +255,6 @@ def test_cli_session_spend_is_reported_with_the_user_and_team_of_the_session( ) -@pytest.mark.timeout(300) -def test_usage_ai_chat_hands_the_model_the_usage_summary_without_any_key_owner( - gateway: Gateway, tmp_path: Path -) -> None: - question: Final = f"what did we spend {uuid.uuid4().hex}" - owner: Final = f"integration-{uuid.uuid4().hex}" - ownerless_key: Final = f"integration-ownerless-{uuid.uuid4().hex}" - with ( - scratch_database() as scratch_url, - wire_server(_usage_analyst) as wire, - owned_proxy( - gateway, - tmp_path, - { - "DATABASE_URL": scratch_url, - "OPENAI_API_BASE": f"{wire.url}/v1", - "OPENAI_BASE_URL": f"{wire.url}/v1", - "OPENAI_API_KEY": "integration-provider-key", - }, - remove_environment=("DATABASE_URL_READ_REPLICA",), - ) as candidate, - ): - candidate.post("/user/new", {"user_id": owner, "user_email": f"{owner}@example.com", "auto_create_key": False}) - with daily_rows((user_row(owner, ownerless_key, DAY),), database_url=scratch_url): - answer: Final = candidate.request( - "POST", - "/usage/ai/chat", - {"messages": [{"role": "user", "content": question}], "model": "openai/gpt-4o-mini"}, - ) - assert answer.status_code == 200, answer.text - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": DAY, "end_date": DAY}, - } - events: Final = [ - json.loads(line.removeprefix("data: ")) for line in answer.text.splitlines() if line.startswith("data: ") - ] - assert events == [ - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": ANSWER}, - {"type": "done"}, - ], answer.text - asked, analysed = wire.drain() - assert [asked.target, analysed.target] == ["/v1/chat/completions", "/v1/chat/completions"] - assert json.loads(asked.body)["messages"][-1] == {"role": "user", "content": question} - assert json.loads(analysed.body)["messages"][-1] == { - "role": "tool", - "tool_call_id": TOOL_CALL, - "content": SUMMARY_OF_ONE_SEEDED_ROW, - } - assert owner not in analysed.body.decode() - assert ownerless_key not in analysed.body.decode() - - @pytest.mark.timeout(300) def test_owner_is_reported_on_every_route_while_a_burst_of_requests_waits_on_the_provider(gateway: Gateway) -> None: released: Final = threading.Event() diff --git a/tests/unit/AGENTS.md b/tests/unit/AGENTS.md index 9ce6c58cff7..ca717e8efb0 100644 --- a/tests/unit/AGENTS.md +++ b/tests/unit/AGENTS.md @@ -1,6 +1,9 @@ # tests/unit -In-process. No network or subprocess +In-process. No network or subprocess. `interactions/test_openapi_compliance.py` reads the spec snapshot beside it, +and only the `interactions-spec-canary` workflow passes `--interactions-spec-url` (registered in +`interactions/conftest.py`, so name that file or directory on the command line) with `ai.google.dev` allowed, to run +the same tests against Google's live spec Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed diff --git a/tests/unit/integrations/otel/test_otel_v2_components.py b/tests/unit/integrations/otel/test_otel_v2_components.py index 3f4389765d3..8af13fd6e7e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_components.py +++ b/tests/unit/integrations/otel/test_otel_v2_components.py @@ -14,6 +14,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer import pytest import requests +from requests_mock import Adapter pytest.importorskip("opentelemetry") @@ -22,16 +23,18 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa: ) from opentelemetry import baggage # noqa: E402 from opentelemetry.context import attach, detach # noqa: E402 -from opentelemetry._logs.severity import SeverityNumber # noqa: E402 -from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402 -from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402 +from opentelemetry._logs import LogRecord, SeverityNumber # noqa: E402 # optional SDK checked above +from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402 # optional SDK checked above +from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402 # optional SDK checked above +from opentelemetry.sdk.resources import Resource # noqa: E402 # optional SDK checked above from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402 -from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 # optional SDK checked above from opentelemetry.sdk.trace.export import ( # noqa: E402 BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor, + SpanExportResult, ) from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, @@ -46,6 +49,7 @@ import litellm # noqa: E402 from tests.unit.integrations.conftest import TlsSink # noqa: E402 from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter # noqa: E402 # optional SDK checked above from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 from litellm.integrations.otel.mappers.genai import GenAIMapper # noqa: E402 from litellm.integrations.otel.mappers.legacy import LegacyMapper # noqa: E402 @@ -733,7 +737,9 @@ def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict] class Collector(BaseHTTPRequestHandler): def do_POST(self): body = self.rfile.read(int(self.headers["Content-Length"])) - received.append({"path": self.path, "headers": dict(self.headers), "body": body}) + received.append( + {"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body} + ) self.send_response(200) self.end_headers() @@ -793,6 +799,95 @@ def test_http_json_exporter_posts_otlp_json_to_traces_endpoint(): assert child["parentSpanId"] == format(parent_id, "016x") +@pytest.mark.parametrize( + ("statuses", "expected"), + [ + ((200,), SpanExportResult.SUCCESS), + ((400,), SpanExportResult.FAILURE), + ((503, 200), SpanExportResult.SUCCESS), + ], +) +def test_http_json_exporter_preserves_http_failure_and_retry_results( + statuses: tuple[int, ...], expected: SpanExportResult +) -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri( + "POST", + endpoint, + [{"status_code": status, "headers": {"Retry-After": "0"}} for status in statuses], + ) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter( + endpoint=endpoint, + headers={ + "CONTENT-TYPE": "application/x-protobuf", + "x-collector-token": "secret", + }, + session=session, + ) + try: + assert exporter.export(()) is expected + finally: + exporter.shutdown() + assert len(adapter.request_history) == len(statuses) + for request in adapter.request_history: + assert request.headers["Content-Type"] == "application/json" + assert request.headers["x-collector-token"] == "secret" + assert request.json() == {"resourceSpans": []} + + +@pytest.mark.parametrize( + ("size_limit", "expected", "request_count"), + [ + (0, SpanExportResult.SUCCESS, 1), + (len(b'{"resourceSpans":[]}'), SpanExportResult.SUCCESS, 1), + (len(b'{"resourceSpans":[]}') - 1, SpanExportResult.FAILURE, 0), + ], +) +def test_http_json_exporter_checks_json_size_before_sending( + size_limit: int, expected: SpanExportResult, request_count: int +) -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session, max_request_size=size_limit) + try: + assert exporter.export(()) is expected + finally: + exporter.shutdown() + assert len(adapter.request_history) == request_count + + +def test_http_json_exporter_rejects_export_after_shutdown() -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session) + exporter.shutdown() + assert exporter.export(()) is SpanExportResult.FAILURE + assert adapter.request_history == [] + + +def test_http_json_exporter_returns_failure_for_unencodable_spans() -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session) + try: + assert exporter.export((ReadableSpan("missing-context"),)) is SpanExportResult.FAILURE + finally: + exporter.shutdown() + assert adapter.request_history == [] + + def test_http_protobuf_exporter_still_posts_protobuf(): (_child_request, parent_request), (trace_id, _parent_id, _child_id) = _export_one_trace_to_local_collector( "http/protobuf" @@ -1630,9 +1725,13 @@ def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.Monkey severity_number=SeverityNumber.INFO, body="v2-tls-test", ) - log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test")) + log_data: Final = ReadableLogRecord( + log_record=record, + resource=Resource({}), + instrumentation_scope=InstrumentationScope("v2-tls-test"), + ) result = exporter.export([log_data]) - assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}" assert tls_sink.received.get(timeout=5) == "/v1/logs" finally: exporter.shutdown() diff --git a/tests/unit/integrations/otel/test_otel_v2_mount.py b/tests/unit/integrations/otel/test_otel_v2_mount.py index 3a27c3ec4e4..61f8b9ae0d9 100644 --- a/tests/unit/integrations/otel/test_otel_v2_mount.py +++ b/tests/unit/integrations/otel/test_otel_v2_mount.py @@ -382,7 +382,9 @@ def test_every_traced_response_names_its_own_server_span(monkeypatch): assert responses["/stream"].text == "data: first\n\ndata: [DONE]\n\n" for path, response in responses.items(): context = server_spans[path].context - assert response.headers.get_list("traceresponse") == [f"00-{context.trace_id:032x}-{context.span_id:016x}-01"] + assert response.headers.get_list("traceresponse") == [ + f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}" + ] @pytest.mark.parametrize("path", ["/relayed", "/relayed/health"]) @@ -545,4 +547,4 @@ def test_the_unhandled_error_helper_names_the_span_it_is_given_in_any_context(mo headers = contextvars.Context().run(trace_response_headers, span) - assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-01" + assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}" diff --git a/tests/unit/integrations/otel/test_otel_v2_presets.py b/tests/unit/integrations/otel/test_otel_v2_presets.py index 64366626784..570c27a24e1 100644 --- a/tests/unit/integrations/otel/test_otel_v2_presets.py +++ b/tests/unit/integrations/otel/test_otel_v2_presets.py @@ -2,13 +2,19 @@ event loop: the preset does no network I/O, and a custom exporter mints the JWT lazily on its first export (in the BatchSpanProcessor worker thread).""" +import json + import httpx import pytest +import requests_mock +import respx +from opentelemetry.sdk.trace.export import SpanExportResult from litellm.integrations.otel.plumbing import providers from litellm.integrations.otel.model.config import ExporterSpec from litellm.integrations.otel.presets import agentops as agentops_mod from litellm.integrations.otel.presets.agentops import ( + _AGENTOPS_AUTH_ENDPOINT, _AGENTOPS_ENDPOINT, _AGENTOPS_EXPORTER_KIND, _build_agentops_exporter, @@ -75,50 +81,44 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): ) -def test_agentops_exporter_mints_jwt_lazily(monkeypatch): - pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - monkeypatch.setattr( - agentops_mod, "_fetch_agentops_jwt", lambda _k: {"token": "jwt-xyz"} - ) - spec = ExporterSpec( - kind=_AGENTOPS_EXPORTER_KIND, - endpoint=_AGENTOPS_ENDPOINT, - options={"api_key": "ak"}, - ) - exporter = _build_agentops_exporter(spec) - - # No auth header until the first export triggers the (off-loop) fetch. - assert "Authorization" not in exporter._session.headers - exporter._ensure_authenticated() - assert exporter._session.headers["Authorization"] == "Bearer jwt-xyz" - - # Cached: a second resolution does not re-fetch. - calls = [] - monkeypatch.setattr( - agentops_mod, - "_fetch_agentops_jwt", - lambda k: calls.append(k) or {"token": "again"}, - ) - exporter._ensure_authenticated() - assert calls == [] - - -def test_agentops_exporter_tolerates_fetch_failure(monkeypatch): - pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - - def _raise(_k): - raise RuntimeError("auth down") - - monkeypatch.setattr(agentops_mod, "_fetch_agentops_jwt", _raise) - exporter = _build_agentops_exporter( +def _agentops_exporter(): + return _build_agentops_exporter( ExporterSpec( kind=_AGENTOPS_EXPORTER_KIND, endpoint=_AGENTOPS_ENDPOINT, options={"api_key": "ak"}, ) ) - exporter._ensure_authenticated() # must not raise - assert "Authorization" not in exporter._session.headers + + +def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch(): + pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") + with respx.mock() as auth, requests_mock.Mocker() as collector: + token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"}) + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + exporter = _agentops_exporter() + assert token_route.call_count == 0, "building the exporter must not fetch the JWT" + + results = [exporter.export(()), exporter.export(())] + + assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS] + assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}] + assert [request.headers.get("Authorization") for request in collector.request_history] == [ + "Bearer jwt-xyz", + "Bearer jwt-xyz", + ] + + +def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails(): + pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") + with respx.mock() as auth, requests_mock.Mocker() as collector: + auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down")) + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + + result = _agentops_exporter().export(()) + + assert result is SpanExportResult.SUCCESS + assert [request.headers.get("Authorization") for request in collector.request_history] == [None] def test_fetch_jwt_uses_owned_client_not_shared_pool(monkeypatch): diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 9f536ab0dac..c7608159515 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -513,6 +513,22 @@ class TestLangfuseOtelIntegration: # Endpoint assertion removed as side effect is gone +def _exported_auth_headers(exporter): + import requests_mock + from opentelemetry.sdk.trace.export import SpanExportResult + + with requests_mock.Mocker() as collector: + collector.post(requests_mock.ANY, status_code=200) + assert exporter.export(()) is SpanExportResult.SUCCESS + return [ + ( + request.url, + {name: request.headers[name] for name in ("Authorization", "x-langfuse-ingestion-version")}, + ) + for request in collector.request_history + ] + + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target (OTLP endpoint + auth), not just auth headers on the init-time exporter.""" @@ -617,15 +633,15 @@ class TestLangfuseOtelKeyDynamicConfig: exporter = span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._endpoint == "https://langfuse.example.com/api/public/otel/v1/traces" - import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert exporter._headers == { - "Authorization": f"Basic {expected_auth}", - "x-langfuse-ingestion-version": "4", - } + assert _exported_auth_headers(exporter) == [ + ( + "https://langfuse.example.com/api/public/otel/v1/traces", + {"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"}, + ) + ] def test_key_dynamic_params_reuse_cached_provider(self): with self._clean_env(): @@ -678,10 +694,12 @@ class TestLangfuseOtelKeyDynamicConfig: provider = next(iter(logger._tracer_provider_cache.values())).provider exporter = provider._active_span_processor._span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._headers == { - "Authorization": f"Basic {secret}", - "x-langfuse-ingestion-version": "4", - } + assert _exported_auth_headers(exporter) == [ + ( + "https://langfuse.example.com/api/public/otel/v1/traces", + {"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"}, + ) + ] class TestLangfuseOtelResponsesAPI: diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index d14703e48a1..f908c685382 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -19,14 +19,16 @@ import pytest # Adds the grandparent directory to sys.path to allow importing project modules from opentelemetry import trace +from opentelemetry._logs import LogRecord from opentelemetry._logs.severity import SeverityNumber -from opentelemetry.sdk._logs import LogData, LogRecord +from opentelemetry.sdk._logs import ReadableLogRecord from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider -from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor +from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogRecordExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData +from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanExportResult from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from parameterized import parameterized @@ -2321,9 +2323,13 @@ def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPat severity_number=SeverityNumber.INFO, body="tls-export-test", ) - log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test")) + log_data: Final = ReadableLogRecord( + log_record=record, + resource=Resource({}), + instrumentation_scope=InstrumentationScope("tls-export-test"), + ) result: Final = exporter.export([log_data]) - assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}" assert tls_sink.received.get(timeout=5) == "/v1/logs" finally: exporter.shutdown() @@ -6639,7 +6645,7 @@ class TestOpenTelemetryProviderlessCallAttributes(unittest.TestCase): time.sleep(self.POLL_INTERVAL) return None - def _emitted_log_records(self, semconv_opt_in: str) -> tuple[LogData, ...]: + def _emitted_log_records(self, semconv_opt_in: str) -> tuple[ReadableLogRecord, ...]: log_exporter = InMemoryLogExporter() logger_provider = OTLoggerProvider() logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) @@ -6713,7 +6719,12 @@ class TestDynamicTracerProviderCache(unittest.TestCase): must be bounded and must shut down whatever it drops (LIT-5437: threads accumulated until pods were OOMKilled).""" - BSP_THREAD_NAME = "OtelBatchSpanProcessor" + def setUp(self) -> None: + processor: Final = BatchSpanProcessor(InMemorySpanExporter()) + try: + self.bsp_thread_name = processor._batch_processor._worker_thread.name + finally: + processor.shutdown() def _logger(self, cap=3, exporter="console"): logger = OpenTelemetry( @@ -6730,7 +6741,7 @@ class TestDynamicTracerProviderCache(unittest.TestCase): logger._tracer_provider_cache.clear() def _live_exporter_threads(self): - return [t for t in threading.enumerate() if t.name == self.BSP_THREAD_NAME] + return [t for t in threading.enumerate() if t.name == self.bsp_thread_name] def _wait_for_exporter_threads(self, expected, timeout=10.0): """Dropped providers are shut down off-thread, so poll instead of sleeping.""" @@ -6751,8 +6762,6 @@ class TestDynamicTracerProviderCache(unittest.TestCase): logger._get_tracer_with_dynamic_headers({"authorization": f"Basic tenant-{i}"}) self.assertEqual(len(logger._tracer_provider_cache), 3) - # Guards the thread-name constant: a rename upstream would make this read 0 and the - # bound assertion below would pass while measuring nothing. self.assertGreaterEqual(len(self._live_exporter_threads()), 1) self.assertLessEqual(self._wait_for_exporter_threads(before + 3) - before, 3) diff --git a/tests/unit/interactions/conftest.py b/tests/unit/interactions/conftest.py new file mode 100644 index 00000000000..b53763c7036 --- /dev/null +++ b/tests/unit/interactions/conftest.py @@ -0,0 +1,9 @@ +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--interactions-spec-url", + default=None, + help="Run the Interactions OpenAPI compliance tests against this live spec instead of the vendored snapshot", + ) diff --git a/tests/unit/interactions/interactions.openapi.json b/tests/unit/interactions/interactions.openapi.json new file mode 100644 index 00000000000..3830f5093cb --- /dev/null +++ b/tests/unit/interactions/interactions.openapi.json @@ -0,0 +1,10024 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Gemini API", + "description": "The Gemini Interactions API allows developers to build generative AI applications using Gemini models. Gemini is our most capable model, built from the ground up to be multimodal. It can generalize and seamlessly understand, operate across, and combine different types of information including language, images, audio, video, and code. You can use the Gemini API for use cases like reasoning across text and images, content generation, dialogue agents, summarization and classification systems, and more.", + "version": "v1beta", + "x-google-revision": "0" + }, + "servers": [ + { + "url": "https://generativelanguage.googleapis.com", + "description": "Global Endpoint" + } + ], + "paths": { + "/upload/{api_version}/environments/{environment}/files/{path}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + }, + { + "name": "environment", + "in": "path", + "description": "The ID of the environment that owns the destination file.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "path", + "description": "The relative destination path inside the environment workspace.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "put": { + "operationId": "StartEnvironmentFileUpload", + "description": "Starts a resumable upload session for a file in an environment workspace.\nUpload the file bytes to the URL returned in the `X-Goog-Upload-URL`\nresponse header, using the resumable upload protocol.\n", + "parameters": [ + { + "$ref": "#/components/parameters/EnvironmentFileExtract" + }, + { + "$ref": "#/components/parameters/EnvironmentFileOverwrite" + }, + { + "$ref": "#/components/parameters/EnvironmentFileUploadCommand" + }, + { + "$ref": "#/components/parameters/EnvironmentFileUploadContentLength" + }, + { + "$ref": "#/components/parameters/EnvironmentFileUploadContentType" + }, + { + "$ref": "#/components/parameters/EnvironmentFileUploadProtocol" + } + ], + "responses": { + "200": { + "description": "Upload session created successfully. Send the file bytes to the returned upload URL.", + "headers": { + "X-Goog-Upload-URL": { + "$ref": "#/components/headers/EnvironmentFileUploadUrl" + } + } + }, + "4XX": { + "description": "The upload session request failed with a client error." + }, + "5XX": { + "description": "The upload session request failed with a server error." + } + }, + "summary": "Start an environment file upload", + "x-codeSamples": [ + { + "label": "start_upload", + "lang": "sh", + "source": "curl -i -X PUT \\\n 'https://generativelanguage.googleapis.com/upload/v1beta/environments/env_abc123/files/main.py?overwrite=true' \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H 'X-Goog-Upload-Protocol: resumable' \\\n -H 'X-Goog-Upload-Command: start' \\\n -H \"X-Goog-Upload-Header-Content-Length: $(wc -c < main.py)\" \\\n -H 'X-Goog-Upload-Header-Content-Type: text/x-python'\n" + } + ] + } + }, + "/{api_version}/agents": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListAgents", + "description": "Lists all Agents.", + "parameters": [ + { + "name": "page_size", + "in": "query", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "parent", + "in": "query", + "description": "Required. The parent resource to list agents from.\nFormat: `projects/{project}/locations/{location}`", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "examples": { + "list": { + "summary": "List Agents", + "value": { + "agents": [ + { + "description": "A helpful research assistant.", + "base_agent": "antigravity-preview-05-2026", + "id": "research-assistant-abc123", + "system_instruction": "You are a helpful research assistant.", + "tools": [ + { + "type": "google_search" + } + ] + } + ] + } + } + }, + "schema": { + "$ref": "#/components/schemas/ListAgentsResponse" + } + } + } + } + }, + "tags": [ + "agents" + ], + "x-codeSamples": [ + { + "label": "list", + "lang": "sh", + "source": "curl -X GET https://generativelanguage.googleapis.com/v1beta/agents \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "list", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.agents.list()\nfor agent in response.agents or []:\n print(agent.id)\n" + }, + { + "label": "list", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst agents = await ai.agents.list();\nfor (const agent of agents.agents ?? []) {\n console.log(agent.id);\n}\n" + }, + { + "label": "list", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.gaos.models.agents.AgentListResponse;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.ListAgentsResponse;\n\nClient client = new Client();\nListAgentsResponse response = client.agents.list().call();\nList agents =\n response.agentListResponse().flatMap(AgentListResponse::agents).orElse(List.of());\nfor (Agent agent : agents) {\n agent.id().ifPresent(System.out::println);\n}\n" + }, + { + "label": "list", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n res, err := client.Agents.List(ctx, operations.ListAgentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.AgentListResponse != nil {\n for _, agent := range res.AgentListResponse.Agents {\n if agent.ID != nil {\n fmt.Println(*agent.ID)\n }\n }\n }\n}\n" + } + ] + }, + "post": { + "operationId": "CreateAgent", + "description": "Creates a new Agent (Typed version for SDK).", + "parameters": [ + { + "name": "parent", + "in": "query", + "description": "Required. The parent resource where this agent will be created.\nFormat: `projects/{project}/locations/{location}`", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Required. The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Agent" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "examples": { + "create": { + "summary": "Create Agent", + "value": { + "description": "A helpful research assistant.", + "base_agent": "antigravity-preview-05-2026", + "base_environment": "remote", + "id": "research-assistant-abc123", + "system_instruction": "You are a helpful research assistant.", + "tools": [ + { + "type": "google_search" + } + ] + } + }, + "fork_from_env": { + "summary": "Agent Forked from Environment", + "value": { + "base_agent": "antigravity-preview-05-2026", + "base_environment": "env_abc123", + "id": "my-data-analyst-abc123", + "system_instruction": "You are a data analyst. Use the template at /workspace/template.py for all reports." + } + }, + "with_sources": { + "summary": "Agent with Sources", + "value": { + "base_agent": "antigravity-preview-05-2026", + "base_environment": { + "type": "remote", + "sources": [ + { + "type": "inline", + "content": "Always use matplotlib for charts. Include a summary table in every report.", + "target": ".agents/AGENTS.md" + }, + { + "type": "repository", + "source": "https://github.com/my-org/analysis-templates", + "target": "/workspace/templates" + } + ] + }, + "id": "data-analyst-abc123", + "system_instruction": "You are a data analyst. Always include visualizations and export results as PDF." + } + } + }, + "schema": { + "$ref": "#/components/schemas/Agent" + } + } + } + } + }, + "tags": [ + "agents" + ], + "x-codeSamples": [ + { + "label": "create", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/agents \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"id\": \"research-assistant-abc123\",\n \"base_agent\": \"antigravity-preview-05-2026\",\n \"description\": \"A helpful research assistant.\",\n \"system_instruction\": \"You are a helpful research assistant.\",\n \"base_environment\": \"remote\",\n \"tools\": [{\"type\": \"google_search\"}]\n }'\n" + }, + { + "label": "with_sources", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/agents \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"id\": \"data-analyst-abc123\",\n \"base_agent\": \"antigravity-preview-05-2026\",\n \"system_instruction\": \"You are a data analyst. Always include visualizations and export results as PDF.\",\n \"base_environment\": {\n \"type\": \"remote\",\n \"sources\": [\n {\n \"type\": \"inline\",\n \"target\": \".agents/AGENTS.md\",\n \"content\": \"Always use matplotlib for charts. Include a summary table in every report.\"\n },\n {\n \"type\": \"repository\",\n \"source\": \"https://github.com/my-org/analysis-templates\",\n \"target\": \"/workspace/templates\"\n }\n ]\n }\n }'\n" + }, + { + "label": "fork_from_env", + "lang": "sh", + "source": "# Step 1: Set up the environment interactively\nRESPONSE=$(curl -s -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"antigravity-preview-05-2026\",\n \"input\": \"Write a basic Hello World template to /workspace/template.py.\",\n \"environment\": \"remote\"\n }')\nENV_ID=$(echo \"$RESPONSE\" | python3 -c \"import sys,json; print(json.load(sys.stdin)['environment_id'])\")\n\n# Step 2: Fork that environment into a named agent\ncurl -X POST https://generativelanguage.googleapis.com/v1beta/agents \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d \"{\n \\\"id\\\": \\\"my-data-analyst\\\",\n \\\"base_agent\\\": \\\"antigravity-preview-05-2026\\\",\n \\\"system_instruction\\\": \\\"You are a data analyst. Use the template at /workspace/template.py for all reports.\\\",\n \\\"base_environment\\\": \\\"$ENV_ID\\\"\n }\"\n" + }, + { + "label": "create", + "lang": "python", + "source": "import uuid\nfrom google import genai\n\nclient = genai.Client()\nagent = client.agents.create(\n id=f\"research-assistant-{uuid.uuid4().hex[:8]}\",\n base_agent=\"antigravity-preview-05-2026\",\n description=\"A helpful research assistant.\",\n system_instruction=\"You are a helpful research assistant.\",\n base_environment=\"remote\",\n tools=[{\"type\": \"google_search\"}],\n)\nprint(agent.id)\n" + }, + { + "label": "with_sources", + "lang": "python", + "source": "import uuid\nfrom google import genai\n\nclient = genai.Client()\nagent = client.agents.create(\n id=f\"data-analyst-{uuid.uuid4().hex[:8]}\",\n base_agent=\"antigravity-preview-05-2026\",\n system_instruction=(\n \"You are a data analyst. Always include visualizations and export\"\n \" results as PDF.\"\n ),\n base_environment={\n \"type\": \"remote\",\n \"sources\": [\n {\n \"type\": \"inline\",\n \"target\": \".agents/AGENTS.md\",\n \"content\": (\n \"Always use matplotlib for charts. Include a summary\"\n \" table in every report.\"\n ),\n },\n {\n \"type\": \"repository\",\n \"source\": \"https://github.com/my-org/analysis-templates\",\n \"target\": \"/workspace/templates\",\n },\n ],\n },\n)\nprint(f\"Created agent: {agent.id}\")\n" + }, + { + "label": "fork_from_env", + "lang": "python", + "source": "import uuid\nfrom google import genai\n\nclient = genai.Client()\n\n# Step 1: Set up the environment interactively.\ninteraction = client.interactions.create(\n agent=\"antigravity-preview-05-2026\",\n input=\"Write a basic Hello World template to /workspace/template.py.\",\n environment=\"remote\",\n)\n\n# Step 2: Fork that environment into a named agent.\nagent_id = f\"my-data-analyst-{uuid.uuid4().hex[:8]}\"\nagent = client.agents.create(\n id=agent_id,\n base_agent=\"antigravity-preview-05-2026\",\n system_instruction=(\n \"You are a data analyst. Use the template at /workspace/template.py\"\n \" for all reports.\"\n ),\n base_environment=interaction.environment_id,\n)\nprint(f\"Forked agent: {agent.id}\")\n" + }, + { + "label": "create", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst agentId = `research-assistant-${crypto.randomUUID().slice(0, 8)}`;\nconst agent = await ai.agents.create({\n id: agentId,\n base_agent: 'antigravity-preview-05-2026',\n description: 'A helpful research assistant.',\n system_instruction: 'You are a helpful research assistant.',\n base_environment: 'remote',\n tools: [{type: 'google_search'}],\n});\nif (!agent.id) {\n throw new Error('Agent creation failed: ID is undefined');\n}\nconsole.log(agent.id);\n" + }, + { + "label": "with_sources", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst agentId = `data-analyst-${crypto.randomUUID().slice(0, 8)}`;\nconst agent = await ai.agents.create({\n id: agentId,\n base_agent: 'antigravity-preview-05-2026',\n system_instruction:\n 'You are a data analyst. Always include visualizations and export results as PDF.',\n base_environment: {\n type: 'remote',\n sources: [\n {\n type: 'inline',\n target: '.agents/AGENTS.md',\n content:\n 'Always use matplotlib for charts. Include a summary table in every report.',\n },\n {\n type: 'repository',\n source: 'https://github.com/my-org/analysis-templates',\n target: '/workspace/templates',\n },\n ],\n },\n});\nconsole.log(`Created agent: ${agent.id}`);\n" + }, + { + "label": "fork_from_env", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n// Step 1: Set up the environment interactively.\nconst interaction = await ai.interactions.create({\n agent: 'antigravity-preview-05-2026',\n input: 'Write a basic Hello World template to /workspace/template.py.',\n environment: 'remote',\n});\n\n// Step 2: Fork that environment into a named agent.\nconst agentId = `my-data-analyst-${crypto.randomUUID().slice(0, 8)}`;\nconst agent = await ai.agents.create({\n id: agentId,\n base_agent: 'antigravity-preview-05-2026',\n system_instruction:\n 'You are a data analyst. Use the template at /workspace/template.py for all reports.',\n base_environment: interaction.environment_id,\n});\nconsole.log(`Forked agent: ${agent.id}`);\n" + }, + { + "label": "create", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.gaos.models.agents.BaseEnvironment;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateAgentResponse;\nimport com.google.genai.gaos.models.interactions.GoogleSearch;\n\nClient client = new Client();\nString agentId = \"research-assistant-\" + UUID.randomUUID().toString().substring(0, 8);\nAgent agent =\n Agent.builder()\n .id(agentId)\n .baseAgent(\"antigravity-preview-05-2026\")\n .description(\"A helpful research assistant.\")\n .systemInstruction(\"You are a helpful research assistant.\")\n .baseEnvironment(BaseEnvironment.of(\"remote\"))\n .tools(List.of(new GoogleSearch()))\n .build();\nCreateAgentResponse response = client.agents.create(agent);\nSystem.out.println(response.agent().flatMap(Agent::id).orElse(\"\"));\n" + }, + { + "label": "with_sources", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.gaos.models.agents.BaseEnvironment;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateAgentResponse;\nimport com.google.genai.gaos.models.environments.Environment;\nimport com.google.genai.gaos.models.interactions.Source;\nimport com.google.genai.gaos.models.interactions.SourceType;\n\nClient client = new Client();\nString agentId = \"data-analyst-\" + UUID.randomUUID().toString().substring(0, 8);\ncom.google.genai.gaos.models.interactions.Environment env =\n com.google.genai.gaos.models.interactions.Environment.builder()\n .sources(\n List.of(\n Source.builder()\n .type(SourceType.INLINE)\n .target(\".agents/AGENTS.md\")\n .content(\n \"Always use matplotlib for charts. Include a summary table in every\"\n + \" report.\")\n .build(),\n Source.builder()\n .type(SourceType.REPOSITORY)\n .source(\"https://github.com/my-org/analysis-templates\")\n .target(\"/workspace/templates\")\n .build()))\n .build();\nAgent agent =\n Agent.builder()\n .id(agentId)\n .baseAgent(\"antigravity-preview-05-2026\")\n .systemInstruction(\n \"You are a data analyst. Always include visualizations and export results as PDF.\")\n .baseEnvironment(BaseEnvironment.of(env))\n .build();\nCreateAgentResponse response = client.agents.create(agent);\nSystem.out.println(\"Created agent: \" + response.agent().flatMap(Agent::id).orElse(\"\"));\n" + }, + { + "label": "fork_from_env", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.gaos.models.agents.BaseEnvironment;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteractionEnvironment;\nimport com.google.genai.gaos.models.operations.CreateAgentResponse;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.Step;\n\nClient client = new Client();\n\n// Step 1: Set up the environment interactively.\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"antigravity-preview-05-2026\"))\n .input(\n InteractionsInput.of(\n \"Write a basic Hello World template to /workspace/template.py.\"))\n .environment(CreateAgentInteractionEnvironment.of(\"remote\"))\n .build();\nCreateInteractionResponse interactionResponse =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n interactionResponse\n .interaction()\n .orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\n\n// Step 2: Fork that environment into a named agent.\nString agentId = \"my-data-analyst-\" + UUID.randomUUID().toString().substring(0, 8);\nAgent agent =\n Agent.builder()\n .id(agentId)\n .baseAgent(\"antigravity-preview-05-2026\")\n .systemInstruction(\n \"You are a data analyst. Use the template at /workspace/template.py for all\"\n + \" reports.\")\n .baseEnvironment(BaseEnvironment.of(interaction.environmentId().orElse(\"\")))\n .build();\nCreateAgentResponse agentResponse = client.agents.create(agent);\nSystem.out.println(\"Forked agent: \" + agentResponse.agent().flatMap(Agent::id).orElse(\"\"));\n" + }, + { + "label": "create", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"time\"\n \"google.golang.org/genai\"\n gaos_agents \"google.golang.org/genai/interactions/models/agents\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n agentID := fmt.Sprintf(\"research-assistant-%d\", time.Now().UnixNano())\n req := operations.CreateAgentRequest{\n Body: gaos_agents.Agent{\n ID: genai.Ptr(agentID),\n BaseAgent: genai.Ptr(\"antigravity-preview-05-2026\"),\n Description: genai.Ptr(\"A helpful research assistant.\"),\n SystemInstruction: genai.Ptr(\"You are a helpful research assistant.\"),\n BaseEnvironment: genai.Ptr(gaos_agents.NewBaseEnvironment(\"remote\")),\n Tools: []gaos_agents.AgentTool{\n gaos_agents.NewAgentTool(gaos_interactions.GoogleSearch{}),\n },\n },\n }\n\n res, err := client.Agents.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Agent != nil && res.Agent.ID != nil {\n fmt.Println(*res.Agent.ID)\n }\n}\n" + }, + { + "label": "with_sources", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"time\"\n \"google.golang.org/genai\"\n gaos_agents \"google.golang.org/genai/interactions/models/agents\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n agentID := fmt.Sprintf(\"data-analyst-%d\", time.Now().UnixNano())\n env := gaos_interactions.Environment{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\".agents/AGENTS.md\"),\n Content: genai.Ptr(\"Always use matplotlib for charts. Include a summary table in every report.\"),\n },\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeRepository),\n Source: genai.Ptr(\"https://github.com/my-org/analysis-templates\"),\n Target: genai.Ptr(\"/workspace/templates\"),\n },\n },\n }\n\n req := operations.CreateAgentRequest{\n Body: gaos_agents.Agent{\n ID: genai.Ptr(agentID),\n BaseAgent: genai.Ptr(\"antigravity-preview-05-2026\"),\n SystemInstruction: genai.Ptr(\"You are a data analyst. Always include visualizations and export results as PDF.\"),\n BaseEnvironment: genai.Ptr(gaos_agents.NewBaseEnvironment(env)),\n },\n }\n\n res, err := client.Agents.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Agent != nil && res.Agent.ID != nil {\n fmt.Printf(\"Created agent: %s\\n\", *res.Agent.ID)\n }\n}\n" + }, + { + "label": "fork_from_env", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"time\"\n \"google.golang.org/genai\"\n gaos_agents \"google.golang.org/genai/interactions/models/agents\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n // Step 1: Set up the environment interactively.\n interactionReq := operations.CreateInteractionRequest{\n Body: operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"antigravity-preview-05-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Write a basic Hello World template to /workspace/template.py.\")),\n Environment: genai.Ptr(gaos_interactions.NewCreateAgentInteractionEnvironment(\"remote\")),\n }),\n }\n interactionRes, err := client.Interactions.Create(ctx, interactionReq)\n if err != nil {\n log.Fatal(err)\n }\n envID := \"\"\n if interactionRes.Interaction != nil && interactionRes.Interaction.EnvironmentID != nil {\n envID = *interactionRes.Interaction.EnvironmentID\n }\n\n // Step 2: Fork that environment into a named agent.\n agentID := fmt.Sprintf(\"my-data-analyst-%d\", time.Now().UnixNano())\n req := operations.CreateAgentRequest{\n Body: gaos_agents.Agent{\n ID: genai.Ptr(agentID),\n BaseAgent: genai.Ptr(\"antigravity-preview-05-2026\"),\n SystemInstruction: genai.Ptr(\"You are a data analyst. Use the template at /workspace/template.py for all reports.\"),\n BaseEnvironment: genai.Ptr(gaos_agents.NewBaseEnvironment(envID)),\n },\n }\n res, err := client.Agents.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n if res.Agent != nil && res.Agent.ID != nil {\n fmt.Printf(\"Forked agent: %s\\n\", *res.Agent.ID)\n }\n}\n" + } + ] + } + }, + "/{api_version}/agents/{agentsId}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "delete": { + "operationId": "DeleteAgent", + "description": "Deletes an Agent.", + "parameters": [ + { + "name": "agentsId", + "in": "path", + "description": "Required. The name of the agent to delete.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "examples": { + "delete": { + "summary": "Delete Agent", + "value": {} + } + }, + "schema": { + "$ref": "#/components/schemas/Empty" + } + } + } + } + }, + "tags": [ + "agents" + ], + "x-codeSamples": [ + { + "label": "delete", + "lang": "sh", + "source": "curl -X DELETE https://generativelanguage.googleapis.com/v1beta/agents/ag_abc123 \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "delete", + "lang": "python", + "source": "import uuid\nfrom google import genai\n\nclient = genai.Client()\n\n\nclient.agents.delete(agent_id)\nprint(\"Agent deleted successfully.\")\n" + }, + { + "label": "delete", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nawait ai.agents.delete(agentId);\nconsole.log('Agent deleted successfully.');\n" + }, + { + "label": "delete", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.Client;\n\nClient client = new Client();\n\n\nclient.agents.delete(agentId);\nSystem.out.println(\"Agent deleted successfully.\");\n" + }, + { + "label": "delete", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n _, err = client.Agents.Delete(ctx, operations.DeleteAgentRequest{ID: agentID})\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Agent deleted successfully.\")\n}\n" + } + ] + }, + "get": { + "operationId": "GetAgent", + "description": "Gets a specific Agent.", + "parameters": [ + { + "name": "agentsId", + "in": "path", + "description": "Required. The name of the agent to retrieve.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "examples": { + "get": { + "summary": "Get Agent", + "value": { + "created": "2025-11-26T12:25:15Z", + "display_name": "My Research Agent", + "id": "ag_abc123", + "object": "agent", + "system_instruction": "You are a helpful research assistant.", + "tools": [ + { + "type": "google_search" + } + ], + "updated": "2025-11-26T12:25:15Z" + } + } + }, + "schema": { + "$ref": "#/components/schemas/Agent" + } + } + } + } + }, + "tags": [ + "agents" + ], + "x-codeSamples": [ + { + "label": "get", + "lang": "sh", + "source": "curl -X GET https://generativelanguage.googleapis.com/v1beta/agents/ag_abc123 \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "get", + "lang": "python", + "source": "import uuid\nfrom google import genai\n\nclient = genai.Client()\n\n\nagent = client.agents.get(agent_id)\nprint(agent.id)\n" + }, + { + "label": "get", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nconst agent = await ai.agents.get(agentId);\nif (!agent.id) {\n throw new Error('Agent retrieval failed: ID is undefined');\n}\nconsole.log(agent.id);\n" + }, + { + "label": "get", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.GetAgentResponse;\n\nClient client = new Client();\n\n\nGetAgentResponse response = client.agents.get(agentId);\nAgent agent = response.agent().orElseThrow(() -> new RuntimeException(\"No agent returned\"));\nSystem.out.println(agent.id().orElse(\"\"));\n" + }, + { + "label": "get", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Agents.Get(ctx, operations.GetAgentRequest{ID: agentID})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Agent != nil && res.Agent.ID != nil {\n fmt.Println(*res.Agent.ID)\n }\n}\n" + } + ] + } + }, + "/{api_version}/credentials": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListCredentials", + "description": "Lists credentials.", + "parameters": [ + { + "name": "page_size", + "in": "query", + "description": "Optional. Maximum number of credentials to return.", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Optional. Pagination token.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListCredentialsResponse" + } + } + } + } + }, + "tags": [ + "credentials" + ] + }, + "post": { + "operationId": "CreateCredential", + "description": "Creates a new credential.", + "parameters": [], + "requestBody": { + "description": "Required. The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CredentialCreateParams" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Credential" + } + } + } + } + }, + "tags": [ + "credentials" + ] + } + }, + "/{api_version}/credentials/{id}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "delete": { + "operationId": "DeleteCredential", + "description": "Deletes a credential.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource ID segment making up resource `name`. It identifies the resource\nwithin its parent collection as described in https://google.aip.dev/122.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Empty" + } + } + } + } + }, + "tags": [ + "credentials" + ] + }, + "get": { + "operationId": "GetCredential", + "description": "Gets a credential by ID.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource ID segment making up resource `name`. It identifies the resource\nwithin its parent collection as described in https://google.aip.dev/122.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Credential" + } + } + } + } + }, + "tags": [ + "credentials" + ] + }, + "patch": { + "operationId": "UpdateCredential", + "description": "Updates a credential.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource ID segment making up resource `name`. It identifies the resource\nwithin its parent collection as described in https://google.aip.dev/122.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "update_mask", + "in": "query", + "description": "Optional. The list of fields to update.", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Required. The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CredentialUpdateParams" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Credential" + } + } + } + } + }, + "tags": [ + "credentials" + ] + } + }, + "/{api_version}/environments": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListEnvironments", + "description": "Lists environments.", + "parameters": [ + { + "name": "page_size", + "in": "query", + "description": "Optional. Maximum number of environments to return.\nIf unspecified, defaults to 50. Maximum is 1000.", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Optional. Pagination token.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListEnvironmentsResponse" + } + } + } + } + }, + "tags": [ + "environments" + ], + "x-codeSamples": [ + { + "label": "list", + "lang": "sh", + "source": "curl -X GET https://generativelanguage.googleapis.com/v1beta/environments \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "list", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.environments.list()\nfor environment in response.environments or []:\n print(environment.id)\n" + }, + { + "label": "list", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst response = await ai.environments.list();\nfor (const env of response.environments ?? []) {\n console.log(env.id);\n}\n" + }, + { + "label": "list", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.Environment;\nimport com.google.genai.gaos.models.operations.ListEnvironmentsResponse;\n\nClient client = new Client();\nListEnvironmentsResponse response = client.environments.listEnvironmentsDirect();\nfor (com.google.genai.gaos.models.environments.Environment env :\n response.listEnvironmentsResponse().flatMap(res -> res.environments()).orElse(List.of())) {\n System.out.println(env.id().orElse(\"\"));\n}\n" + }, + { + "label": "list", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n" + } + ] + }, + "post": { + "operationId": "CreateEnvironment", + "description": "Creates an environment.", + "parameters": [], + "requestBody": { + "description": "Required. The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateEnvironmentRequest" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Environment" + } + } + } + } + }, + "tags": [ + "environments" + ], + "x-codeSamples": [ + { + "label": "create", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/environments \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"sources\": [{\n \"type\": \"inline\",\n \"target\": \"main.py\",\n \"content\": \"print(\\\"Hello, World!\\\")\"\n }]\n }'\n" + }, + { + "label": "copy", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/environments \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"from_environment\": \"environments/env_abc123\"\n }'\n" + }, + { + "label": "create", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nenvironment = client.environments.create(\n sources=[{\n \"type\": \"inline\",\n \"target\": \"main.py\",\n \"content\": \"print('Hello, World!')\",\n }]\n)\nprint(environment.id)\n" + }, + { + "label": "copy", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\nenvironment = client.environments.create(\n from_environment=source_env.id,\n)\nprint(environment.id)\n" + }, + { + "label": "create", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst environment = await ai.environments.create({\n sources: [\n {\n type: 'inline',\n target: 'main.py',\n content: \"print('Hello, World!')\",\n },\n ],\n});\nif (!environment.id) {\n throw new Error('Environment creation failed: ID is undefined');\n}\nconsole.log(environment.id);\n" + }, + { + "label": "copy", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nconst environment = await ai.environments.create({\n from_environment: sourceEnv.id,\n});\nif (!environment.id) {\n throw new Error('Environment creation failed: ID is undefined');\n}\nconsole.log(environment.id);\n" + }, + { + "label": "create", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.CreateEnvironmentRequest;\nimport com.google.genai.gaos.models.environments.Environment;\nimport com.google.genai.gaos.models.interactions.Source;\nimport com.google.genai.gaos.models.interactions.SourceType;\n\nClient client = new Client();\nCreateEnvironmentRequest request =\n CreateEnvironmentRequest.builder()\n .sources(\n List.of(\n Source.builder()\n .type(SourceType.INLINE)\n .target(\"main.py\")\n .content(\"print('Hello, World!')\")\n .build()))\n .build();\ncom.google.genai.gaos.models.environments.Environment environment =\n client.environments.createEnvironment(request).environment().orElseThrow();\nSystem.out.println(environment.id().orElse(\"\"));\n" + }, + { + "label": "copy", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.CreateEnvironmentRequest;\nimport com.google.genai.gaos.models.environments.Environment;\n\nClient client = new Client();\n\n\nCreateEnvironmentRequest request =\n CreateEnvironmentRequest.builder().fromEnvironment(sourceEnvId).build();\ncom.google.genai.gaos.models.environments.Environment environment =\n client.environments.createEnvironment(request).environment().orElseThrow();\nSystem.out.println(environment.id().orElse(\"\"));\n" + }, + { + "label": "create", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" + }, + { + "label": "copy", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" + } + ] + } + }, + "/{api_version}/environments/{environment}/files/{path}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + }, + { + "name": "environment", + "in": "path", + "description": "The ID of the environment whose snapshot to read.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "path", + "description": "Path of the file or directory inside the environment workspace, relative to its root (e.g. src).", + "required": true, + "schema": { + "type": "string" + } + } + ], + "get": { + "operationId": "GetEnvironmentFiles", + "description": "Retrieves file metadata or directory contents from an environment's snapshot. To download file content, use the download URL returned in the response.", + "parameters": [ + { + "name": "page_size", + "in": "query", + "description": "Optional. Maximum number of entries to return per page (for directory listing).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Optional. Pagination token for directory listing.", + "schema": { + "type": "string" + } + }, + { + "name": "recursive", + "in": "query", + "description": "Optional. If true and the path is a directory, recursively lists all files.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetEnvironmentFilesResponse" + } + } + } + } + }, + "tags": [ + "environments.files" + ], + "x-codeSamples": [ + { + "label": "list_files", + "lang": "sh", + "source": "curl -X GET 'https://generativelanguage.googleapis.com/v1beta/environments/env_abc123/files/src' \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "get_file", + "lang": "sh", + "source": "curl -X GET 'https://generativelanguage.googleapis.com/v1beta/environments/env_abc123/files/main.py' \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "download_file", + "lang": "sh", + "source": "curl -X GET 'https://generativelanguage.googleapis.com/v1beta/environments/env_abc123/files/src/main.py?alt=media' \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n --output main.py\n" + }, + { + "label": "list_files", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\nresponse = client.environments.files.list(\n environment=created.id,\n path=\"src\",\n)\nfor file in response.files or []:\n print(file.name, file.type, file.size_bytes)\n" + }, + { + "label": "get_file", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\nresponse = client.environments.files.get(\n environment=created.id,\n path=\"main.py\",\n)\nfor file in response.files or []:\n print(file.name, file.size_bytes)\n" + }, + { + "label": "download_file", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\ncontent = client.environments.files.download(\n environment=created.id,\n path=\"src/main.py\",\n)\nprint(content.decode(\"utf-8\"))\n" + }, + { + "label": "list_files", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nconst response = await ai.environments.files.list({\n environment: created.id,\n path: 'src',\n});\nfor (const file of response.files ?? []) {\n console.log(file.name, file.type, file.size_bytes);\n}\n" + }, + { + "label": "get_file", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nconst response = await ai.environments.files.list({\n environment: created.id,\n path: 'main.py',\n});\nfor (const file of response.files ?? []) {\n console.log(file.name, file.size_bytes);\n}\n" + }, + { + "label": "list_files", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.EnvironmentFile;\nimport com.google.genai.gaos.models.operations.GetEnvironmentFilesRequest;\nimport com.google.genai.gaos.models.environments.GetEnvironmentFilesResponse;\n\nClient client = new Client();\n\n\ncom.google.genai.gaos.models.operations.GetEnvironmentFilesResponse response =\n client\n .environments\n .files()\n .list(GetEnvironmentFilesRequest.builder().environment(envId).path(\"src\").build());\nGetEnvironmentFilesResponse filesResponse =\n response.getEnvironmentFilesResponse().orElseThrow();\nfor (EnvironmentFile file : filesResponse.files().orElse(List.of())) {\n System.out.println(\n file.name().orElse(\"\")\n + \" \"\n + file.type().map(Object::toString).orElse(\"\")\n + \" \"\n + file.sizeBytes().orElse(\"\"));\n}\n" + }, + { + "label": "get_file", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.EnvironmentFile;\nimport com.google.genai.gaos.models.operations.GetEnvironmentFilesRequest;\nimport com.google.genai.gaos.models.environments.GetEnvironmentFilesResponse;\n\nClient client = new Client();\n\n\ncom.google.genai.gaos.models.operations.GetEnvironmentFilesResponse response =\n client\n .environments\n .files()\n .list(GetEnvironmentFilesRequest.builder().environment(envId).path(\"main.py\").build());\nGetEnvironmentFilesResponse filesResponse =\n response.getEnvironmentFilesResponse().orElseThrow();\nfor (EnvironmentFile file : filesResponse.files().orElse(List.of())) {\n System.out.println(file.name().orElse(\"\") + \" \" + file.sizeBytes().orElse(\"\"));\n}\n" + }, + { + "label": "list_files", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n" + }, + { + "label": "get_file", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n" + } + ] + } + }, + "/{api_version}/environments/{id}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "delete": { + "operationId": "DeleteEnvironment", + "description": "Deletes an environment.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource ID segment making up resource `name`. It identifies the resource\nwithin its parent collection as described in https://google.aip.dev/122.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Empty" + } + } + } + } + }, + "tags": [ + "environments" + ], + "x-codeSamples": [ + { + "label": "delete", + "lang": "sh", + "source": "curl -X DELETE https://generativelanguage.googleapis.com/v1beta/environments/env_abc123 \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "delete", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\nclient.environments.delete(id=created.id)\nprint(\"Environment deleted successfully.\")\n" + }, + { + "label": "delete", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nawait ai.environments.delete(created.id);\nconsole.log('Environment deleted successfully.');\n" + }, + { + "label": "delete", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.Environment;\n\nClient client = new Client();\n\n\nclient.environments.deleteEnvironment(envId);\nSystem.out.println(\n \"com.google.genai.gaos.models.environments.Environment deleted successfully.\");\n" + }, + { + "label": "delete", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n" + } + ] + }, + "get": { + "operationId": "GetEnvironment", + "description": "Gets an environment.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource ID segment making up resource `name`. It identifies the resource\nwithin its parent collection as described in https://google.aip.dev/122.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Environment" + } + } + } + } + }, + "tags": [ + "environments" + ], + "x-codeSamples": [ + { + "label": "get", + "lang": "sh", + "source": "curl -X GET https://generativelanguage.googleapis.com/v1beta/environments/env_abc123 \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "get", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\nenvironment = client.environments.get(id=created.id)\nprint(environment.id)\n" + }, + { + "label": "get", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nconst environment = await ai.environments.get(created.id);\nconsole.log(environment.id);\n" + }, + { + "label": "get", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.environments.Environment;\n\nClient client = new Client();\n\n\ncom.google.genai.gaos.models.environments.Environment environment =\n client.environments.getEnvironment(envId).environment().orElseThrow();\nSystem.out.println(environment.id().orElse(\"\"));\n" + }, + { + "label": "get", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n" + } + ] + } + }, + "/{api_version}/interactions": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "post": { + "operationId": "CreateInteraction", + "description": "Creates a new interaction.", + "requestBody": { + "description": "The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/CreateAgentInteractionParams" + }, + { + "$ref": "#/components/schemas/CreateModelInteractionParams" + } + ] + } + } + } + }, + "responses": { + "200": { + "description": "Successful operation", + "content": { + "application/json": { + "examples": { + "antigravity": { + "summary": "Antigravity Agent", + "value": { + "agent": "antigravity-preview-05-2026", + "created": "2025-11-26T12:22:47Z", + "environment_id": "env_abc123", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "I've summarized the top 5 Hacker News stories and saved the results to /workspace/summary.md." + } + ] + } + ], + "updated": "2025-11-26T12:22:47Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 50 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 50, + "total_output_tokens": 500, + "total_thought_tokens": 200, + "total_tokens": 750, + "total_tool_use_tokens": 0 + } + } + }, + "deep_research": { + "summary": "Deep Research", + "value": { + "agent": "deep-research-pro-preview-12-2025", + "created": "2025-11-26T12:22:47Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "Here is a comprehensive research report on the current state of cancer research..." + } + ] + } + ], + "updated": "2025-11-26T12:22:47Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 20 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 20, + "total_output_tokens": 1000, + "total_thought_tokens": 500, + "total_tokens": 1520, + "total_tool_use_tokens": 0 + } + } + }, + "function_calling": { + "summary": "Function Calling", + "value": { + "created": "2025-11-26T12:22:47Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "requires_action", + "steps": [ + { + "name": "get_weather", + "type": "function_call", + "arguments": { + "location": "Boston, MA" + }, + "id": "gth23981" + } + ], + "updated": "2025-11-26T12:22:47Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 100 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 100, + "total_output_tokens": 25, + "total_thought_tokens": 0, + "total_tokens": 125, + "total_tool_use_tokens": 50 + } + } + }, + "multi_turn": { + "summary": "Multi-turn", + "value": { + "created": "2025-11-26T12:22:47Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "The capital of France is Paris." + } + ] + } + ], + "updated": "2025-11-26T12:22:47Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 50 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 50, + "total_output_tokens": 10, + "total_thought_tokens": 0, + "total_tokens": 60, + "total_tool_use_tokens": 0 + } + } + }, + "multimodal_image": { + "summary": "Image Input", + "value": { + "created": "2025-11-26T12:22:47Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "A white humanoid robot with glowing blue eyes stands holding a red skateboard." + } + ] + } + ], + "updated": "2025-11-26T12:22:47Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 10 + }, + { + "modality": "image", + "tokens": 258 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 268, + "total_output_tokens": 20, + "total_thought_tokens": 0, + "total_tokens": 288, + "total_tool_use_tokens": 0 + } + } + }, + "reuse_env": { + "summary": "Reuse Environment", + "value": { + "agent": "antigravity-preview-05-2026", + "created": "2025-11-26T12:23:00Z", + "environment_id": "env_abc123", + "id": "v1_Chd2ZTJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ejAxMjM0NTY3ODkwMTIzNDU2Nzg", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "I've updated /workspace/hello.py to accept a name argument and greet the user." + } + ] + } + ], + "updated": "2025-11-26T12:23:00Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 80 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 80, + "total_output_tokens": 200, + "total_thought_tokens": 100, + "total_tokens": 380, + "total_tool_use_tokens": 0 + } + } + }, + "simple": { + "summary": "Simple Request", + "value": { + "created": "2025-11-26T12:25:15Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "Hello! I'm functioning perfectly and ready to assist you.\n\nHow are you doing today?" + } + ] + } + ], + "updated": "2025-11-26T12:25:15Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 7 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 7, + "total_output_tokens": 20, + "total_thought_tokens": 22, + "total_tokens": 49, + "total_tool_use_tokens": 0 + } + } + } + }, + "schema": { + "$ref": "#/components/schemas/Interaction" + } + }, + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/InteractionSseStreamEvent" + }, + "x-speakeasy-sse-sentinel": "[DONE]" + } + } + }, + "4XX": { + "description": "Error creating interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "5XX": { + "description": "Error creating interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + } + }, + "summary": "Creating an interaction", + "x-speakeasy-exports": [ + { + "name": "InteractionCreateParams", + "group": "interactions", + "representation": "input" + } + ], + "x-speakeasy-group": "interactions", + "x-speakeasy-name-override": "create", + "x-codeSamples": [ + { + "label": "simple", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"input\": \"Hello, how are you?\"\n }'\n" + }, + { + "label": "multi_turn", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"input\": [\n { \"type\": \"user_input\", \"content\": [{ \"type\": \"text\", \"text\": \"Hello!\" }] },\n { \"type\": \"model_output\", \"content\": [{ \"type\": \"text\", \"text\": \"Hi there! How can I help you today?\" }] },\n { \"type\": \"user_input\", \"content\": [{ \"type\": \"text\", \"text\": \"What is the capital of France?\" }] }\n ]\n }'\n" + }, + { + "label": "multimodal_image", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"input\": [\n {\n \"type\": \"text\",\n \"text\": \"What is in this picture?\"\n },\n {\n \"type\": \"image\",\n \"data\": \"BASE64_ENCODED_IMAGE\",\n \"mime_type\": \"image/png\"\n }\n ]\n }'\n" + }, + { + "label": "function_calling", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n }\n },\n \"required\": [\n \"location\"\n ]\n }\n }\n ],\n \"input\": \"What is the weather like in Boston, MA?\"\n }'\n" + }, + { + "label": "deep_research", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-pro-preview-12-2025\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n" + }, + { + "label": "antigravity", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"antigravity-preview-05-2026\",\n \"input\": \"Read Hacker News, summarize the top 5 stories, and save results as a markdown file.\",\n \"environment\": \"remote\"\n }'\n" + }, + { + "label": "reuse_env", + "lang": "sh", + "source": "# Step 1: Create an interaction with a fresh remote environment.\nRESPONSE=$(curl -s -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -H \"Api-Revision: 2026-05-20\" \\\n -d '{\n \"agent\": \"antigravity-preview-05-2026\",\n \"input\": \"Write a hello world script at /workspace/hello.py.\",\n \"environment\": \"remote\"\n }')\nINTERACTION_ID=$(echo \"$RESPONSE\" | python3 -c \"import sys,json; print(json.load(sys.stdin)['id'])\")\nENV_ID=$(echo \"$RESPONSE\" | python3 -c \"import sys,json; print(json.load(sys.stdin)['environment_id'])\")\n\n# Step 2: Reuse the same environment in a follow-up interaction.\ncurl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d \"{\n \\\"agent\\\": \\\"antigravity-preview-05-2026\\\",\n \\\"input\\\": \\\"Modify the script to accept a name argument and greet the user.\\\",\n \\\"environment\\\": \\\"$ENV_ID\\\",\n \\\"previous_interaction_id\\\": \\\"$INTERACTION_ID\\\"\n }\"\n" + }, + { + "label": "with_sources", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"antigravity-preview-05-2026\",\n \"input\": \"List all files under /workspace and summarize what you find.\",\n \"environment\": {\n \"type\": \"remote\",\n \"sources\": [\n {\n \"type\": \"repository\",\n \"source\": \"https://github.com/octocat/Spoon-Knife\",\n \"target\": \"/workspace/repo\"\n },\n {\n \"type\": \"inline\",\n \"content\": \"Focus on Python files only.\",\n \"target\": \"/workspace/notes.txt\"\n }\n ]\n }\n }'\n" + }, + { + "label": "custom_agent", + "lang": "sh", + "source": "# Step 1: Create a custom agent.\ncurl -X POST https://generativelanguage.googleapis.com/v1beta/agents \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"id\": \"code-reviewer\",\n \"base_agent\": \"antigravity-preview-05-2026\",\n \"system_instruction\": \"You are a senior code reviewer. Check every file for bugs, style issues, and security vulnerabilities.\",\n \"base_environment\": {\n \"type\": \"remote\",\n \"sources\": [{\n \"type\": \"repository\",\n \"source\": \"https://github.com/octocat/Spoon-Knife\",\n \"target\": \"/workspace/repo\"\n }]\n }\n }'\n\n# Step 2: Use the custom agent.\ncurl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"code-reviewer\",\n \"input\": \"Review the latest changes in /workspace/repo/src and file a summary.\",\n \"environment\": \"remote\"\n }'\n" + }, + { + "label": "simple", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n model=\"gemini-flash-latest\",\n input=\"Hello, how are you?\",\n)\nprint(interaction.output_text)\n" + }, + { + "label": "multi_turn", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n input=[\n {\n \"type\": \"user_input\",\n \"content\": [{\"type\": \"text\", \"text\": \"Hello!\"}],\n },\n {\n \"type\": \"model_output\",\n \"content\": [{\n \"type\": \"text\",\n \"text\": \"Hi there! How can I help you today?\",\n }],\n },\n {\n \"type\": \"user_input\",\n \"content\": [\n {\"type\": \"text\", \"text\": \"What is the capital of France?\"}\n ],\n },\n ],\n)\nprint(response.output_text)\n" + }, + { + "label": "multimodal_image", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n input=[\n {\"type\": \"text\", \"text\": \"What is in this picture?\"},\n {\n \"type\": \"image\",\n \"data\": \"BASE64_ENCODED_IMAGE\",\n \"mime_type\": \"image/png\",\n },\n ],\n)\nprint(response.output_text)\n" + }, + { + "label": "function_calling", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[{\n \"type\": \"function\",\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": (\n \"The city and state, e.g. San Francisco, CA\"\n ),\n }\n },\n \"required\": [\"location\"],\n },\n }],\n input=\"What is the weather like in Boston, MA?\",\n)\nprint(response.steps[-1])\n" + }, + { + "label": "deep_research", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-pro-preview-12-2025\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n" + }, + { + "label": "antigravity", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"antigravity-preview-05-2026\",\n input=(\n \"Read Hacker News, summarize the top 5 stories, and save results as\"\n \" a markdown file.\"\n ),\n environment=\"remote\",\n)\nprint(interaction.output_text)\n" + }, + { + "label": "reuse_env", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n# Step 1: Create an interaction with a fresh remote environment.\ninteraction = client.interactions.create(\n agent=\"antigravity-preview-05-2026\",\n input=\"Write a hello world script at /workspace/hello.py.\",\n environment=\"remote\",\n)\nprint(f\"Environment ID: {interaction.environment_id}\")\n\n# Step 2: Reuse the same environment in a follow-up interaction.\ninteraction_2 = client.interactions.create(\n agent=\"antigravity-preview-05-2026\",\n input=\"Modify the script to accept a name argument and greet the user.\",\n environment=interaction.environment_id,\n previous_interaction_id=interaction.id,\n)\nprint(interaction_2.output_text)\n" + }, + { + "label": "with_sources", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"antigravity-preview-05-2026\",\n input=\"List all files under /workspace and summarize what you find.\",\n environment={\n \"type\": \"remote\",\n \"sources\": [\n {\n \"type\": \"repository\",\n \"source\": \"https://github.com/octocat/Spoon-Knife\",\n \"target\": \"/workspace/repo\",\n },\n {\n \"type\": \"inline\",\n \"content\": \"Focus on Python files only.\",\n \"target\": \"/workspace/notes.txt\",\n },\n ],\n },\n)\nprint(interaction.output_text)\n" + }, + { + "label": "custom_agent", + "lang": "python", + "source": "import uuid\nfrom google import genai\n\nclient = genai.Client()\n\n# Step 1: Create a custom agent.\nagent_id = f\"code-reviewer-{uuid.uuid4().hex[:8]}\"\nclient.agents.create(\n id=agent_id,\n base_agent=\"antigravity-preview-05-2026\",\n system_instruction=(\n \"You are a senior code reviewer. Check every file for bugs, style\"\n \" issues, and security vulnerabilities.\"\n ),\n base_environment={\n \"type\": \"remote\",\n \"sources\": [{\n \"type\": \"repository\",\n \"source\": \"https://github.com/octocat/Spoon-Knife\",\n \"target\": \"/workspace/repo\",\n }],\n },\n)\n\n# Step 2: Use the custom agent.\nresult = client.interactions.create(\n agent=agent_id,\n input=(\n \"Review the latest changes in /workspace/repo/src and file a\"\n \" summary.\"\n ),\n environment=\"remote\",\n)\nprint(result.output_text)\n" + }, + { + "label": "simple", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n input: 'Hello, how are you?',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "multi_turn", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n input: [\n {type: 'user_input', content: [{type: 'text', text: 'Hello'}]},\n {\n type: 'model_output',\n content: [\n {type: 'text', text: 'Hi there! How can I help you today?'},\n ],\n },\n {\n type: 'user_input',\n content: [{type: 'text', text: 'What is the capital of France?'}],\n },\n ],\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "multimodal_image", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n input: [\n {type: 'text', text: 'What is in this picture?'},\n {type: 'image', data: 'BASE64_ENCODED_IMAGE', mime_type: 'image/png'},\n ],\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "function_calling", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [\n {\n type: 'function',\n name: 'get_weather',\n description: 'Get the current weather in a given location',\n parameters: {\n type: 'object',\n properties: {\n location: {\n type: 'string',\n description: 'The city and state, e.g. San Francisco, CA',\n },\n },\n required: ['location'],\n },\n },\n ],\n input: 'What is the weather like in Boston, MA?',\n});\nconsole.log(interaction.steps.at(-1));\n" + }, + { + "label": "deep_research", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-pro-preview-12-2025',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n" + }, + { + "label": "antigravity", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'antigravity-preview-05-2026',\n input:\n 'Read Hacker News, summarize the top 5 stories, and save results as a markdown file.',\n environment: 'remote',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "reuse_env", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n// Step 1: Create an interaction with a fresh remote environment.\nconst interaction = await ai.interactions.create({\n agent: 'antigravity-preview-05-2026',\n input: 'Write a hello world script at /workspace/hello.py.',\n environment: 'remote',\n});\nconsole.log(`Environment ID: ${interaction.environment_id}`);\n\n// Step 2: Reuse the same environment in a follow-up interaction.\nconst interaction2 = await ai.interactions.create({\n agent: 'antigravity-preview-05-2026',\n input: 'Modify the script to accept a name argument and greet the user.',\n environment: interaction.environment_id,\n previous_interaction_id: interaction.id,\n});\nconsole.log(interaction2.output_text);\n" + }, + { + "label": "with_sources", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'antigravity-preview-05-2026',\n input: 'List all files under /workspace and summarize what you find.',\n environment: {\n type: 'remote',\n sources: [\n {\n type: 'repository',\n source: 'https://github.com/octocat/Spoon-Knife',\n target: '/workspace/repo',\n },\n {\n type: 'inline',\n content: 'Focus on Python files only.',\n target: '/workspace/notes.txt',\n },\n ],\n },\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "custom_agent", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n// Step 1: Create a custom agent.\nconst agentId = `code-reviewer-${crypto.randomUUID().slice(0, 8)}`;\nawait ai.agents.create({\n id: agentId,\n base_agent: 'antigravity-preview-05-2026',\n system_instruction:\n 'You are a senior code reviewer. Check every file for bugs, style issues, and security vulnerabilities.',\n base_environment: {\n type: 'remote',\n sources: [\n {\n type: 'repository',\n source: 'https://github.com/octocat/Spoon-Knife',\n target: '/workspace/repo',\n },\n ],\n },\n});\n\n// Step 2: Use the custom agent.\nconst result = await ai.interactions.create({\n agent: agentId,\n input:\n 'Review the latest changes in /workspace/repo/src and file a summary.',\n environment: 'remote',\n});\nconsole.log(result.output_text);\n" + }, + { + "label": "simple", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .input(InteractionsInput.of(\"Hello, how are you?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "multi_turn", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.ModelOutputStep;\nimport com.google.genai.gaos.models.interactions.Step;\nimport com.google.genai.gaos.models.interactions.TextContent;\nimport com.google.genai.gaos.models.interactions.UserInputStep;\n\nClient client = new Client();\nList conversation =\n List.of(\n UserInputStep.builder()\n .content(List.of(TextContent.builder().text(\"Hello!\").build()))\n .build(),\n ModelOutputStep.builder()\n .content(\n List.of(\n TextContent.builder().text(\"Hi there! How can I help you today?\").build()))\n .build(),\n UserInputStep.builder()\n .content(\n List.of(TextContent.builder().text(\"What is the capital of France?\").build()))\n .build());\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .input(InteractionsInput.ofStep(conversation))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "multimodal_image", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.Content;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.ImageContent;\nimport com.google.genai.gaos.models.interactions.ImageContentMimeType;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.TextContent;\n\nClient client = new Client();\nList content =\n List.of(\n TextContent.builder().text(\"What is in this picture?\").build(),\n ImageContent.builder()\n .data(\"BASE64_ENCODED_IMAGE\")\n .mimeType(ImageContentMimeType.IMAGE_PNG)\n .build());\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .input(InteractionsInput.ofContent(content))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "function_calling", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Function;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.Step;\n\nClient client = new Client();\nMap parameters =\n Map.of(\n \"type\", \"object\",\n \"properties\",\n Map.of(\n \"location\",\n Map.of(\n \"type\", \"string\",\n \"description\", \"The city and state, e.g. San Francisco, CA\")),\n \"required\", List.of(\"location\"));\nFunction functionTool =\n Function.builder()\n .name(\"get_weather\")\n .description(\"Get the current weather in a given location\")\n .parameters(parameters)\n .build();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(functionTool))\n .input(InteractionsInput.of(\"What is the weather like in Boston, MA?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nList steps = interaction.steps().orElse(List.of());\nif (!steps.isEmpty()) {\n System.out.println(steps.get(steps.size() - 1));\n}\n" + }, + { + "label": "deep_research", + "lang": "java", + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-pro-preview-12-2025\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" + }, + { + "label": "antigravity", + "lang": "java", + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteractionEnvironment;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"antigravity-preview-05-2026\"))\n .input(\n InteractionsInput.of(\n \"Read Hacker News, summarize the top 5 stories, and save results as a markdown\"\n + \" file.\"))\n .environment(CreateAgentInteractionEnvironment.of(\"remote\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "reuse_env", + "lang": "java", + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteractionEnvironment;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.environments.Environment;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.Step;\n\nClient client = new Client();\n\n// Step 1: Create an interaction with a fresh remote environment.\nCreateAgentInteraction params1 =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"antigravity-preview-05-2026\"))\n .input(InteractionsInput.of(\"Write a hello world script at /workspace/hello.py.\"))\n .environment(CreateAgentInteractionEnvironment.of(\"remote\"))\n .build();\nCreateInteractionResponse response1 =\n client.interactions.create(CreateInteractionRequestBody.of(params1));\nInteraction interaction1 =\n response1.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(\"Environment ID: \" + interaction1.environmentId().orElse(\"\"));\n\n// Step 2: Reuse the same environment in a follow-up interaction.\nCreateAgentInteraction params2 =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"antigravity-preview-05-2026\"))\n .input(\n InteractionsInput.of(\n \"Modify the script to accept a name argument and greet the user.\"))\n .environment(\n CreateAgentInteractionEnvironment.of(interaction1.environmentId().orElse(\"\")))\n .previousInteractionId(interaction1.id().orElse(null))\n .build();\nCreateInteractionResponse response2 =\n client.interactions.create(CreateInteractionRequestBody.of(params2));\nInteraction interaction2 =\n response2.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction2.outputText().orElse(\"\"));\n" + }, + { + "label": "with_sources", + "lang": "java", + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteractionEnvironment;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.environments.Environment;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.Source;\nimport com.google.genai.gaos.models.interactions.SourceType;\n\nClient client = new Client();\ncom.google.genai.gaos.models.interactions.Environment env =\n com.google.genai.gaos.models.interactions.Environment.builder()\n .sources(\n List.of(\n Source.builder()\n .type(SourceType.REPOSITORY)\n .source(\"https://github.com/octocat/Spoon-Knife\")\n .target(\"/workspace/repo\")\n .build(),\n Source.builder()\n .type(SourceType.INLINE)\n .content(\"Focus on Python files only.\")\n .target(\"/workspace/notes.txt\")\n .build()))\n .build();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"antigravity-preview-05-2026\"))\n .input(\n InteractionsInput.of(\n \"List all files under /workspace and summarize what you find.\"))\n .environment(CreateAgentInteractionEnvironment.of(env))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "custom_agent", + "lang": "java", + "source": "import com.google.genai.gaos.models.agents.Agent;\nimport com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.gaos.models.agents.BaseEnvironment;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteractionEnvironment;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.environments.Environment;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.Source;\nimport com.google.genai.gaos.models.interactions.SourceType;\nimport com.google.genai.gaos.models.interactions.Step;\n\nClient client = new Client();\n\n// Step 1: Create a custom agent.\nString agentId = \"code-reviewer-\" + UUID.randomUUID().toString().substring(0, 8);\ncom.google.genai.gaos.models.interactions.Environment baseEnv =\n com.google.genai.gaos.models.interactions.Environment.builder()\n .sources(\n List.of(\n Source.builder()\n .type(SourceType.REPOSITORY)\n .source(\"https://github.com/octocat/Spoon-Knife\")\n .target(\"/workspace/repo\")\n .build()))\n .build();\nAgent customAgent =\n Agent.builder()\n .id(agentId)\n .baseAgent(\"antigravity-preview-05-2026\")\n .systemInstruction(\n \"You are a senior code reviewer. Check every file for bugs, style issues, and\"\n + \" security vulnerabilities.\")\n .baseEnvironment(BaseEnvironment.of(baseEnv))\n .build();\nclient.agents.create(customAgent);\n\n// Step 2: Use the custom agent.\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(agentId))\n .input(\n InteractionsInput.of(\n \"Review the latest changes in /workspace/repo/src and file a summary.\"))\n .environment(CreateAgentInteractionEnvironment.of(\"remote\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "simple", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Hello, how are you?\")),\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + }, + { + "label": "multi_turn", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n history := []gaos_interactions.Step{\n gaos_interactions.NewStep(gaos_interactions.UserInputStep{\n Content: []gaos_interactions.Content{\n gaos_interactions.NewContent(gaos_interactions.TextContent{\n Text: \"Hello!\",\n }),\n },\n }),\n gaos_interactions.NewStep(gaos_interactions.ModelOutputStep{\n Content: []gaos_interactions.Content{\n gaos_interactions.NewContent(gaos_interactions.TextContent{\n Text: \"Hi there! How can I help you today?\",\n }),\n },\n }),\n gaos_interactions.NewStep(gaos_interactions.UserInputStep{\n Content: []gaos_interactions.Content{\n gaos_interactions.NewContent(gaos_interactions.TextContent{\n Text: \"What is the capital of France?\",\n }),\n },\n }),\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(history)),\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + }, + { + "label": "multimodal_image", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n contents := []gaos_interactions.Content{\n gaos_interactions.NewContent(gaos_interactions.TextContent{\n Text: \"What is in this picture?\",\n }),\n gaos_interactions.NewContent(gaos_interactions.ImageContent{\n Data: genai.Ptr(\"BASE64_ENCODED_IMAGE\"),\n MimeType: gaos_interactions.ImageContentMimeTypeImagePng.ToPointer(),\n }),\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(contents)),\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + }, + { + "label": "function_calling", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n function := gaos_interactions.Function{\n Name: genai.Ptr(\"get_weather\"),\n Description: genai.Ptr(\"Get the current weather in a given location\"),\n Parameters: map[string]any{\n \"type\": \"object\",\n \"properties\": map[string]any{\n \"location\": map[string]any{\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\",\n },\n },\n \"required\": []string{\"location\"},\n },\n }\n\n tool := gaos_interactions.NewTool(function)\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"What is the weather like in Boston, MA?\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n for _, step := range res.Interaction.Steps {\n if step.FunctionCallStep != nil {\n fmt.Println(\"Function Call:\", step.FunctionCallStep.Name)\n fmt.Println(\"Arguments:\", step.FunctionCallStep.Arguments)\n }\n }\n }\n}\n" + }, + { + "label": "deep_research", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-pro-preview-12-2025\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n" + }, + { + "label": "antigravity", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"antigravity-preview-05-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Read Hacker News, summarize the top 5 stories, and save results as a markdown file.\")),\n Environment: genai.Ptr(gaos_interactions.NewCreateAgentInteractionEnvironment(\"remote\")),\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + }, + { + "label": "reuse_env", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n // Step 1: Create an interaction with a fresh remote environment.\n body1 := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"antigravity-preview-05-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Write a hello world script at /workspace/hello.py.\")),\n Environment: genai.Ptr(gaos_interactions.NewCreateAgentInteractionEnvironment(\"remote\")),\n })\n\n res1, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body1})\n if err != nil {\n log.Fatal(err)\n }\n\n if res1.Interaction == nil || res1.Interaction.EnvironmentID == nil {\n log.Fatal(\"No environment ID returned\")\n }\n fmt.Printf(\"Environment ID: %s\\n\", *res1.Interaction.EnvironmentID)\n\n // Step 2: Reuse the same environment in a follow-up interaction.\n body2 := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"antigravity-preview-05-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Modify the script to accept a name argument and greet the user.\")),\n Environment: genai.Ptr(gaos_interactions.NewCreateAgentInteractionEnvironment(*res1.Interaction.EnvironmentID)),\n PreviousInteractionID: res1.Interaction.ID,\n })\n\n res2, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body2})\n if err != nil {\n log.Fatal(err)\n }\n\n if res2.Interaction != nil && res2.Interaction.OutputText != nil {\n fmt.Println(*res2.Interaction.OutputText)\n }\n}\n" + }, + { + "label": "with_sources", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n env := gaos_interactions.Environment{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeRepository),\n Source: genai.Ptr(\"https://github.com/octocat/Spoon-Knife\"),\n Target: genai.Ptr(\"/workspace/repo\"),\n },\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Content: genai.Ptr(\"Focus on Python files only.\"),\n Target: genai.Ptr(\"/workspace/notes.txt\"),\n },\n },\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"antigravity-preview-05-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"List all files under /workspace and summarize what you find.\")),\n Environment: genai.Ptr(gaos_interactions.NewCreateAgentInteractionEnvironment(env)),\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + }, + { + "label": "custom_agent", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"time\"\n \"google.golang.org/genai\"\n gaos_agents \"google.golang.org/genai/interactions/models/agents\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n // Step 1: Create a custom agent.\n agentID := fmt.Sprintf(\"code-reviewer-%d\", time.Now().UnixNano())\n baseEnv := gaos_interactions.Environment{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeRepository),\n Source: genai.Ptr(\"https://github.com/octocat/Spoon-Knife\"),\n Target: genai.Ptr(\"/workspace/repo\"),\n },\n },\n }\n\n _, err = client.Agents.Create(ctx, operations.CreateAgentRequest{\n Body: gaos_agents.Agent{\n ID: genai.Ptr(agentID),\n BaseAgent: genai.Ptr(\"antigravity-preview-05-2026\"),\n SystemInstruction: genai.Ptr(\"You are a senior code reviewer. Check every file for bugs, style issues, and security vulnerabilities.\"),\n BaseEnvironment: genai.Ptr(gaos_agents.NewBaseEnvironment(baseEnv)),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n // Step 2: Use the custom agent.\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(agentID),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Review the latest changes in /workspace/repo/src and file a summary.\")),\n Environment: genai.Ptr(gaos_interactions.NewCreateAgentInteractionEnvironment(\"remote\")),\n })\n\n res, err := client.Interactions.Create(ctx, operations.CreateInteractionRequest{Body: body})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + } + }, + "/{api_version}/interactions/{id}": { + "delete": { + "operationId": "deleteInteraction", + "description": "Deletes the interaction by id.", + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + }, + { + "name": "id", + "in": "path", + "description": "The unique identifier of the interaction to delete.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "interactionsId", + "in": "path", + "description": "Required. The name of the interaction to delete.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful deletion of the interaction." + }, + "4XX": { + "description": "Error deleting interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "5XX": { + "description": "Error deleting interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + } + }, + "summary": "Deleting an interaction", + "x-speakeasy-exports": [ + { + "name": "InteractionDeleteParams", + "group": "interactions", + "representation": "input" + } + ], + "x-speakeasy-group": "interactions", + "x-speakeasy-name-override": "delete", + "x-codeSamples": [ + { + "label": "delete", + "lang": "sh", + "source": "curl -X DELETE \"https://generativelanguage.googleapis.com/v1beta/interactions/$INTERACTION_ID\" \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "delete", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\nclient.interactions.delete(id=created.id)\nprint(\"Interaction deleted successfully.\")\n" + }, + { + "label": "delete", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nawait ai.interactions.delete(created.id);\nconsole.log('Interaction deleted successfully.');\n" + }, + { + "label": "delete", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.Interaction;\n\nClient client = new Client();\n\n\nclient.interactions.delete(interactionId);\nSystem.out.println(\"Interaction deleted successfully.\");\n" + }, + { + "label": "delete", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n req := operations.DeleteInteractionRequest{\n ID: interactionID,\n }\n\n _, err = client.Interactions.Delete(\n ctx,\n req,\n )\n if err != nil {\n log.Fatal(err)\n }\n\n fmt.Println(\"Interaction deleted successfully.\")\n}\n" + } + ] + }, + "get": { + "operationId": "getInteractionById", + "description": "Retrieves the full details of a single interaction based on its `Interaction.id`.", + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + }, + { + "name": "id", + "in": "path", + "description": "The unique identifier of the interaction to retrieve.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "last_event_id", + "in": "query", + "description": "Optional. If set, resumes the interaction stream from the next chunk after the event marked by the event id. Can only be used if `stream` is true.", + "schema": { + "type": "string" + } + }, + { + "name": "stream", + "in": "query", + "description": "If set to true, the generated content will be streamed incrementally.", + "schema": { + "type": "boolean", + "default": false + } + } + ], + "responses": { + "200": { + "description": "Successful retrieval of the interaction.", + "content": { + "application/json": { + "examples": { + "get": { + "summary": "Get Interaction", + "value": { + "created": "2025-11-26T12:25:15Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "I'm doing great, thank you for asking! How can I help you today?" + } + ] + } + ], + "updated": "2025-11-26T12:25:15Z" + } + } + }, + "schema": { + "$ref": "#/components/schemas/Interaction" + } + }, + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/InteractionSseStreamEvent" + }, + "x-speakeasy-sse-sentinel": "[DONE]" + } + } + }, + "4XX": { + "description": "Error getting interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "5XX": { + "description": "Error getting interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + } + }, + "summary": "Retrieving an interaction", + "x-speakeasy-exports": [ + { + "name": "InteractionGetParams", + "group": "interactions", + "representation": "input" + } + ], + "x-speakeasy-group": "interactions", + "x-speakeasy-name-override": "get", + "x-speakeasy-sse-overload": true, + "x-codeSamples": [ + { + "label": "get", + "lang": "sh", + "source": "curl -X GET \"https://generativelanguage.googleapis.com/v1beta/interactions/$INTERACTION_ID\" \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Api-Revision: 2026-05-20\"\n" + }, + { + "label": "get", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n\ninteraction = client.interactions.get(id=created.id)\nprint(interaction.status)\n" + }, + { + "label": "get", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n\nconst interaction = await ai.interactions.get(created.id);\nconsole.log(interaction.status);\n" + }, + { + "label": "get", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.GetInteractionByIdRequest;\nimport com.google.genai.gaos.models.operations.GetInteractionByIdResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\n\nClient client = new Client();\n\n\nGetInteractionByIdResponse getResponse =\n client.interactions.get(new GetInteractionByIdRequest(interactionId));\nInteraction interaction =\n getResponse\n .interaction()\n .orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" + }, + { + "label": "get", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n interaction, err := client.Interactions.Get(ctx, operations.GetInteractionByIDRequest{\n ID: *created.Interaction.ID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if interaction.Interaction != nil {\n fmt.Println(interaction.Interaction.Status)\n }\n}\n" + } + ] + } + }, + "/{api_version}/interactions/{id}/cancel": { + "post": { + "operationId": "cancelInteractionById", + "description": "Cancels an interaction by id. This only applies to background interactions that are still running.", + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + }, + { + "name": "id", + "in": "path", + "description": "The unique identifier of the interaction to cancel.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful operation", + "content": { + "application/json": { + "examples": { + "cancel": { + "summary": "Cancel Interaction", + "value": { + "created": "2025-11-26T12:25:15Z", + "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "cancelled", + "updated": "2025-11-26T12:25:15Z" + } + } + }, + "schema": { + "$ref": "#/components/schemas/Interaction" + } + } + } + }, + "4XX": { + "description": "Error cancelling interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "5XX": { + "description": "Error cancelling interaction", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "$ref": "#/components/schemas/Error" + } + }, + "required": [ + "error" + ] + } + } + } + } + }, + "summary": "Canceling an interaction", + "x-speakeasy-group": "interactions", + "x-speakeasy-name-override": "cancel", + "x-codeSamples": [ + { + "label": "cancel", + "lang": "sh", + "source": "curl -X POST \"https://generativelanguage.googleapis.com/v1beta/interactions/$INTERACTION_ID/cancel\" \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "cancel", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n# Start a background interaction so it stays in-progress.\ncreated = client.interactions.create(\n model=\"gemini-flash-latest\",\n input=\"Write a long essay about the history of computing.\",\n tools=[{\"type\": \"computer_use\"}],\n background=True,\n)\n\n# Cancel the in-progress interaction.\ninteraction = client.interactions.cancel(id=created.id)\nprint(interaction.status)\n" + }, + { + "label": "cancel", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n// Start a background interaction so it stays in-progress.\nconst created = await ai.interactions.create({\n model: 'gemini-flash-latest',\n input: 'Write a long essay about the history of computing.',\n tools: [{type: 'computer_use'}],\n background: true,\n});\n\n// Cancel the in-progress interaction.\nconst interaction = await ai.interactions.cancel(created.id);\nconsole.log(interaction.status);\n" + }, + { + "label": "cancel", + "lang": "java", + "source": "import com.google.genai.gaos.models.operations.CancelInteractionByIdResponse;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.ComputerUse;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\n\n// Start a background interaction so it stays in-progress.\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .input(InteractionsInput.of(\"Write a long essay about the history of computing.\"))\n .tools(List.of(new ComputerUse()))\n .background(true)\n .build();\nCreateInteractionResponse created =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nString interactionId = created.interaction().flatMap(Interaction::id).orElseThrow();\n\n// Cancel the in-progress interaction.\nCancelInteractionByIdResponse cancelResponse = client.interactions.cancel(interactionId);\nInteraction interaction =\n cancelResponse\n .interaction()\n .orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" + }, + { + "label": "cancel", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n req := operations.CancelInteractionByIDRequest{\n ID: interactionID,\n }\n\n res, err := client.Interactions.Cancel(\n ctx,\n req,\n )\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n" + } + ] + } + }, + "/{api_version}/triggers": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListTriggers", + "description": "Lists triggers for a project.", + "parameters": [ + { + "name": "filter", + "in": "query", + "description": "Optional. Filter expression (e.g., by state).", + "schema": { + "type": "string" + } + }, + { + "name": "page_size", + "in": "query", + "description": "Optional. The maximum number of triggers to return per page.", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Optional. A page token from a previous ListTriggers call.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListTriggersResponse" + } + } + } + } + }, + "tags": [ + "triggers" + ] + }, + "post": { + "operationId": "CreateTrigger", + "description": "Creates a new trigger that will invoke the specified agent on the given\ncron schedule.", + "parameters": [], + "requestBody": { + "description": "Required. The trigger configuration to create.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trigger" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trigger" + } + } + } + } + }, + "tags": [ + "triggers" + ], + "x-codeSamples": [ + { + "label": "create", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/triggers \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"schedule\": \"0 9 * * *\",\n \"time_zone\": \"America/New_York\",\n \"interaction\": {\n \"agent\": \"antigravity-preview-05-2026\",\n \"input\": \"Summarize top news stories.\",\n \"environment\": \"remote\"\n }\n }'\n" + }, + { + "label": "create", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\ntrigger = client.triggers.create(\n schedule=\"0 9 * * *\",\n time_zone=\"America/New_York\",\n interaction={\n \"agent\": \"antigravity-preview-05-2026\",\n \"input\": \"Summarize top news stories.\",\n \"environment\": \"remote\",\n },\n)\nprint(trigger.id, trigger.interaction.agent)\n" + }, + { + "label": "create", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst trigger = await ai.triggers.create({\n schedule: '0 9 * * *',\n time_zone: 'America/New_York',\n interaction: {\n agent: 'antigravity-preview-05-2026',\n input: 'Summarize top news stories.',\n environment: 'remote',\n },\n});\nconsole.log(trigger.id, trigger.interaction.agent);\n" + }, + { + "label": "create", + "lang": "java", + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteractionEnvironment;\nimport com.google.genai.gaos.models.operations.CreateTriggerResponse;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.triggers.Trigger;\nimport com.google.genai.gaos.models.triggers.TriggerCreateParams;\n\nClient client = new Client();\nCreateAgentInteraction interaction =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"antigravity-preview-05-2026\"))\n .input(InteractionsInput.of(\"Summarize top news stories.\"))\n .environment(CreateAgentInteractionEnvironment.of(\"remote\"))\n .build();\nTriggerCreateParams params =\n TriggerCreateParams.builder()\n .schedule(\"0 9 * * *\")\n .timeZone(\"America/New_York\")\n .interaction(interaction)\n .build();\nCreateTriggerResponse response = client.triggers.create(params);\nTrigger trigger =\n response.trigger().orElseThrow(() -> new RuntimeException(\"No trigger returned\"));\nSystem.out.println(\n trigger.id().orElse(\"\")\n + \" \"\n + trigger.interaction().flatMap(CreateAgentInteraction::agent).orElse(null));\n" + } + ] + } + }, + "/{api_version}/triggers/{id}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "delete": { + "operationId": "DeleteTrigger", + "description": "Deletes a trigger.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource name of the trigger.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Empty" + } + } + } + } + }, + "tags": [ + "triggers" + ] + }, + "get": { + "operationId": "GetTrigger", + "description": "Gets details of a single trigger.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource name of the trigger.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trigger" + } + } + } + } + }, + "tags": [ + "triggers" + ] + }, + "patch": { + "operationId": "UpdateTrigger", + "description": "Updates a trigger.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. Resource name of the trigger.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Required. The trigger with updated fields.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateTriggerRequest" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trigger" + } + } + } + } + }, + "tags": [ + "triggers" + ] + } + }, + "/{api_version}/triggers/{triggerId}/executions": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListTriggerExecutions", + "description": "Lists executions for a trigger.", + "parameters": [ + { + "name": "filter", + "in": "query", + "description": "Optional. Filter expression.", + "schema": { + "type": "string" + } + }, + { + "name": "page_size", + "in": "query", + "description": "The maximum number of executions to return per page.", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "A page token from a previous ListTriggerExecutions call.", + "schema": { + "type": "string" + } + }, + { + "name": "triggerId", + "in": "path", + "description": "Required. The trigger ID to list executions from.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListTriggerExecutionsResponse" + } + } + } + } + }, + "tags": [ + "triggers.executions" + ] + }, + "post": { + "operationId": "RunTrigger", + "description": "Runs a trigger immediately.", + "parameters": [ + { + "name": "triggerId", + "in": "path", + "description": "Required. Resource name of the trigger.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TriggerExecution" + } + } + } + } + }, + "tags": [ + "triggers" + ] + } + }, + "/{api_version}/voices": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListVoices", + "description": "Lists custom stored voices owned by the caller (ordered newest first)\nfollowed by prebuilt system voices from Google's voice catalog.", + "parameters": [ + { + "name": "accent", + "in": "query", + "description": "Optional. Filter by accent description (e.g. \"American\", \"British\").\nCase-insensitive exact match. If multiple values are specified, matches\nvoices with any of the specified accents (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "context", + "in": "query", + "description": "Optional. Filter by intended context or domain (e.g. \"News, Commercial\").\nCase-insensitive exact match. If multiple values are specified, matches\nvoices with any of the specified contexts (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "gender", + "in": "query", + "description": "Optional. Filter by gender presentation (e.g. \"female\", \"male\", \"neutral\").\nCase-insensitive exact match. If multiple values are specified, matches\nvoices with any of the specified genders (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "language_code", + "in": "query", + "description": "Optional. Filter by BCP-47 language code (e.g. \"en-US\").\nCase-insensitive exact match. If multiple values are specified, matches\nvoices with any of the specified language codes (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "page_size", + "in": "query", + "description": "Optional. The maximum number of voices to return per page. The service may\nreturn fewer than this value. If unspecified, at most 50 voices are\nreturned. The maximum value is 1000; values above 1000 are coerced to\n1000.", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Optional. A page token received from a previous `ListVoices` call. Provide\nthis to retrieve the subsequent page.\n\nWhen paginating, all filter query parameters (`language_code`,\n`region_code`, `accent`, `persona`, `context`, `gender`, `pitch`, `type`,\nand `search`) must match the call that returned this token; otherwise the\nrequest fails with `INVALID_ARGUMENT`. `page_size` may change between\npages.", + "schema": { + "type": "string" + } + }, + { + "name": "persona", + "in": "query", + "description": "Optional. Filter by vocal persona (e.g. \"Warm, Friendly\").\nCase-insensitive exact match. If multiple values are specified, matches\nvoices with any of the specified personas (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "pitch", + "in": "query", + "description": "Optional. Filter by vocal pitch. Accepts `\"low\"`, `\"medium\"`, `\"high\"`\n(case-insensitive). If multiple values are specified, matches voices with\nany of the specified pitches (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "region_code", + "in": "query", + "description": "Optional. Filter by ISO 3166-1 alpha-2 or UN M.49 region code (e.g. \"US\",\n\"001\").\nCase-insensitive exact match. If multiple values are specified, matches\nvoices with any of the specified region codes (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "search", + "in": "query", + "description": "Optional. Free-text substring search query matched case-insensitively\nagainst both `display_name` and `description`. Maximum 2048 bytes.", + "schema": { + "type": "string" + } + }, + { + "name": "type", + "in": "query", + "description": "Optional. Filter by voice type. Accepts `\"prebuilt\"`, `\"replicated\"`,\n`\"prompted\"` (case-insensitive). If multiple values are specified, matches\nvoices with any of the specified types (OR).", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListVoicesResponse" + } + } + } + } + }, + "tags": [ + "voices" + ], + "x-codeSamples": [ + { + "label": "list", + "lang": "sh", + "source": "curl -X GET https://generativelanguage.googleapis.com/v1beta/voices \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "list", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.voices.list()\nfor voice in response.voices or []:\n print(voice.id, voice.display_name)\n" + }, + { + "label": "list", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst response = await ai.voices.list();\nfor (const voice of response.voices ?? []) {\n console.log(voice.id, voice.display_name);\n}\n" + }, + { + "label": "list", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n res, err := client.Voices.List(ctx, operations.ListVoicesRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListVoicesResponse != nil {\n for _, voice := range res.ListVoicesResponse.Voices {\n if voice.ID != nil && voice.DisplayName != nil {\n fmt.Println(*voice.ID, *voice.DisplayName)\n }\n }\n }\n}\n" + } + ] + }, + "post": { + "operationId": "CreateVoice", + "description": "Creates a custom voice from a natural-language prompt\n(`VOICE_TYPE_PROMPTED`) or from reference and consent audio recordings\n(`VOICE_TYPE_REPLICATED`).", + "parameters": [], + "requestBody": { + "description": "The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateVoiceRequest" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Voice" + } + } + } + } + }, + "tags": [ + "voices" + ], + "x-codeSamples": [ + { + "label": "create", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/voices \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"store\": true,\n \"voice\": {\n \"type\": \"prompted\",\n \"display_name\": \"Warm Narrator\",\n \"language_code\": \"en-US\",\n \"prompted\": {\n \"input\": \"A warm, friendly narrator voice with a calm pace.\"\n }\n }\n }'\n" + }, + { + "label": "create", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nvoice = client.voices.create(\n store=True,\n voice={\n \"type\": \"prompted\",\n \"display_name\": \"Warm Narrator\",\n \"language_code\": \"en-US\",\n \"prompted\": {\n \"input\": \"A warm, friendly narrator voice with a calm pace.\",\n },\n },\n)\nprint(voice.id)\n" + }, + { + "label": "create", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst voice = await ai.voices.create({\n store: true,\n voice: {\n type: 'prompted',\n display_name: 'Warm Narrator',\n language_code: 'en-US',\n prompted: {\n input: 'A warm, friendly narrator voice with a calm pace.',\n },\n },\n});\nconsole.log(voice.id);\n" + }, + { + "label": "create", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_voices \"google.golang.org/genai/interactions/models/voices\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n req := operations.CreateVoiceRequest{\n Body: gaos_voices.CreateVoiceRequest{\n Store: genai.Ptr(true),\n Voice: gaos_voices.VoiceInput{\n Type: gaos_voices.VoiceTypePrompted,\n DisplayName: genai.Ptr(\"Warm Narrator\"),\n LanguageCode: genai.Ptr(\"en-US\"),\n Prompted: &gaos_voices.PromptedVoice{\n Input: \"A warm, friendly narrator voice with a calm pace.\",\n },\n },\n },\n }\n\n res, err := client.Voices.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Voice != nil && res.Voice.ID != nil {\n fmt.Println(*res.Voice.ID)\n }\n}\n" + } + ] + } + }, + "/{api_version}/voices/{voicesId}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "delete": { + "operationId": "DeleteVoice", + "description": "Deletes a custom stored voice (`store = true`) by resource name.\nPrebuilt catalog voices (`VOICE_TYPE_PREBUILT`) cannot be deleted.", + "parameters": [ + { + "name": "voicesId", + "in": "path", + "description": "Required. The resource name of the custom stored voice to delete\n(for example, `voices/voice_abc123def456`).", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteVoiceResponse" + } + } + } + } + }, + "tags": [ + "voices" + ], + "x-codeSamples": [ + { + "label": "delete", + "lang": "sh", + "source": "curl -X DELETE https://generativelanguage.googleapis.com/v1beta/voices/voice_abc123 \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "delete", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nclient.voices.delete(id=\"voice_abc123\")\n" + }, + { + "label": "delete", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nawait ai.voices.delete('voice_abc123');\n" + }, + { + "label": "delete", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n _, err = client.Voices.Delete(ctx, operations.DeleteVoiceRequest{\n ID: \"voice_abc123\",\n })\n if err != nil {\n log.Fatal(err)\n }\n}\n" + } + ] + }, + "get": { + "operationId": "GetVoice", + "description": "Gets a custom stored voice (`store = true`) by resource name.\nPrebuilt catalog voices (`VOICE_TYPE_PREBUILT`) cannot be retrieved via\n`GetVoice`; use `ListVoices` instead.", + "parameters": [ + { + "name": "voicesId", + "in": "path", + "description": "Required. The resource name of the custom stored voice to retrieve\n(for example, `voices/voice_abc123def456`).", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Voice" + } + } + } + } + }, + "tags": [ + "voices" + ], + "x-codeSamples": [ + { + "label": "get", + "lang": "sh", + "source": "curl -X GET https://generativelanguage.googleapis.com/v1beta/voices/voice_abc123 \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\"\n" + }, + { + "label": "get", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nvoice = client.voices.get(id=\"voice_abc123\")\nprint(voice.id, voice.display_name)\n" + }, + { + "label": "get", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst voice = await ai.voices.get('voice_abc123');\nconsole.log(voice.id, voice.display_name);\n" + }, + { + "label": "get", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n res, err := client.Voices.Get(ctx, operations.GetVoiceRequest{\n ID: \"voice_abc123\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Voice != nil && res.Voice.ID != nil && res.Voice.DisplayName != nil {\n fmt.Println(*res.Voice.ID, *res.Voice.DisplayName)\n }\n}\n" + } + ] + } + }, + "/{api_version}/webhooks": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "get": { + "operationId": "ListWebhooks", + "description": "Lists all Webhooks.", + "parameters": [ + { + "name": "page_size", + "in": "query", + "description": "Optional. The maximum number of webhooks to return. The service may return\nfewer than this value. If unspecified, at most 50 webhooks will be\nreturned. The maximum value is 1000.", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Optional. A page token, received from a previous `ListWebhooks` call.\nProvide this to retrieve the subsequent page.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListWebhooksResponse" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + }, + "post": { + "operationId": "CreateWebhook", + "description": "Creates a new Webhook.", + "parameters": [ + { + "name": "webhook_id", + "in": "query", + "description": "The webhook_id to use for the webhook. If not specified, the server will\ngenerate a unique ID.", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Required. The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateWebhookRequest" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + } + }, + "/{api_version}/webhooks/{id}": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "delete": { + "operationId": "DeleteWebhook", + "description": "Deletes a Webhook.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. The ID of the webhook to delete.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Empty" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + }, + "get": { + "operationId": "GetWebhook", + "description": "Gets a specific Webhook.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. The ID of the webhook to retrieve.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + }, + "patch": { + "operationId": "UpdateWebhook", + "description": "Updates an existing Webhook.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. The ID of the webhook to update.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "Required. The request body.", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateWebhookRequest" + } + } + } + }, + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Webhook" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + } + }, + "/{api_version}/webhooks/{id}:ping": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "post": { + "operationId": "PingWebhook", + "description": "Sends a ping event to a Webhook.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. The ID of the webhook to ping.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PingWebhookResponse" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + } + }, + "/{api_version}/webhooks/{id}:rotateSigningSecret": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + } + ], + "post": { + "operationId": "RotateSigningSecret", + "description": "Generates a new signing secret for a Webhook.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Required. The ID of the webhook for which to generate a signing secret.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "default": { + "description": "Successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RotateSigningSecretResponse" + } + } + } + } + }, + "tags": [ + "webhooks" + ] + } + } + }, + "components": { + "parameters": { + "EnvironmentFileExtract": { + "name": "extract", + "in": "query", + "description": "Optional. If true, treats the uploaded file as a tar/tar.gz archive and unpacks it into `path`.", + "schema": { + "type": "boolean" + } + }, + "EnvironmentFileOverwrite": { + "name": "overwrite", + "in": "query", + "description": "Optional. Whether to overwrite the destination file if it already exists.", + "schema": { + "type": "boolean" + } + }, + "EnvironmentFileUploadCommand": { + "name": "X-Goog-Upload-Command", + "in": "header", + "description": "Command that starts the resumable upload session.", + "required": true, + "schema": { + "type": "string", + "const": "start", + "default": "start" + } + }, + "EnvironmentFileUploadContentLength": { + "name": "X-Goog-Upload-Header-Content-Length", + "in": "header", + "description": "Total number of file bytes that will be uploaded to the session URL.", + "required": true, + "schema": { + "type": "integer", + "format": "int64", + "minimum": 0 + } + }, + "EnvironmentFileUploadContentType": { + "name": "X-Goog-Upload-Header-Content-Type", + "in": "header", + "description": "MIME type of the file that will be uploaded to the session URL.", + "required": true, + "schema": { + "type": "string", + "minLength": 1 + } + }, + "EnvironmentFileUploadProtocol": { + "name": "X-Goog-Upload-Protocol", + "in": "header", + "description": "Resumable upload protocol selector.", + "required": true, + "schema": { + "type": "string", + "const": "resumable", + "default": "resumable" + } + }, + "api_version": { + "name": "api_version", + "in": "path", + "description": "API version for request routing.", + "required": true, + "schema": { + "type": "string" + } + }, + "google_genai_user_project": { + "name": "x-goog-user-project", + "in": "header", + "description": "Quota project header to send with Google GenAI API requests.", + "schema": { + "type": "string" + }, + "x-speakeasy-name-override": "user_project" + } + }, + "headers": { + "EnvironmentFileUploadUrl": { + "description": "Opaque URL to which file chunks are uploaded and finalized.", + "schema": { + "type": "string", + "format": "uri" + } + } + }, + "schemas": { + "Agent": { + "description": "An agent definition for the CreateAgent API.\nThis message is the target for annotation-parser-based JSON parsing.\nNew format:\n {\n \"id\": \"customer-sentinel\",\n \"base_agent\": \"\",\n \"system_instruction\": \"...\",\n \"base_environment\": { \"type\": \"remote\", \"sources\": [...] },\n \"tools\": [ {\"type\": \"code_execution\"} ]\n }", + "type": "object", + "properties": { + "agent_config": { + "description": "Configuration parameters for the agent.", + "oneOf": [ + { + "$ref": "#/components/schemas/AntigravityAgentConfig" + } + ] + }, + "base_environment": { + "description": "The environment configuration for the agent.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, + "id": { + "description": "The unique identifier for the agent.", + "type": "string" + }, + "base_agent": { + "description": "The base agent to extend.", + "type": "string" + }, + "system_instruction": { + "description": "System instruction for the agent.", + "type": "string" + }, + "description": { + "description": "Agent description for developers to quickly read and understand.", + "type": "string" + }, + "tools": { + "description": "The tools available to the agent.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AgentTool" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "AgentOption": { + "title": "Agent", + "description": "The agent to interact with.", + "type": "string", + "enum": [ + "deep-research-pro-preview-12-2025", + "deep-research-preview-04-2026", + "deep-research-max-preview-04-2026", + "antigravity-preview-05-2026" + ], + "x-speakeasy-enum-descriptions": [ + "Gemini Deep Research Agent", + "Gemini Deep Research Agent", + "Gemini Deep Research Max Agent", + "Use the Antigravity managed agent to perform multi-step tasks that require reasoning, file operations, and tool use." + ], + "x-speakeasy-enum-format": "union", + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-unknown-values": "allow" + }, + "AgentTool": { + "description": "A tool that the agent can use.", + "oneOf": [ + { + "$ref": "#/components/schemas/CodeExecution" + }, + { + "$ref": "#/components/schemas/Function" + }, + { + "$ref": "#/components/schemas/GoogleSearch" + }, + { + "$ref": "#/components/schemas/McpServer" + }, + { + "$ref": "#/components/schemas/UrlContext" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "AllowedTools": { + "description": "The configuration for allowed tools.", + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "auto", + "any", + "none", + "validated" + ], + "x-google-enum-descriptions": [ + "Auto tool choice.", + "Any tool choice.", + "No tool choice.", + "Validated tool choice." + ], + "x-speakeasy-model-namespace": "interactions", + "description": "The mode of the tool choice." + }, + "tools": { + "description": "The names of the allowed tools.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "Annotation": { + "description": "Citation information for model-generated content.", + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/FileCitation" + }, + { + "$ref": "#/components/schemas/PlaceCitation" + }, + { + "$ref": "#/components/schemas/SpeechAnnotation" + }, + { + "$ref": "#/components/schemas/UrlCitation" + }, + { + "$ref": "#/components/schemas/WordInfo" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "AntigravityAgentConfig": { + "description": "Configuration for the Antigravity agent runtime.\nProvides server-side control over the agent's execution environment\nand tool configuration.", + "type": "object", + "properties": { + "type": { + "const": "antigravity" + }, + "model": { + "description": "The model to use for agent reasoning.", + "type": "string" + }, + "max_total_tokens": { + "description": "Max total tokens for the agent run.", + "type": "string", + "format": "int64" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ArgumentsDelta": { + "type": "object", + "properties": { + "type": { + "const": "arguments_delta" + }, + "arguments": { + "type": "string" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "AudioContent": { + "description": "An audio content block.", + "type": "object", + "properties": { + "type": { + "const": "audio" + }, + "data": { + "description": "The audio content.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "description": "The URI of the audio.", + "type": "string" + }, + "channels": { + "description": "The number of audio channels.", + "type": "integer", + "format": "int32" + }, + "sample_rate": { + "description": "The sample rate of the audio.", + "type": "integer", + "format": "int32" + }, + "mime_type": { + "description": "The mime type of the audio.", + "type": "string", + "enum": [ + "audio/wav", + "audio/mp3", + "audio/aiff", + "audio/aac", + "audio/ogg", + "audio/flac", + "audio/mpeg", + "audio/m4a", + "audio/l16", + "audio/opus", + "audio/alaw", + "audio/mulaw", + "audio/webm" + ], + "x-google-enum-descriptions": [ + "WAV audio format", + "MP3 audio format", + "AIFF audio format", + "AAC audio format", + "OGG audio format", + "FLAC audio format", + "MPEG audio format", + "M4A audio format", + "L16 audio format", + "OPUS audio format", + "ALAW audio format", + "MULAW audio format", + "WEBM audio format" + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "audio": { + "summary": "Audio", + "value": { + "type": "audio", + "data": "BASE64_ENCODED_AUDIO", + "mime_type": "audio/wav" + } + } + } + ], + "x-speakeasy-exports": [ + { + "name": "audio_content_param", + "group": "interactions", + "representation": "input" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "AudioData": { + "description": "Audio payload used for voice creation.", + "type": "object", + "properties": { + "mime_type": { + "description": "Required. The IANA MIME type of the audio data (for example, `audio/wav` or\n`audio/mpeg`).", + "type": "string" + }, + "data": { + "description": "Required. The raw audio bytes.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "data", + "mime_type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "AudioDelta": { + "type": "object", + "properties": { + "type": { + "const": "audio" + }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, + "mime_type": { + "type": "string", + "enum": [ + "audio/wav", + "audio/mp3", + "audio/aiff", + "audio/aac", + "audio/ogg", + "audio/flac", + "audio/mpeg", + "audio/m4a", + "audio/l16", + "audio/opus", + "audio/alaw", + "audio/mulaw", + "audio/webm" + ], + "x-google-enum-descriptions": [ + "WAV audio format", + "MP3 audio format", + "AIFF audio format", + "AAC audio format", + "OGG audio format", + "FLAC audio format", + "MPEG audio format", + "M4A audio format", + "L16 audio format", + "OPUS audio format", + "ALAW audio format", + "MULAW audio format", + "WEBM audio format" + ] + }, + "rate": { + "description": "Deprecated. Use sample_rate instead. The value is ignored.", + "type": "integer", + "deprecated": true, + "format": "int32" + }, + "sample_rate": { + "description": "The sample rate of the audio.", + "type": "integer", + "format": "int32" + }, + "channels": { + "description": "The number of audio channels.", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "AudioResponseFormat": { + "description": "Configuration for audio output format.", + "type": "object", + "properties": { + "type": { + "const": "audio" + }, + "mime_type": { + "description": "The MIME type of the audio output.", + "type": "string", + "enum": [ + "audio/mp3", + "audio/ogg_opus", + "audio/l16", + "audio/wav", + "audio/alaw", + "audio/mulaw" + ], + "x-google-enum-descriptions": [ + "MP3 audio format.", + "OGG Opus audio format.", + "Raw PCM (L16) audio format.", + "WAV audio format.", + "A-law audio format.", + "Mu-law audio format." + ] + }, + "delivery": { + "description": "The delivery mode for the audio output.", + "type": "string", + "enum": [ + "inline", + "uri" + ], + "x-google-enum-descriptions": [ + "Audio data is returned inline in the response.", + "Audio data is returned as a URI." + ] + }, + "sample_rate": { + "description": "Sample rate in Hz.", + "type": "integer", + "format": "int32" + }, + "bit_rate": { + "description": "Bit rate in bits per second (bps). Only applicable for compressed formats\n(MP3, Opus).", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "examples": [ + { + "audio_response_format": { + "summary": "Audio Output", + "value": { + "type": "audio", + "sample_rate": 24000 + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CodeExecution": { + "description": "A tool that can be used by the model to execute code.", + "type": "object", + "properties": { + "type": { + "const": "code_execution" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "code_execution", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"code_execution\"\n }],\n \"input\": \"Calculate the first 10 Fibonacci numbers\"\n }'\n" + }, + { + "label": "code_execution", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[{\"type\": \"code_execution\"}],\n input=\"Calculate the first 10 Fibonacci numbers\",\n)\nprint(response.output_text)\n" + }, + { + "label": "code_execution", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [{type: 'code_execution'}],\n input: 'Calculate the first 10 Fibonacci numbers',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "code_execution", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CodeExecution;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(new CodeExecution()))\n .input(InteractionsInput.of(\"Calculate the first 10 Fibonacci numbers\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "code_execution", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n tool := gaos_interactions.NewTool(gaos_interactions.CodeExecution{})\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Calculate the first 10 Fibonacci numbers\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "CodeExecutionCallArguments": { + "description": "The arguments to pass to the code execution.", + "type": "object", + "properties": { + "language": { + "description": "Programming language of the `code`.", + "type": "string", + "enum": [ + "python", + "bash" + ], + "x-google-enum-descriptions": [ + "Python >= 3.10, with numpy and simpy available.", + "Bash. Only available for Gemini 3.0 model or above." + ] + }, + "code": { + "description": "The code to be executed.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "CodeExecutionCallDelta": { + "type": "object", + "properties": { + "arguments": { + "$ref": "#/components/schemas/CodeExecutionCallArguments" + }, + "type": { + "const": "code_execution_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "arguments", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CodeExecutionCallStep": { + "description": "Code execution call step.", + "type": "object", + "properties": { + "arguments": { + "description": "Required. The arguments to pass to the code execution.", + "$ref": "#/components/schemas/CodeExecutionCallStepArguments" + }, + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "code_execution_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "arguments", + "id", + "type" + ], + "examples": [ + { + "code_execution_call": { + "summary": "CodeExecutionCallStep", + "value": { + "type": "code_execution_call", + "arguments": { + "code": "print(sum(range(1, 11)))" + }, + "id": "code_call_71021" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CodeExecutionCallStepArguments": { + "description": "The arguments to pass to the code execution.", + "type": "object", + "properties": { + "language": { + "description": "Programming language of the `code`.", + "type": "string", + "enum": [ + "python", + "bash" + ], + "x-google-enum-descriptions": [ + "Python >= 3.10, with numpy and simpy available.", + "Bash. Only available for Gemini 3.0 model or above." + ] + }, + "code": { + "description": "The code to be executed.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "CodeExecutionCallArguments" + }, + "CodeExecutionResultDelta": { + "type": "object", + "properties": { + "result": { + "type": "string" + }, + "type": { + "const": "code_execution_result" + }, + "is_error": { + "type": "boolean" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "result", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CodeExecutionResultStep": { + "description": "Code execution result step.", + "type": "object", + "properties": { + "result": { + "description": "Required. The output of the code execution.", + "type": "string" + }, + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "code_execution_result" + }, + "is_error": { + "description": "Whether the code execution resulted in an error.", + "type": "boolean" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "call_id", + "result", + "type" + ], + "examples": [ + { + "code_execution_result": { + "summary": "CodeExecutionResultStep", + "value": { + "type": "code_execution_result", + "call_id": "code_call_71021", + "result": "55\n" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ComputerUse": { + "description": "A tool that can be used by the model to interact with the computer.", + "type": "object", + "properties": { + "type": { + "const": "computer_use" + }, + "environment": { + "description": "The environment being operated.", + "type": "string", + "enum": [ + "browser", + "mobile", + "desktop" + ], + "x-google-enum-descriptions": [ + "Operates in a web browser.", + "Operates in a mobile environment.", + "Operates in a desktop environment." + ] + }, + "excluded_predefined_functions": { + "description": "The list of predefined functions that are excluded from the model call.", + "type": "array", + "items": { + "type": "string" + } + }, + "enable_prompt_injection_detection": { + "description": "Whether enable the prompt injection detection check on computer-use\nrequest.", + "type": "boolean" + }, + "disabled_safety_policies": { + "description": "Optional. Disabled safety policies for computer use.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "financial_transactions", + "sensitive_data_modification", + "communication_tool", + "account_creation", + "data_modification", + "user_consent_management", + "legal_terms_and_agreements" + ], + "x-google-enum-descriptions": [ + "Safety policy for financial transactions.", + "Safety policy for sensitive data modification.", + "Safety policy for communication tools (e.g. Gmail, Chat, Meet).", + "Safety policy for account creation.", + "Safety policy for data modification.", + "Safety policy for user consent management.", + "Safety policy for legal terms and agreements." + ] + } + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "computer_use", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-2.5-computer-use-preview-10-2025\",\n \"tools\": [{\n \"type\": \"computer_use\"\n }],\n \"input\": \"Find a flight to Tokyo\"\n }'\n" + }, + { + "label": "computer_use", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-2.5-computer-use-preview-10-2025\",\n tools=[{\"type\": \"computer_use\"}],\n input=\"Find a flight to Tokyo\",\n)\nprint(response.output_text)\n" + }, + { + "label": "computer_use", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-2.5-computer-use-preview-10-2025',\n tools: [{type: 'computer_use'}],\n input: 'Find a flight to Tokyo',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "computer_use", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.ComputerUse;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-2.5-computer-use-preview-10-2025\")\n .tools(List.of(new ComputerUse()))\n .input(InteractionsInput.of(\"Find a flight to Tokyo\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "computer_use", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n tool := gaos_interactions.NewTool(gaos_interactions.ComputerUse{})\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-2.5-computer-use-preview-10-2025\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Find a flight to Tokyo\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "Content": { + "description": "The content of the response.", + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/AudioContent" + }, + { + "$ref": "#/components/schemas/DocumentContent" + }, + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/TextContent" + }, + { + "$ref": "#/components/schemas/VideoContent" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CreateAgentInteractionParams": { + "description": "Parameters for creating agent interactions", + "properties": { + "agent": { + "description": "The name of the `Agent` used for generating the interaction.", + "$ref": "#/components/schemas/AgentOption" + }, + "input": { + "$ref": "#/components/schemas/InteractionsInput" + }, + "agent_config": { + "description": "Configuration parameters for the agent interaction.", + "oneOf": [ + { + "$ref": "#/components/schemas/AntigravityAgentConfig" + }, + { + "$ref": "#/components/schemas/DeepResearchAgentConfig" + }, + { + "$ref": "#/components/schemas/DynamicAgentConfig" + } + ] + }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, + "id": { + "description": "Required. Output only. A unique identifier for the interaction completion.", + "type": "string", + "readOnly": true + }, + "status": { + "description": "Required. Output only. The status of the interaction.", + "type": "string", + "enum": [ + "in_progress", + "requires_action", + "completed", + "failed", + "cancelled", + "incomplete", + "budget_exceeded" + ], + "readOnly": true, + "x-google-enum-deprecated": [ + false, + false, + false, + false, + false, + false, + true + ], + "x-google-enum-descriptions": [ + "The interaction is in progress.", + "The interaction requires action/input from the user.", + "The interaction is completed.", + "The interaction failed.", + "The interaction was cancelled.", + "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + ] + }, + "created": { + "description": "Required. Output only. The time at which the response was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "updated": { + "description": "Required. Output only. The time at which the response was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "system_instruction": { + "description": "System instruction for the interaction.", + "type": "string" + }, + "tools": { + "description": "A list of tool declarations the model may call during interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Tool" + } + }, + "response_modalities": { + "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", + "type": "array", + "deprecated": true, + "items": { + "$ref": "#/components/schemas/ResponseModality" + } + }, + "response_mime_type": { + "description": "The mime type of the response. This is required if response_format is set.", + "type": "string", + "deprecated": true + }, + "previous_interaction_id": { + "description": "The ID of the previous interaction, if any.", + "type": "string" + }, + "environment_id": { + "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", + "type": "string", + "readOnly": true + }, + "service_tier": { + "description": "The service tier for the interaction.", + "$ref": "#/components/schemas/ServiceTier" + }, + "webhook_config": { + "description": "Optional. Webhook configuration for receiving notifications when the\ninteraction completes.", + "$ref": "#/components/schemas/WebhookConfig" + }, + "safety_settings": { + "description": "Safety settings for the interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SafetySetting" + } + }, + "labels": { + "description": "The labels with user-defined metadata for the request.\n\nLabel keys and values can be no longer than 63 characters\n(Unicode codepoints) and can only contain lowercase letters, numeric\ncharacters, underscores, and dashes. International characters are allowed.\nLabel values are optional. Label keys must start with a letter.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "stream": { + "description": "Input only. Whether the interaction will be streamed.", + "type": "boolean", + "writeOnly": true + }, + "store": { + "description": "Input only. Whether to store the response and request for later retrieval.", + "type": "boolean", + "writeOnly": true + }, + "background": { + "description": "Input only. Whether to run the model interaction in the background.", + "type": "boolean", + "writeOnly": true + } + }, + "required": [ + "agent", + "input" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "CreateAgentInteraction" + }, + "CreateEnvironmentRequest": { + "description": "Request for `CreateEnvironment`.", + "type": "object", + "properties": { + "network": { + "description": "Network configuration for the environment.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" + }, + { + "type": "string", + "enum": [ + "disabled" + ], + "x-google-enum-descriptions": [ + "All network egress is blocked." + ] + } + ] + }, + "sources": { + "description": "Sources to be mounted into the environment.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Source" + } + }, + "from_environment": { + "description": "Optional. The source environment to copy/fork from.\nFormat: `environments/{environment_id}` or `{environment_id}`.\nWhen specified, `sources` and `env` must be empty.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "CreateModelInteractionParams": { + "description": "Parameters for creating model interactions", + "properties": { + "model": { + "description": "The name of the `Model` used for generating the interaction.", + "$ref": "#/components/schemas/ModelOption" + }, + "input": { + "$ref": "#/components/schemas/InteractionsInput" + }, + "generation_config": { + "description": "Input only. Configuration parameters for the model interaction.", + "$ref": "#/components/schemas/GenerationConfig", + "writeOnly": true + }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, + "id": { + "description": "Required. Output only. A unique identifier for the interaction completion.", + "type": "string", + "readOnly": true + }, + "status": { + "description": "Required. Output only. The status of the interaction.", + "type": "string", + "enum": [ + "in_progress", + "requires_action", + "completed", + "failed", + "cancelled", + "incomplete", + "budget_exceeded" + ], + "readOnly": true, + "x-google-enum-deprecated": [ + false, + false, + false, + false, + false, + false, + true + ], + "x-google-enum-descriptions": [ + "The interaction is in progress.", + "The interaction requires action/input from the user.", + "The interaction is completed.", + "The interaction failed.", + "The interaction was cancelled.", + "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + ] + }, + "created": { + "description": "Required. Output only. The time at which the response was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "updated": { + "description": "Required. Output only. The time at which the response was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "system_instruction": { + "description": "System instruction for the interaction.", + "type": "string" + }, + "tools": { + "description": "A list of tool declarations the model may call during interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Tool" + } + }, + "response_modalities": { + "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", + "type": "array", + "deprecated": true, + "items": { + "$ref": "#/components/schemas/ResponseModality" + } + }, + "response_mime_type": { + "description": "The mime type of the response. This is required if response_format is set.", + "type": "string", + "deprecated": true + }, + "previous_interaction_id": { + "description": "The ID of the previous interaction, if any.", + "type": "string" + }, + "environment_id": { + "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", + "type": "string", + "readOnly": true + }, + "service_tier": { + "description": "The service tier for the interaction.", + "$ref": "#/components/schemas/ServiceTier" + }, + "webhook_config": { + "description": "Optional. Webhook configuration for receiving notifications when the\ninteraction completes.", + "$ref": "#/components/schemas/WebhookConfig" + }, + "safety_settings": { + "description": "Safety settings for the interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SafetySetting" + } + }, + "labels": { + "description": "The labels with user-defined metadata for the request.\n\nLabel keys and values can be no longer than 63 characters\n(Unicode codepoints) and can only contain lowercase letters, numeric\ncharacters, underscores, and dashes. International characters are allowed.\nLabel values are optional. Label keys must start with a letter.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "stream": { + "description": "Input only. Whether the interaction will be streamed.", + "type": "boolean", + "writeOnly": true + }, + "store": { + "description": "Input only. Whether to store the response and request for later retrieval.", + "type": "boolean", + "writeOnly": true + }, + "background": { + "description": "Input only. Whether to run the model interaction in the background.", + "type": "boolean", + "writeOnly": true + } + }, + "required": [ + "input", + "model" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "CreateModelInteraction" + }, + "CreateVoiceRequest": { + "description": "Request message for `VoicesService.CreateVoice`.", + "type": "object", + "properties": { + "voice": { + "description": "Required. The voice to create.\n`voice.model` is optional; if omitted, the service selects the default\nvoice creation model. Output-only fields on `Voice` (`id`, `key`,\n`expire_time`, `usage`) are ignored if set.", + "$ref": "#/components/schemas/Voice" + }, + "store": { + "description": "Optional. Whether the created voice is persisted and managed by Google.\n\n* When `true`, Google stores the voice and returns `Voice.id` (for example,\n `voice_abc123def456`), which can be managed via `GetVoice`, `ListVoices`,\n and `DeleteVoice` and referenced by ID in synthesis requests. Stored\n voices expire after 1 year of inactivity; using a stored voice in speech\n synthesis or as a `base_voice` in `CreateVoice` extends its\n `expire_time`. Projects are subject to a maximum active stored voice\n quota; exceeding the quota returns `RESOURCE_EXHAUSTED`.\n* When `false` (default), the voice is not stored by Google and `Voice.key`\n (for example, `voicekey_...`) is returned for client-side storage and\n synthesis. Optional discovery metadata fields on `voice` are not\n persisted or returned when `store` is `false`.\n* Required to be `true` when `voice.type` is `\"prompted\"`\n (otherwise fails with `INVALID_ARGUMENT`).", + "type": "boolean" + } + }, + "required": [ + "voice" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CreateWebhookRequest": { + "description": "Request message for WebhookService.CreateWebhook.", + "type": "object", + "properties": { + "webhook": { + "description": "Required. The webhook to create.", + "$ref": "#/components/schemas/Webhook" + } + }, + "required": [ + "webhook" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Credential": { + "description": "Server-managed credential resource stored in Secret Manager.", + "type": "object", + "properties": { + "id": { + "description": "Required. Output only. Identifier. Unique identifier for the credential.", + "type": "string", + "readOnly": true, + "x-google-identifier": true + }, + "type": { + "description": "Required. Output only. The type of credential.", + "type": "string", + "enum": [ + "bearer_token", + "oauth2", + "environment_variable" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "Static token injected as header. No refresh logic.", + "Auto-refresh expired access tokens using stored refresh token.", + "Environment variable injected into sandbox container." + ] + }, + "status": { + "description": "Output only. Current status of the credential.", + "type": "string", + "enum": [ + "active", + "revoked" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "The credential is active and valid for use.", + "The credential has been revoked and is no longer valid." + ] + }, + "create_time": { + "description": "Output only. The timestamp when the credential was created.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "update_time": { + "description": "Output only. The timestamp when the credential was last updated.", + "type": "string", + "format": "date-time", + "readOnly": true + } + }, + "required": [ + "id", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CredentialCreateParams": { + "description": "Represents the fields of a Credential provided on creation.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentVariableConfig" + }, + { + "$ref": "#/components/schemas/HttpBearerConfig" + }, + { + "$ref": "#/components/schemas/OAuth2Config" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "CredentialUpdateParams": { + "description": "Represents the fields of a Credential that can be updated.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentVariableUpdateConfig" + }, + { + "$ref": "#/components/schemas/HttpBearerUpdateConfig" + }, + { + "$ref": "#/components/schemas/OAuth2UpdateConfig" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "DeepResearchAgentConfig": { + "description": "Configuration for the Deep Research agent.", + "type": "object", + "properties": { + "type": { + "const": "deep-research" + }, + "thinking_summaries": { + "description": "Whether to include thought summaries in the response.", + "$ref": "#/components/schemas/ThinkingSummaries" + }, + "visualization": { + "description": "Whether to include visualizations in the response.", + "type": "string", + "enum": [ + "off", + "auto" + ], + "x-google-enum-descriptions": [ + "Do not include visualizations.", + "Automatically include visualizations." + ] + }, + "collaborative_planning": { + "description": "Enables human-in-the-loop planning for the Deep Research agent. If set to\ntrue, the Deep Research agent will provide a research plan in its response.\nThe agent will then proceed only if the user confirms the plan in the next\nturn.", + "type": "boolean" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "DeleteVoiceResponse": { + "description": "Response message for `VoicesService.DeleteVoice`.", + "type": "object", + "x-speakeasy-model-namespace": "interactions" + }, + "DocumentContent": { + "description": "A document content block.", + "type": "object", + "properties": { + "type": { + "const": "document" + }, + "data": { + "description": "The document content.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "description": "The URI of the document.", + "type": "string" + }, + "mime_type": { + "description": "The mime type of the document.", + "type": "string", + "enum": [ + "application/pdf", + "text/csv" + ], + "x-google-enum-descriptions": [ + "PDF document format", + "CSV document format" + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "document": { + "summary": "Document", + "value": { + "type": "document", + "data": "BASE64_ENCODED_DOCUMENT", + "mime_type": "application/pdf" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "DocumentDelta": { + "type": "object", + "properties": { + "type": { + "const": "document" + }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, + "mime_type": { + "type": "string", + "enum": [ + "application/pdf", + "text/csv" + ], + "x-google-enum-descriptions": [ + "PDF document format", + "CSV document format" + ] + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "DynamicAgentConfig": { + "description": "Configuration for dynamic agents.", + "type": "object", + "properties": { + "type": { + "const": "dynamic" + } + }, + "required": [ + "type" + ], + "additionalProperties": { + "description": "For agents that are not supported statically in the API definition." + }, + "x-speakeasy-model-namespace": "interactions" + }, + "EgressRule": { + "description": "A single domain allowlist rule with optional header injection.", + "type": "object", + "properties": { + "domain": { + "description": "Domain to allow outbound requests to. Supports wildcards (e.g.\n'*.googleapis.com'). Use '*' to allow all domains.", + "type": "string" + }, + "transform": { + "description": "Headers to inject into requests matching this rule.\nKey: header name (e.g., \"Authorization\").\nValue: header value (e.g., \"Bearer your-token\").", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "credential": { + "description": "Optional. Reference to a server-managed Credential resource by ID.", + "type": "string" + } + }, + "required": [ + "domain" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Empty": { + "description": "A generic empty message that you can re-use to avoid defining duplicated\nempty messages in your APIs. A typical example is to use it as the request\nor the response type of an API method. For instance:\n\n service Foo {\n rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);\n }", + "type": "object", + "x-speakeasy-model-namespace": "interactions" + }, + "EnvVar": { + "description": "An environment variable to set in the execution environment.", + "type": "object", + "properties": { + "value": { + "description": "Direct string value for plain environment variables.", + "type": "string" + }, + "credential": { + "description": "Optional reference to a server-managed Credential resource by ID.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "Environment": { + "description": "An execution environment for an agent.", + "type": "object", + "properties": { + "id": { + "description": "Required. Output only. The ID of the environment.", + "type": "string", + "readOnly": true + }, + "network": { + "description": "Network configuration for the environment.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" + }, + { + "type": "string", + "enum": [ + "disabled" + ], + "x-google-enum-descriptions": [ + "All network egress is blocked." + ] + } + ] + }, + "sources": { + "description": "Sources to be mounted into the environment.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Source" + } + }, + "created": { + "description": "Output only. The time at which the environment was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "updated": { + "description": "Output only. The time at which the environment was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "last_accessed": { + "description": "Output only. The time at which the environment was last accessed in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "status": { + "description": "Output only. The status of the environment container.", + "type": "string", + "enum": [ + "active", + "expired" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "", + "" + ] + }, + "file_count": { + "description": "Output only. The number of files in the environment, output only.", + "type": "string", + "format": "int64", + "readOnly": true + }, + "size_bytes": { + "description": "Output only. The total size of the environment files in bytes, output only.", + "type": "string", + "format": "int64", + "readOnly": true + } + }, + "required": [ + "id" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "EnvironmentConfig": { + "description": "Configuration for a custom environment.", + "type": "object", + "properties": { + "type": { + "const": "remote" + }, + "network": { + "description": "Network configuration for the environment.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" + }, + { + "type": "string", + "enum": [ + "disabled" + ], + "x-google-enum-descriptions": [ + "All network egress is blocked." + ] + } + ] + }, + "sources": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Source" + } + }, + "environment_id": { + "description": "Optional. The environment ID for the interaction. If specified, the request will\nupdate the existing environment instead of creating a new one.", + "type": "string" + }, + "env": { + "description": "Environment variables to set in the sandbox environment.", + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/EnvVar" + } + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "inline_sources": { + "summary": "Inline Sources", + "value": { + "type": "remote", + "sources": [ + { + "type": "inline", + "content": "You are a data analyst. Always include visualizations and export results as PDF.", + "target": ".agents/AGENTS.md" + }, + { + "type": "inline", + "content": "---\nname: slide-maker\ndescription: Create HTML slide decks\n---\n# Slide Maker\n\nWhen asked to create a presentation:\n1. Analyze the input data\n2. Create an HTML slide deck with reveal.js\n3. Save to /workspace/output/slides.html", + "target": ".agents/skills/slide-maker/SKILL.md" + } + ] + } + } + }, + { + "external_sources": { + "summary": "External Sources", + "value": { + "type": "remote", + "sources": [ + { + "type": "repository", + "source": "https://github.com/my-org/my-skills.git", + "target": ".agents/skills" + }, + { + "type": "gcs", + "source": "gs://my-bucket/my-folder", + "target": "/workspace/data" + } + ] + } + } + }, + { + "network_allowlist": { + "summary": "Network Allowlist", + "value": { + "type": "remote", + "network": { + "allowlist": [ + { + "domain": "pypi.org" + }, + { + "domain": "*.github.com" + } + ] + } + } + } + }, + { + "proxy_credentials": { + "summary": "Proxy Credentials", + "value": { + "type": "remote", + "network": { + "allowlist": [ + { + "domain": "api.github.com", + "transform": { + "Authorization": "Bearer YOUR_GITHUB_TOKEN" + } + } + ] + } + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "EnvironmentFile": { + "description": "Metadata for a file or directory within an environment.", + "type": "object", + "properties": { + "name": { + "description": "Output only. The name of the file or directory (e.g., \"main.py\" or \"src\").", + "type": "string", + "readOnly": true + }, + "path": { + "description": "Output only. The full relative path within the environment\n(e.g., \"workspace/src/main.py\").", + "type": "string", + "readOnly": true + }, + "type": { + "description": "Output only. The type of the entry.", + "type": "string", + "enum": [ + "file", + "directory" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "A regular file.", + "A directory." + ] + }, + "size_bytes": { + "description": "Output only. The size of the file/directory in bytes.\nNOLINT", + "type": "string", + "format": "int64", + "readOnly": true + }, + "mime_type": { + "description": "Output only. The MIME type of the file (e.g., \"text/python\", \"image/png\").\nEmpty for directories.\nNOLINT", + "type": "string", + "readOnly": true + }, + "created": { + "description": "Output only. The creation time of the file/directory.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "modified": { + "description": "Output only. The modification time of the file/directory.", + "type": "string", + "format": "date-time", + "readOnly": true + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "EnvironmentNetworkEgressAllowlist": { + "description": "Network egress configuration for the environment.", + "type": "object", + "properties": { + "allowlist": { + "description": "List of allowed domains and their configurations. Set to `\"disabled\"`\nto block all network egress.", + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/components/schemas/EgressRule" + } + }, + { + "type": "string", + "enum": [ + "disabled" + ], + "x-google-enum-descriptions": [ + "All network egress is blocked." + ] + } + ] + } + }, + "examples": [ + { + "network_allowlist": { + "summary": "Network Allowlist", + "value": { + "allowlist": [ + { + "domain": "github.com", + "transform": [ + { + "Authorization": "Bearer your-token" + } + ] + }, + { + "domain": "*.googleapis.com" + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "EnvironmentVariableConfig": { + "description": "Configuration for environment variable credentials.", + "type": "object", + "properties": { + "value": { + "description": "Required. Input only. Secret value of the environment variable. Write-only; never\nreturned in responses.", + "type": "string", + "writeOnly": true + }, + "injection_location": { + "description": "Required. Locations where the environment variable can be injected in\noutgoing HTTP requests. Must contain at least one location.\nAccepts either a single location (e.g. \"header\") or an array of locations.", + "oneOf": [ + { + "$ref": "#/components/schemas/InjectionLocation" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/InjectionLocation" + } + } + ] + }, + "id": { + "type": "string" + }, + "type": { + "const": "environment_variable" + }, + "trusted_domains": { + "description": "Optional. List of domains allowed to receive this environment variable\nvalue in HTTP requests.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "id", + "injection_location", + "type", + "value" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "EnvironmentVariableUpdateConfig": { + "description": "Configuration for updating environment variable credentials.", + "type": "object", + "properties": { + "type": { + "const": "environment_variable" + }, + "value": { + "description": "Optional. Input only. Secret value of the environment variable. Write-only; never\nreturned in responses.", + "type": "string", + "writeOnly": true + }, + "trusted_domains": { + "description": "Optional. List of domains allowed to receive this environment variable\nvalue in HTTP requests.", + "type": "array", + "items": { + "type": "string" + } + }, + "injection_location": { + "description": "Optional. Locations where the environment variable can be injected in\noutgoing HTTP requests.\nAccepts either a single location (e.g. \"header\") or an array of locations.", + "oneOf": [ + { + "$ref": "#/components/schemas/InjectionLocation" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/InjectionLocation" + } + } + ] + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Error": { + "description": "Error message from an interaction.", + "type": "object", + "properties": { + "code": { + "description": "A URI that identifies the error type.", + "type": "string" + }, + "message": { + "description": "A human-readable error message.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ErrorEvent": { + "type": "object", + "properties": { + "event_type": { + "const": "error" + }, + "error": { + "$ref": "#/components/schemas/Error" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + } + }, + "required": [ + "event_type" + ], + "examples": [ + { + "error_event": { + "summary": "Error Event", + "value": { + "error": { + "code": "not_found", + "message": "Failed to get completed interaction: Result not found." + }, + "event_type": "error" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FileCitation": { + "description": "A file citation annotation.", + "type": "object", + "properties": { + "type": { + "const": "file_citation" + }, + "document_uri": { + "description": "The URI of the file.", + "type": "string" + }, + "file_name": { + "description": "The name of the file.", + "type": "string" + }, + "source": { + "description": "Source attributed for a portion of the text.", + "type": "string" + }, + "custom_metadata": { + "description": "User provided metadata about the retrieved context.", + "type": "object", + "additionalProperties": { + "description": "Properties of the object." + } + }, + "page_number": { + "description": "Page number of the cited document, if applicable.", + "type": "integer", + "format": "int32" + }, + "media_id": { + "description": "Media ID in-case of image citations, if applicable.", + "type": "string" + }, + "start_index": { + "description": "Start of segment of the response that is attributed to this source.\n\nIndex indicates the start of the segment, measured in bytes.", + "type": "integer", + "format": "int32" + }, + "end_index": { + "description": "End of the attributed segment, exclusive.", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FileSearch": { + "description": "A tool that can be used by the model to search files.", + "type": "object", + "properties": { + "type": { + "const": "file_search" + }, + "file_search_store_names": { + "description": "The file search store names to search.", + "type": "array", + "items": { + "type": "string" + } + }, + "top_k": { + "description": "The number of semantic retrieval chunks to retrieve.", + "type": "integer", + "format": "int32" + }, + "metadata_filter": { + "description": "Metadata filter to apply to the semantic retrieval documents and chunks.", + "type": "string" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "file_search", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"file_search\",\n \"file_search_store_names\": [\"fileSearchStores/m64d1sevsr4y-xfyawui3fxqg\"]\n }],\n \"input\": \"Who is the author of the book?\"\n }'\n" + }, + { + "label": "file_search", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\n\n# Create a file search store so we have a valid one to use.\nstore = client.file_search_stores.create()\n\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[\n {\"type\": \"file_search\", \"file_search_store_names\": [store.name]}\n ],\n input=\"What documents are available?\",\n)\nprint(response.output_text)\n" + }, + { + "label": "file_search", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\n\n// Create a file search store so we have a valid one to use.\nconst store = await ai.fileSearchStores.create({});\nif (!store.name) {\n throw new Error('Store creation failed: Name is undefined');\n}\n\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [\n {\n type: 'file_search',\n file_search_store_names: [store.name],\n },\n ],\n input: 'What documents are available?',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "file_search", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.types.CreateFileSearchStoreConfig;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.FileSearch;\nimport com.google.genai.types.FileSearchStore;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\n\n// Create a file search store so we have a valid one to use.\nFileSearchStore store =\n client.fileSearchStores.create(CreateFileSearchStoreConfig.builder().build());\nString storeName = store.name().orElseThrow();\n\nFileSearch tool = FileSearch.builder().fileSearchStoreNames(List.of(storeName)).build();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(tool))\n .input(InteractionsInput.of(\"What documents are available?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "file_search", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n tool := gaos_interactions.NewTool(gaos_interactions.FileSearch{\n FileSearchStoreNames: []string{store.Name},\n })\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"What documents are available?\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "FileSearchCallDelta": { + "type": "object", + "properties": { + "type": { + "const": "file_search_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FileSearchCallStep": { + "description": "File Search call step.", + "type": "object", + "properties": { + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "file_search_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "id", + "type" + ], + "examples": [ + { + "file_search_call": { + "summary": "FileSearchCallStep", + "value": { + "type": "file_search_call", + "id": "file_call_88192" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FileSearchResult": { + "description": "The result of the File Search.", + "type": "object", + "x-speakeasy-model-namespace": "interactions" + }, + "FileSearchResultDelta": { + "type": "object", + "properties": { + "result": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FileSearchResult" + } + }, + "type": { + "const": "file_search_result" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "result", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FileSearchResultStep": { + "description": "File Search result step.", + "type": "object", + "properties": { + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "file_search_result" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "call_id", + "type" + ], + "examples": [ + { + "file_search_result": { + "summary": "FileSearchResultStep", + "value": { + "type": "file_search_result", + "call_id": "file_call_88192" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Function": { + "description": "A tool that can be used by the model.", + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "name": { + "description": "The name of the function.", + "type": "string" + }, + "description": { + "description": "A description of the function.", + "type": "string" + }, + "parameters": { + "description": "The JSON Schema for the function's parameters.\n\nSupports a subset of JSON Schema. `minLength`, `maxLength`,\n`minProperties`, `maxProperties`, `uniqueItems`, `contains`,\n`minContains`, and `maxContains` are ignored and not enforced." + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "function_calling", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"function\",\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n }\n },\n \"required\": [\"location\"]\n }\n }],\n \"input\": \"What is the weather like in Boston, MA?\"\n }'\n" + }, + { + "label": "function_calling", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[{\n \"type\": \"function\",\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather in a given location\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": (\n \"The city and state, e.g. San Francisco, CA\"\n ),\n }\n },\n \"required\": [\"location\"],\n },\n }],\n input=\"What is the weather like in Boston?\",\n)\nprint(response.steps[-1])\n" + }, + { + "label": "function_calling", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [\n {\n type: 'function',\n name: 'get_weather',\n description: 'Get the current weather in a given location',\n parameters: {\n type: 'object',\n properties: {\n location: {\n type: 'string',\n description: 'The city and state, e.g. San Francisco, CA',\n },\n },\n required: ['location'],\n },\n },\n ],\n input: 'What is the weather like in Boston?',\n});\nconsole.log(interaction.steps.at(-1));\n" + }, + { + "label": "function_calling", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Function;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.Step;\n\nClient client = new Client();\nMap parameters =\n Map.of(\n \"type\", \"object\",\n \"properties\",\n Map.of(\n \"location\",\n Map.of(\n \"type\", \"string\",\n \"description\", \"The city and state, e.g. San Francisco, CA\")),\n \"required\", List.of(\"location\"));\nFunction functionTool =\n Function.builder()\n .name(\"get_weather\")\n .description(\"Get the current weather in a given location\")\n .parameters(parameters)\n .build();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(functionTool))\n .input(InteractionsInput.of(\"What is the weather like in Boston?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nList steps = interaction.steps().orElse(List.of());\nif (!steps.isEmpty()) {\n System.out.println(steps.get(steps.size() - 1));\n}\n" + }, + { + "label": "function_calling", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n function := gaos_interactions.Function{\n Name: genai.Ptr(\"get_weather\"),\n Description: genai.Ptr(\"Get the current weather in a given location\"),\n Parameters: map[string]any{\n \"type\": \"object\",\n \"properties\": map[string]any{\n \"location\": map[string]any{\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\",\n },\n },\n \"required\": []string{\"location\"},\n },\n }\n\n tool := gaos_interactions.NewTool(function)\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"What is the weather like in Boston, MA?\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n for _, step := range res.Interaction.Steps {\n if step.FunctionCallStep != nil {\n fmt.Println(\"Function Call:\", step.FunctionCallStep.Name)\n fmt.Println(\"Arguments:\", step.FunctionCallStep.Arguments)\n }\n }\n }\n}\n" + } + ] + }, + "FunctionCallStep": { + "description": "A function tool call step.", + "type": "object", + "properties": { + "name": { + "description": "Required. The name of the tool to call.", + "type": "string" + }, + "arguments": { + "description": "Required. The arguments to pass to the function.", + "type": "object", + "additionalProperties": { + "description": "Properties of the object." + } + }, + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "function_call" + } + }, + "required": [ + "arguments", + "id", + "name", + "type" + ], + "examples": [ + { + "function_call": { + "summary": "FunctionCallStep", + "value": { + "name": "get_weather", + "type": "function_call", + "arguments": { + "location": "Boston, MA" + }, + "id": "call_98231" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FunctionResultDelta": { + "type": "object", + "properties": { + "result": { + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/TextContent" + } + ] + } + }, + { + "type": "object" + }, + { + "type": "string" + } + ] + }, + "type": { + "const": "function_result" + }, + "name": { + "type": "string" + }, + "is_error": { + "type": "boolean" + } + }, + "required": [ + "result", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "FunctionResultStep": { + "description": "Result of a function tool call.", + "type": "object", + "properties": { + "result": { + "description": "Required. The result of the tool call.", + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/TextContent" + } + ] + } + }, + { + "type": "object" + }, + { + "type": "string" + } + ] + }, + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "function_result" + }, + "name": { + "description": "The name of the tool that was called.", + "type": "string" + }, + "is_error": { + "description": "Whether the tool call resulted in an error.", + "type": "boolean" + } + }, + "required": [ + "call_id", + "result", + "type" + ], + "examples": [ + { + "function_result": { + "summary": "FunctionResultStep", + "value": { + "name": "get_weather", + "type": "function_result", + "call_id": "call_98231", + "result": [ + { + "type": "text", + "text": "{\"weather\":\"sunny\"}" + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GenerationConfig": { + "description": "Configuration parameters for model interactions.", + "type": "object", + "properties": { + "tool_choice": { + "description": "The tool choice configuration.", + "oneOf": [ + { + "$ref": "#/components/schemas/ToolChoiceConfig" + }, + { + "type": "string", + "enum": [ + "auto", + "any", + "none", + "validated" + ], + "x-google-enum-descriptions": [ + "Auto tool choice.", + "Any tool choice.", + "No tool choice.", + "Validated tool choice." + ], + "x-speakeasy-model-namespace": "interactions" + } + ] + }, + "temperature": { + "description": "Controls the randomness of the output.", + "type": "number", + "deprecated": true, + "format": "float" + }, + "top_p": { + "description": "The maximum cumulative probability of tokens to consider when sampling.", + "type": "number", + "deprecated": true, + "format": "float" + }, + "seed": { + "description": "Seed used in decoding for reproducibility.", + "type": "integer", + "format": "int32" + }, + "stop_sequences": { + "description": "A list of character sequences that will stop output interaction.", + "type": "array", + "items": { + "type": "string" + } + }, + "thinking_level": { + "description": "The level of thought tokens that the model should generate.", + "$ref": "#/components/schemas/ThinkingLevel" + }, + "thinking_summaries": { + "description": "Whether to include thought summaries in the response.", + "$ref": "#/components/schemas/ThinkingSummaries" + }, + "max_output_tokens": { + "description": "The maximum number of tokens to include in the response.", + "type": "integer", + "format": "int32" + }, + "image_config": { + "description": "Configuration for image interaction.", + "$ref": "#/components/schemas/ImageConfig", + "deprecated": true + }, + "video_config": { + "description": "Configuration for video generation.", + "$ref": "#/components/schemas/VideoConfig" + }, + "transcription_config": { + "description": "Optional. Configuration for speech recognition (transcription). If present, ASR is\nenabled.", + "$ref": "#/components/schemas/TranscriptionConfig" + }, + "structured_speech_config": { + "description": "Speech and multi-speaker configuration.", + "oneOf": [ + { + "$ref": "#/components/schemas/SpeakerConfig" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/SpeechConfig" + } + } + ] + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GetEnvironmentFilesResponse": { + "description": "Response for `GetEnvironmentFiles`.", + "type": "object", + "properties": { + "files": { + "description": "If the requested path is a directory, this contains its contents.\nIf the requested path is a file, this contains a single entry with the\nfile's metadata.\nIf alt=media was specified, this is empty (content is served via `blob`).", + "type": "array", + "items": { + "$ref": "#/components/schemas/EnvironmentFile" + } + }, + "next_page_token": { + "description": "Pagination token for directory listing.\nNOLINT", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMaps": { + "description": "A tool that can be used by the model to call Google Maps.", + "type": "object", + "properties": { + "type": { + "const": "google_maps" + }, + "enable_widget": { + "description": "Whether to return a widget context token in the tool call result of the\nresponse.", + "type": "boolean" + }, + "latitude": { + "description": "The latitude of the user's location.", + "type": "number", + "format": "double" + }, + "longitude": { + "description": "The longitude of the user's location.", + "type": "number", + "format": "double" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "google_maps", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"google_maps\",\n \"latitude\": 37.7749,\n \"longitude\": -122.4194\n }],\n \"input\": \"What is the best food near me?\"\n }'\n" + }, + { + "label": "google_maps", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[\n {\"type\": \"google_maps\", \"latitude\": 37.7749, \"longitude\": -122.4194}\n ],\n input=\"What is the best food near me?\",\n)\nprint(response.output_text)\n" + }, + { + "label": "google_maps", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [\n {\n type: 'google_maps',\n latitude: 37.7749,\n longitude: -122.4194,\n },\n ],\n input: 'What is the best food near me?',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "google_maps", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.GoogleMaps;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nGoogleMaps tool = GoogleMaps.builder().latitude(37.7749).longitude(-122.4194).build();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(tool))\n .input(InteractionsInput.of(\"What is the best food near me?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "google_maps", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n tool := gaos_interactions.NewTool(gaos_interactions.GoogleMaps{\n Latitude: genai.Ptr(37.7749),\n Longitude: genai.Ptr(-122.4194),\n })\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"What is the best food near me?\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "GoogleMapsCallArguments": { + "description": "The arguments to pass to the Google Maps tool.", + "type": "object", + "properties": { + "queries": { + "description": "The queries to be executed.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMapsCallDelta": { + "type": "object", + "properties": { + "type": { + "const": "google_maps_call" + }, + "arguments": { + "description": "The arguments to pass to the Google Maps tool.", + "$ref": "#/components/schemas/GoogleMapsCallArguments" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMapsCallStep": { + "description": "Google Maps call step.", + "type": "object", + "properties": { + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "google_maps_call" + }, + "arguments": { + "description": "The arguments to pass to the Google Maps tool.", + "$ref": "#/components/schemas/GoogleMapsCallStepArguments" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "id", + "type" + ], + "examples": [ + { + "google_maps_call": { + "summary": "GoogleMapsCallStep", + "value": { + "type": "google_maps_call", + "arguments": { + "latitude": 37.7749, + "longitude": -122.4194 + }, + "id": "maps_call_39201" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMapsCallStepArguments": { + "description": "The arguments to pass to the Google Maps tool.", + "type": "object", + "properties": { + "queries": { + "description": "The queries to be executed.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "GoogleMapsCallArguments" + }, + "GoogleMapsResult": { + "description": "The result of the Google Maps.", + "type": "object", + "properties": { + "places": { + "description": "The places that were found.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Places" + } + }, + "widget_context_token": { + "description": "Resource name of the Google Maps widget context token.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMapsResultDelta": { + "type": "object", + "properties": { + "type": { + "const": "google_maps_result" + }, + "result": { + "description": "The results of the Google Maps.", + "type": "array", + "items": { + "$ref": "#/components/schemas/GoogleMapsResult" + } + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMapsResultItem": { + "description": "The result of the Google Maps.", + "type": "object", + "properties": { + "places": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GoogleMapsResultPlaces" + } + }, + "widget_context_token": { + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "GoogleMapsResult" + }, + "GoogleMapsResultPlaces": { + "type": "object", + "properties": { + "place_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "review_snippets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ReviewSnippet" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleMapsResultStep": { + "description": "Google Maps result step.", + "type": "object", + "properties": { + "result": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GoogleMapsResultItem" + } + }, + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "google_maps_result" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "call_id", + "result", + "type" + ], + "examples": [ + { + "google_maps_result": { + "summary": "GoogleMapsResultStep", + "value": { + "type": "google_maps_result", + "call_id": "maps_call_39201", + "result": [ + { + "name": "Golden Gate Park", + "place_id": "ChIJIQBpAG2ahYAR9R7bNdTLg8M", + "rating": 4.8 + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleSearch": { + "description": "A tool that can be used by the model to search Google.", + "type": "object", + "properties": { + "type": { + "const": "google_search" + }, + "search_types": { + "description": "The types of search grounding to enable.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "web_search", + "image_search" + ], + "x-google-enum-descriptions": [ + "Setting this field enables web search. Only text results are returned.", + "Setting this field enables image search. Image bytes are returned." + ] + } + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "google_search", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"google_search\"\n }],\n \"input\": \"Who is the current president of France?\"\n }'\n" + }, + { + "label": "google_search", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[{\"type\": \"google_search\"}],\n input=\"Who is the current president of France?\",\n)\nprint(response.output_text)\n" + }, + { + "label": "google_search", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [{type: 'google_search'}],\n input: 'Who is the current president of France?',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "google_search", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.GoogleSearch;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(new GoogleSearch()))\n .input(InteractionsInput.of(\"Who is the current president of France?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "google_search", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n tool := gaos_interactions.NewTool(gaos_interactions.GoogleSearch{})\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Who is the current president of France?\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "GoogleSearchCallArguments": { + "description": "The arguments to pass to Google Search.", + "type": "object", + "properties": { + "queries": { + "description": "Web search queries for the following-up web search.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleSearchCallDelta": { + "type": "object", + "properties": { + "arguments": { + "$ref": "#/components/schemas/GoogleSearchCallArguments" + }, + "type": { + "const": "google_search_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "arguments", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleSearchCallStep": { + "description": "Google Search call step.", + "type": "object", + "properties": { + "arguments": { + "description": "Required. The arguments to pass to Google Search.", + "$ref": "#/components/schemas/GoogleSearchCallStepArguments" + }, + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "google_search_call" + }, + "search_type": { + "description": "The type of search grounding enabled.", + "type": "string", + "enum": [ + "web_search", + "image_search" + ], + "x-google-enum-descriptions": [ + "Setting this field enables web search. Only text results are returned.", + "Setting this field enables image search. Image bytes are returned." + ] + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "arguments", + "id", + "type" + ], + "examples": [ + { + "google_search_call": { + "summary": "GoogleSearchCallStep", + "value": { + "type": "google_search_call", + "arguments": { + "query": "Who won the men's 100m in Paris 2024?" + }, + "id": "search_call_19201" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleSearchCallStepArguments": { + "description": "The arguments to pass to Google Search.", + "type": "object", + "properties": { + "queries": { + "description": "Web search queries for the following-up web search.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "GoogleSearchCallArguments" + }, + "GoogleSearchResult": { + "description": "The result of the Google Search.", + "type": "object", + "properties": { + "search_suggestions": { + "description": "Web content snippet that can be embedded in a web page or an app webview.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleSearchResultDelta": { + "type": "object", + "properties": { + "result": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GoogleSearchResult" + } + }, + "type": { + "const": "google_search_result" + }, + "is_error": { + "type": "boolean" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "result", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GoogleSearchResultItem": { + "description": "The result of the Google Search.", + "type": "object", + "properties": { + "search_suggestions": { + "description": "Web content snippet that can be embedded in a web page or an app webview.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "GoogleSearchResult" + }, + "GoogleSearchResultStep": { + "description": "Google Search result step.", + "type": "object", + "properties": { + "result": { + "description": "Required. The results of the Google Search.", + "type": "array", + "items": { + "$ref": "#/components/schemas/GoogleSearchResultItem" + } + }, + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "google_search_result" + }, + "is_error": { + "description": "Whether the Google Search resulted in an error.", + "type": "boolean" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "call_id", + "result", + "type" + ], + "examples": [ + { + "google_search_result": { + "summary": "GoogleSearchResultStep", + "value": { + "type": "google_search_result", + "call_id": "search_call_19201", + "result": [ + { + "title": "Paris 2024 Olympics: Noah Lyles wins men's 100m gold", + "url": "https://olympics.com/en/news/paris-2024-noah-lyles-wins-mens-100m-gold", + "snippet": "American Noah Lyles won the Olympic men's 100m gold medal in a photo finish." + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "GroundingToolCount": { + "description": "The number of grounding tool counts.", + "type": "object", + "properties": { + "type": { + "description": "The grounding tool type associated with the count.", + "type": "string", + "enum": [ + "google_search", + "google_maps" + ], + "x-google-enum-descriptions": [ + "Grounding with Google Web Search and Image Search, & Web Grounding\nfor Enterprise.", + "Grounding with Google Maps." + ] + }, + "count": { + "description": "The number of grounding tool counts.", + "type": "integer", + "format": "int32" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "HarmCategory": { + "type": "string", + "enum": [ + "hate_speech", + "dangerous_content", + "harassment", + "sexually_explicit", + "jailbreak" + ], + "x-google-enum-descriptions": [ + "Content that promotes violence or incites hatred against individuals or\ngroups based on certain attributes.", + "Content that promotes, facilitates, or enables dangerous activities.", + "Abusive, threatening, or content intended to bully, torment, or ridicule.", + "Content that contains sexually explicit material.", + "Prompts designed to bypass safety filters." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "HttpBearerConfig": { + "description": "Configuration for HTTP Bearer token credentials.", + "type": "object", + "properties": { + "token": { + "description": "Required. Input only. The static bearer token. Write-only; never returned in responses.", + "type": "string", + "writeOnly": true + }, + "id": { + "type": "string" + }, + "type": { + "const": "bearer_token" + }, + "header_name": { + "description": "Optional. Header name to inject the token into. Defaults to\n'Authorization'.", + "type": "string" + }, + "prefix": { + "description": "Optional. Prefix to prepend to the token. Defaults to 'Bearer'. Set to ''\nfor no prefix.", + "type": "string" + } + }, + "required": [ + "id", + "token", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "HttpBearerUpdateConfig": { + "description": "Configuration for updating HTTP Bearer token credentials.", + "type": "object", + "properties": { + "type": { + "const": "bearer_token" + }, + "token": { + "description": "Optional. Input only. The static bearer token. Write-only; never returned in responses.", + "type": "string", + "writeOnly": true + }, + "header_name": { + "description": "Optional. Header name to inject the token into. Defaults to\n'Authorization'.", + "type": "string" + }, + "prefix": { + "description": "Optional. Prefix to prepend to the token. Defaults to 'Bearer'. Set to ''\nfor no prefix.", + "type": "string" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "HttpBody": { + "description": "Message that represents an arbitrary HTTP body. It should only be used for\npayload formats that can't be represented as JSON, such as raw binary or\nan HTML page.\n\n\nThis message can be used both in streaming and non-streaming API methods in\nthe request as well as the response.\n\nIt can be used as a top-level request field, which is convenient if one\nwants to extract parameters from either the URL or HTTP template into the\nrequest fields and also want access to the raw HTTP body.\n\nExample:\n\n message GetResourceRequest {\n // A unique request id.\n string request_id = 1;\n\n // The raw HTTP body is bound to this field.\n google.api.HttpBody http_body = 2;\n\n }\n\n service ResourceService {\n rpc GetResource(GetResourceRequest)\n returns (google.api.HttpBody);\n rpc UpdateResource(google.api.HttpBody)\n returns (google.protobuf.Empty);\n\n }\n\nExample with streaming methods:\n\n service CaldavService {\n rpc GetCalendar(stream google.api.HttpBody)\n returns (stream google.api.HttpBody);\n rpc UpdateCalendar(stream google.api.HttpBody)\n returns (stream google.api.HttpBody);\n\n }\n\nUse of this type only changes how the request and response bodies are\nhandled, all other features will continue to work unchanged.", + "type": "object", + "properties": { + "content_type": { + "description": "The HTTP Content-Type header value specifying the content type of the body.", + "type": "string" + }, + "data": { + "description": "The HTTP request/response body as raw binary.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "extensions": { + "description": "Application specific response metadata. Must be set in the first response\nfor streaming APIs.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "description": "Properties of the object. Contains field @type with type URL." + } + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ImageConfig": { + "description": "The configuration for image interaction.", + "type": "object", + "properties": { + "aspect_ratio": { + "description": "The aspect ratio of the image to generate. Supported aspect ratios: 1:1,\n2:3, 3:2, 3:4, 4:3, 9:16, 16:9, 21:9.\n\nIf not specified, the model will choose a default aspect ratio based on any\nreference images provided.", + "type": "string" + }, + "image_size": { + "description": "Specifies the size of generated images. Supported values are `1K`, `2K`,\n`4K`. If not specified, the model will use default value `1K`.", + "type": "string" + } + }, + "deprecated": true, + "x-speakeasy-model-namespace": "interactions" + }, + "ImageContent": { + "description": "An image content block.", + "type": "object", + "properties": { + "type": { + "const": "image" + }, + "data": { + "description": "The image content.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "description": "The URI of the image.", + "type": "string" + }, + "resolution": { + "description": "The resolution of the media.", + "$ref": "#/components/schemas/MediaResolution" + }, + "mime_type": { + "description": "The mime type of the image.", + "type": "string", + "enum": [ + "image/png", + "image/jpeg", + "image/webp", + "image/heic", + "image/heif", + "image/gif", + "image/bmp", + "image/tiff" + ], + "x-google-enum-descriptions": [ + "PNG image format", + "JPEG image format", + "WebP image format", + "HEIC image format", + "HEIF image format", + "GIF image format", + "BMP image format", + "TIFF image format" + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "image": { + "summary": "Image", + "value": { + "type": "image", + "data": "BASE64_ENCODED_IMAGE", + "mime_type": "image/png" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ImageDelta": { + "type": "object", + "properties": { + "type": { + "const": "image" + }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, + "mime_type": { + "type": "string", + "enum": [ + "image/png", + "image/jpeg", + "image/webp", + "image/heic", + "image/heif", + "image/gif", + "image/bmp", + "image/tiff" + ], + "x-google-enum-descriptions": [ + "PNG image format", + "JPEG image format", + "WebP image format", + "HEIC image format", + "HEIF image format", + "GIF image format", + "BMP image format", + "TIFF image format" + ] + }, + "resolution": { + "description": "The resolution of the media.", + "$ref": "#/components/schemas/MediaResolution" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ImageResponseFormat": { + "description": "Configuration for image output format.", + "type": "object", + "properties": { + "type": { + "const": "image" + }, + "mime_type": { + "description": "The MIME type of the image output.", + "type": "string", + "enum": [ + "image/jpeg" + ], + "x-google-enum-descriptions": [ + "JPEG image format." + ] + }, + "delivery": { + "description": "The delivery mode for the image output.", + "type": "string", + "enum": [ + "inline", + "uri" + ], + "x-google-enum-descriptions": [ + "Image data is returned inline in the response.", + "Image data is returned as a URI." + ] + }, + "aspect_ratio": { + "description": "The aspect ratio for the image output.", + "type": "string", + "enum": [ + "1:1", + "2:3", + "3:2", + "3:4", + "4:3", + "4:5", + "5:4", + "9:16", + "16:9", + "21:9", + "1:8", + "8:1", + "1:4", + "4:1" + ], + "x-google-enum-descriptions": [ + "1:1 aspect ratio.", + "2:3 aspect ratio.", + "3:2 aspect ratio.", + "3:4 aspect ratio.", + "4:3 aspect ratio.", + "4:5 aspect ratio.", + "5:4 aspect ratio.", + "9:16 aspect ratio.", + "16:9 aspect ratio.", + "21:9 aspect ratio.", + "1:8 aspect ratio.", + "8:1 aspect ratio.", + "1:4 aspect ratio.", + "4:1 aspect ratio." + ] + }, + "image_size": { + "description": "The size of the image output.", + "type": "string", + "enum": [ + "512", + "1K", + "2K", + "4K" + ], + "x-google-enum-descriptions": [ + "512px image size.", + "1K image size.", + "2K image size.", + "4K image size." + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "image_response_format": { + "summary": "Image Output", + "value": { + "type": "image", + "aspect_ratio": "16:9", + "image_size": "1K", + "mime_type": "image/jpeg" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "InjectionLocation": { + "type": "string", + "enum": [ + "header", + "query", + "body" + ], + "x-google-enum-descriptions": [ + "Injected into HTTP request headers.", + "Injected into HTTP URL query parameters.", + "Injected into HTTP request body." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Interaction": { + "description": "The Interaction resource.", + "properties": { + "id": { + "description": "Required. Output only. A unique identifier for the interaction completion.", + "type": "string", + "default": "", + "readOnly": true + }, + "status": { + "description": "Required. Output only. The status of the interaction.", + "type": "string", + "enum": [ + "in_progress", + "requires_action", + "completed", + "failed", + "cancelled", + "incomplete", + "budget_exceeded" + ], + "readOnly": true, + "x-google-enum-deprecated": [ + false, + false, + false, + false, + false, + false, + true + ], + "x-google-enum-descriptions": [ + "The interaction is in progress.", + "The interaction requires action/input from the user.", + "The interaction is completed.", + "The interaction failed.", + "The interaction was cancelled.", + "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + ] + }, + "created": { + "description": "Required. Output only. The time at which the response was created in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "updated": { + "description": "Required. Output only. The time at which the response was last updated in ISO 8601 format\n(YYYY-MM-DDThh:mm:ssZ).", + "type": "string", + "readOnly": true + }, + "steps": { + "description": "Required. Output only. The steps that make up the interaction, when included in the response.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Step" + }, + "readOnly": true + }, + "model": { + "description": "The name of the `Model` used for generating the interaction.", + "$ref": "#/components/schemas/ModelOption" + }, + "generation_config": { + "description": "Input only. Configuration parameters for the model interaction.", + "$ref": "#/components/schemas/GenerationConfig", + "writeOnly": true + }, + "input": { + "$ref": "#/components/schemas/InteractionsInput" + }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, + "system_instruction": { + "description": "System instruction for the interaction.", + "type": "string" + }, + "tools": { + "description": "A list of tool declarations the model may call during interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Tool" + } + }, + "usage": { + "description": "Output only. Statistics on the interaction request's token usage.", + "$ref": "#/components/schemas/Usage", + "readOnly": true + }, + "response_modalities": { + "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", + "type": "array", + "deprecated": true, + "items": { + "$ref": "#/components/schemas/ResponseModality" + } + }, + "response_mime_type": { + "description": "The mime type of the response. This is required if response_format is set.", + "type": "string", + "deprecated": true + }, + "previous_interaction_id": { + "description": "The ID of the previous interaction, if any.", + "type": "string" + }, + "environment_id": { + "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", + "type": "string", + "readOnly": true + }, + "service_tier": { + "description": "The service tier for the interaction.", + "$ref": "#/components/schemas/ServiceTier" + }, + "webhook_config": { + "description": "Optional. Webhook configuration for receiving notifications when the\ninteraction completes.", + "$ref": "#/components/schemas/WebhookConfig" + }, + "safety_settings": { + "description": "Safety settings for the interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SafetySetting" + } + }, + "labels": { + "description": "The labels with user-defined metadata for the request.\n\nLabel keys and values can be no longer than 63 characters\n(Unicode codepoints) and can only contain lowercase letters, numeric\ncharacters, underscores, and dashes. International characters are allowed.\nLabel values are optional. Label keys must start with a letter.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "errors": { + "description": "Output only. Diagnostic faults / platform errors recorded on the interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Error" + }, + "readOnly": true + }, + "stream": { + "description": "Input only. Whether the interaction will be streamed.", + "type": "boolean", + "writeOnly": true + }, + "store": { + "description": "Input only. Whether to store the response and request for later retrieval.", + "type": "boolean", + "writeOnly": true + }, + "background": { + "description": "Input only. Whether to run the model interaction in the background.", + "type": "boolean", + "writeOnly": true + }, + "agent": { + "description": "The name of the `Agent` used for generating the interaction.", + "$ref": "#/components/schemas/AgentOption" + }, + "agent_config": { + "description": "Configuration parameters for the agent interaction.", + "oneOf": [ + { + "$ref": "#/components/schemas/AntigravityAgentConfig" + }, + { + "$ref": "#/components/schemas/DeepResearchAgentConfig" + }, + { + "$ref": "#/components/schemas/DynamicAgentConfig" + } + ] + }, + "max_total_tokens": { + "description": "Max total tokens for the agent run.", + "type": "string", + "format": "int64" + } + }, + "required": [ + "status", + "created", + "id", + "steps", + "updated" + ], + "example": { + "created": "2025-12-04T15:01:45Z", + "id": "v1_ChdXS0l4YWZXTk9xbk0xZThQczhEcmlROBIXV0tJeGFmV05PcW5NMWU4UHM4RHJpUTg", + "model": "gemini-3.6-flash", + "object": "interaction", + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "Hello! I'm doing well, functioning as expected. Thank you for asking! How are you doing today?" + } + ] + } + ], + "updated": "2025-12-04T15:01:45Z", + "usage": { + "input_tokens_by_modality": [ + { + "modality": "text", + "tokens": 7 + } + ], + "total_cached_tokens": 0, + "total_input_tokens": 7, + "total_output_tokens": 23, + "total_thought_tokens": 49, + "total_tokens": 79, + "total_tool_use_tokens": 0 + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "InteractionCompletedEvent": { + "description": "Signals that the Interaction completed. Sent when the Interaction receives\nComplete/Cancel or naturally terminates. No more input can be sent to the\nInteraction after this.", + "type": "object", + "properties": { + "interaction": { + "description": "Required. Partial completed interaction resource emitted at the end of the stream.", + "$ref": "#/components/schemas/InteractionSseEventInteraction" + }, + "event_type": { + "const": "interaction.completed" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + } + }, + "required": [ + "event_type", + "interaction" + ], + "examples": [ + { + "interaction_completed": { + "summary": "Interaction Completed", + "value": { + "event_id": "evt_123", + "event_type": "interaction.completed", + "interaction": { + "created": "2025-12-04T15:01:45Z", + "id": "v1_ChdXS0l4YWZXTk9xbk0xZThQczhEcmlROBIXV0tJeGFmV05PcW5NMWU4UHM4RHJpUTg", + "model": "gemini-3.6-flash", + "status": "completed", + "updated": "2025-12-04T15:01:45Z" + } + } + } + }, + { + "interaction_completed": { + "summary": "Interaction Completed", + "value": { + "event_id": "evt_123", + "event_type": "interaction.completed", + "interaction": { + "created": "2025-12-04T15:01:45Z", + "id": "v1_ChdXS0l4YWZXTk9xbk0xZThQczhEcmlROBIXV0tJeGFmV05PcW5NMWU4UHM4RHJpUTg", + "model": "gemini-3-flash-preview", + "object": "interaction", + "status": "completed", + "updated": "2025-12-04T15:01:45Z" + } + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "InteractionCreatedEvent": { + "description": "Server response confirming that a new interaction was created.", + "type": "object", + "properties": { + "interaction": { + "description": "Required. Partial interaction resource emitted when the stream is created.", + "$ref": "#/components/schemas/InteractionSseEventInteraction" + }, + "event_type": { + "const": "interaction.created" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + } + }, + "required": [ + "event_type", + "interaction" + ], + "examples": [ + { + "interaction_created": { + "summary": "Interaction Created", + "value": { + "event_id": "evt_123", + "event_type": "interaction.created", + "interaction": { + "created": "2025-12-04T15:01:45Z", + "id": "v1_ChdXS0l4YWZXTk9xbk0xZThQczhEcmlROBIXV0tJeGFmV05PcW5NMWU4UHM4RHJpUTg", + "model": "gemini-3.6-flash", + "status": "in_progress", + "updated": "2025-12-04T15:01:45Z" + } + } + } + }, + { + "interaction_created": { + "summary": "Interaction Created", + "value": { + "event_id": "evt_123", + "event_type": "interaction.created", + "interaction": { + "id": "v1_ChdXS0l4YWZXTk9xbk0xZThQczhEcmlROBIXV0tJeGFmV05PcW5NMWU4UHM4RHJpUTg", + "model": "gemini-3-flash-preview", + "object": "interaction", + "status": "in_progress" + } + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "InteractionSseEvent": { + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/ErrorEvent" + }, + { + "$ref": "#/components/schemas/InteractionCompletedEvent" + }, + { + "$ref": "#/components/schemas/InteractionCreatedEvent" + }, + { + "$ref": "#/components/schemas/InteractionStatusUpdate" + }, + { + "$ref": "#/components/schemas/StepDelta" + }, + { + "$ref": "#/components/schemas/StepStart" + }, + { + "$ref": "#/components/schemas/StepStop" + } + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "InteractionSSEEvent" + }, + "InteractionSseEventInteraction": { + "description": "Partial interaction resource emitted by interaction lifecycle SSE events.\nStreaming lifecycle payloads may omit fields that are only available on\nfull non-streaming Interaction responses.\n", + "type": "object", + "properties": { + "id": { + "description": "Required. Output only. A unique identifier for the interaction completion.", + "type": "string", + "readOnly": true + }, + "status": { + "description": "Required. Output only. The status of the interaction.", + "type": "string", + "enum": [ + "in_progress", + "requires_action", + "completed", + "failed", + "cancelled", + "incomplete" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "The interaction is in progress.", + "The interaction requires action/input from the user.", + "The interaction is completed.", + "The interaction failed.", + "The interaction was cancelled.", + "The interaction is completed, but contains incomplete results (e.g. hitting max_tokens)." + ] + }, + "model": { + "description": "The model that will complete your prompt.", + "type": "string" + }, + "created": { + "description": "Output only. The time at which the response was created in ISO 8601 format.", + "type": "string", + "readOnly": true + }, + "updated": { + "description": "Output only. The time at which the response was last updated in ISO 8601 format.", + "type": "string", + "readOnly": true + }, + "usage": { + "description": "Output only. Statistics on the interaction request's token usage.", + "$ref": "#/components/schemas/Usage", + "readOnly": true + }, + "service_tier": { + "description": "The service tier for the interaction.", + "$ref": "#/components/schemas/ServiceTier" + }, + "steps": { + "description": "Output only. The steps that make up the interaction, if included in this event.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Step" + }, + "readOnly": true + }, + "agent": { + "description": "The agent to interact with.", + "type": "string" + }, + "continuation_token": { + "description": "Opaque token to resume a long decode.\nOutput: set when status is INCOMPLETE and decoding can be resumed.\nInput: pass the latest token back unchanged in CreateInteraction to\ncontinue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "object": { + "description": "Output only. The resource type.", + "type": "string", + "readOnly": true + } + }, + "required": [ + "id", + "status" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "InteractionSseStreamEvent": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/InteractionSseEvent" + } + }, + "required": [ + "data" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "InteractionSSEStreamEvent" + }, + "InteractionStatusUpdate": { + "type": "object", + "properties": { + "interaction_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "in_progress", + "requires_action", + "completed", + "failed", + "cancelled", + "incomplete", + "budget_exceeded" + ], + "x-google-enum-deprecated": [ + false, + false, + false, + false, + false, + false, + true + ], + "x-google-enum-descriptions": [ + "The interaction is in progress.", + "The interaction requires action/input from the user.", + "The interaction is completed.", + "The interaction failed.", + "The interaction was cancelled.", + "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + ] + }, + "event_type": { + "const": "interaction.status_update" + }, + "continuation_token": { + "description": "An optional opaque continuation token used to resume decoding from the\nlatest checkpoint after a disconnected stream.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "usage": { + "description": "Model usage statistics up to this checkpoint.", + "$ref": "#/components/schemas/Usage" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + } + }, + "required": [ + "event_type", + "interaction_id", + "status" + ], + "examples": [ + { + "interaction_status_update": { + "summary": "Interaction Status Update", + "value": { + "event_type": "interaction.status_update", + "interaction_id": "v1_ChdTMjQ0YWJ5TUF1TzcxZThQdjRpcnFRcxIXUzI0NGFieU1BdU83MWU4UHY0aXJxUXM", + "status": "in_progress" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "InteractionsInput": { + "description": "The input for the interaction.", + "oneOf": [ + { + "$ref": "#/components/schemas/Content" + }, + { + "title": "StepList", + "type": "array", + "items": { + "$ref": "#/components/schemas/Step" + } + }, + { + "title": "ContentList", + "type": "array", + "items": { + "$ref": "#/components/schemas/Content" + } + }, + { + "type": "string" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ListAgentsResponse": { + "type": "object", + "properties": { + "agents": { + "description": "The list of agents.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Agent" + } + }, + "next_page_token": { + "description": "A token to retrieve the next page of results.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ListCredentialsResponse": { + "type": "object", + "properties": { + "credentials": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Credential" + } + }, + "next_page_token": { + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ListEnvironmentsResponse": { + "description": "Response for `ListEnvironments`.", + "type": "object", + "properties": { + "environments": { + "description": "Environments belonging to the provided project.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Environment" + } + }, + "next_page_token": { + "description": "Pagination token.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ListTriggerExecutionsResponse": { + "description": "Response message for TriggerService.ListTriggerExecutions.", + "type": "object", + "properties": { + "trigger_executions": { + "description": "The list of trigger executions.", + "type": "array", + "items": { + "$ref": "#/components/schemas/TriggerExecution" + } + }, + "next_page_token": { + "description": "A page token, received from a previous `ListTriggerExecutions` call.\nProvide this to retrieve the subsequent page.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ListTriggersResponse": { + "description": "Response message for TriggerService.ListTriggers.", + "type": "object", + "properties": { + "triggers": { + "description": "The list of triggers.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Trigger" + } + }, + "next_page_token": { + "description": "A page token, received from a previous `ListTriggers` call.\nProvide this to retrieve the subsequent page.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ListVoicesResponse": { + "description": "Response message for `VoicesService.ListVoices`.", + "type": "object", + "properties": { + "voices": { + "description": "The voices from the specified collection.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Voice" + } + }, + "next_page_token": { + "description": "A token that can be sent as `page_token` to retrieve the next page.\nIf empty, there are no subsequent pages.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ListWebhooksResponse": { + "description": "Response message for WebhookService.ListWebhooks.", + "type": "object", + "properties": { + "webhooks": { + "description": "The webhooks.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Webhook" + } + }, + "next_page_token": { + "description": "A token, which can be sent as `page_token` to retrieve the next page.\nIf this field is omitted, there are no subsequent pages.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "McpServer": { + "description": "A MCPServer is a server that can be called by the model to perform actions.", + "type": "object", + "properties": { + "type": { + "const": "mcp_server" + }, + "url": { + "description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"", + "type": "string" + }, + "name": { + "description": "The name of the MCPServer.", + "type": "string" + }, + "headers": { + "description": "Optional: Fields for authentication headers, timeouts, etc., if needed.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "allowed_tools": { + "description": "The allowed tools.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AllowedTools" + } + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-codeSamples": [ + { + "label": "mcp_server", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"mcp_server\",\n \"name\": \"weather_service\",\n \"url\": \"https://gemini-api-demos.uc.r.appspot.com/mcp\"\n }],\n \"input\": \"Today is 12-05-2025, what is the temperature today in London?\"\n }'\n" + }, + { + "label": "mcp_server", + "lang": "python", + "source": "import os\n\nfrom google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[{\n \"type\": \"mcp_server\",\n \"name\": \"weather_service\",\n \"url\": \"https://gemini-api-demos.uc.r.appspot.com/mcp\",\n }],\n input=\"Today is 12-05-2025, what is the temperature today in London?\",\n)\nprint(response.output_text)\n" + }, + { + "label": "mcp_server", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [\n {\n type: 'mcp_server',\n name: 'weather_service',\n url: 'https://gemini-api-demos.uc.r.appspot.com/mcp',\n },\n ],\n input: 'Today is 12-05-2025, what is the temperature today in London?',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "mcp_server", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.MCPServer;\n\nClient client = new Client();\nMCPServer mcpTool =\n MCPServer.builder()\n .name(\"weather_service\")\n .url(\"https://gemini-api-demos.uc.r.appspot.com/mcp\")\n .build();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(mcpTool))\n .input(\n InteractionsInput.of(\n \"Today is 12-05-2025, what is the temperature today in London?\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "mcp_server", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"os\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n mcpServer := gaos_interactions.MCPServer{\n Name: genai.Ptr(\"weather_service\"),\n URL: genai.Ptr(\"https://gemini-api-demos.uc.r.appspot.com/mcp\"),\n }\n tool := gaos_interactions.NewTool(mcpServer)\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Today is 12-05-2025, what is the temperature today in London?\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "McpServerToolCallDelta": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "server_name": { + "type": "string" + }, + "arguments": { + "type": "object", + "additionalProperties": { + "description": "Properties of the object." + } + }, + "type": { + "const": "mcp_server_tool_call" + } + }, + "required": [ + "arguments", + "name", + "server_name", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "McpServerToolCallStep": { + "description": "MCPServer tool call step.", + "type": "object", + "properties": { + "name": { + "description": "Required. The name of the tool which was called.", + "type": "string" + }, + "server_name": { + "description": "Required. The name of the used MCP server.", + "type": "string" + }, + "arguments": { + "description": "Required. The JSON object of arguments for the function.", + "type": "object", + "additionalProperties": { + "description": "Properties of the object." + } + }, + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "mcp_server_tool_call" + } + }, + "required": [ + "arguments", + "id", + "name", + "server_name", + "type" + ], + "examples": [ + { + "mcp_server_tool_call": { + "summary": "McpServerToolCallStep", + "value": { + "name": "calculate_tax", + "type": "mcp_server_tool_call", + "arguments": { + "income": 120000, + "state": "CA" + }, + "id": "mcp_call_29012", + "server_name": "financial_mcp_server" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "McpServerToolResultDelta": { + "type": "object", + "properties": { + "result": { + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/TextContent" + } + ] + } + }, + { + "type": "object" + }, + { + "type": "string" + } + ] + }, + "type": { + "const": "mcp_server_tool_result" + }, + "name": { + "type": "string" + }, + "server_name": { + "type": "string" + } + }, + "required": [ + "result", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "McpServerToolResultStep": { + "description": "MCPServer tool result step.", + "type": "object", + "properties": { + "result": { + "description": "Required. The output from the MCP server call. Can be simple text or rich content.", + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/TextContent" + } + ] + } + }, + { + "type": "object" + }, + { + "type": "string" + } + ] + }, + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "mcp_server_tool_result" + }, + "name": { + "description": "Name of the tool which is called for this specific tool call.", + "type": "string" + }, + "server_name": { + "description": "The name of the used MCP server.", + "type": "string" + } + }, + "required": [ + "call_id", + "result", + "type" + ], + "examples": [ + { + "mcp_server_tool_result": { + "summary": "McpServerToolResultStep", + "value": { + "name": "calculate_tax", + "type": "mcp_server_tool_result", + "call_id": "mcp_call_29012", + "result": { + "tax_due": 32400 + }, + "server_name": "financial_mcp_server" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "MediaProcessing": { + "oneOf": [ + { + "$ref": "#/components/schemas/StaticMediaProcessing" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "MediaResolution": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "ultra_high" + ], + "x-google-enum-descriptions": [ + "Low resolution.", + "Medium resolution.", + "High resolution.", + "Ultra high resolution." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ModalityTokens": { + "description": "The token count for a single response modality.", + "type": "object", + "properties": { + "modality": { + "description": "The modality associated with the token count.", + "$ref": "#/components/schemas/ResponseModality" + }, + "tokens": { + "description": "Number of tokens for the modality.", + "type": "integer", + "format": "int32" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ModelOption": { + "title": "Model", + "description": "The model that will complete your prompt.\\n\\nSee [models](https://ai.google.dev/gemini-api/docs/models) for additional details.", + "type": "string", + "enum": [ + "gemma-4-26b-a4b-it", + "gemma-4-31b-it", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-2.5-flash-lite", + "gemini-3.5-flash-lite", + "gemini-2.5-flash-image", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-image", + "gemini-3-pro-image", + "nano-banana-pro-preview", + "gemini-3.1-flash-image", + "gemini-nano-banana-2.1", + "gemini-3.1-flash-tts-preview", + "gemini-3.5-flash", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-3.8-flash-tts", + "gemini-3.8-flash-lite-tts", + "lyria-3-clip-preview", + "lyria-3-pro-preview", + "gemini-robotics-er-1.6-preview", + "gemini-robotics-er-2-preview", + "lyria-3.5", + "gemini-omni-1.1-flash", + "gemini-omni-flash-preview" + ], + "x-speakeasy-enum-descriptions": [ + "Gemma 4 26B A4B IT", + "Gemma 4 31B IT", + "Latest release of Gemini Flash", + "Latest release of Gemini Flash-Lite", + "Latest release of Gemini Pro", + "Our first hybrid reasoning model which supports a 1M token context window and has thinking budgets.", + "Our state-of-the-art multipurpose model, which excels at coding and complex reasoning tasks.", + "Our smallest and most cost effective model, built for at scale usage.", + "Gemini 3.5 Flash-Lite - Our fastest, most cost-effective 3.5 model for high-throughput execution.", + "Our native image generation model, optimized for speed, flexibility, and contextual understanding. Text input and output is priced the same as 2.5 Flash.", + "Our most intelligent model built for speed, combining frontier intelligence with superior search and grounding.", + "Our latest SOTA reasoning model with unprecedented depth and nuance, and powerful multimodal understanding and coding capabilities.", + "Gemini 3.1 Pro Preview optimized for custom tool usage", + "Our most cost-efficient model, optimized for high-volume agentic tasks, translation, and simple data processing.", + "Gemini 3.1 Flash Lite Image.", + "Gemini 3 Pro Image", + "Gemini 3 Pro Image Preview", + "Gemini 3.1 Flash Image.", + "Gemini Nano Banana 2.1.", + "Gemini 3.1 Flash TTS: Powerful, low-latency speech generation. Enjoy natural outputs, steerable prompts, and new expressive audio tags for precise narration control.", + "Gemini 3.5 Flash - Our earlier Flash model, built for speed and foundational performance across routine, high-throughput workloads.", + "Gemini 3.6 Flash - Our previous generation Flash model, balancing speed and multimodal capabilities across general agentic and everyday tasks.", + "Gemini 3.7 Flash - Our high-speed, efficient Flash model built for everyday coding, agentic tool use, and reliable multi-step execution.", + "Gemini 3.8 Flash - Our most intelligent Flash model, engineered for long-horizon software engineering, autonomous agents, and complex enterprise workflows.", + "Gemini 3.8 Flash TTS - Flagship TTS model for Voice Design and dual-speaker screenplay control. Prompt custom vocal personas, direct line-by-line delivery, and add vocal bursts.", + "Gemini 3.8 Flash Lite TTS - High-speed and cost-efficient, ideal for rapid dubbing, media localization, and high-throughput voice agents. Direct replacement for gemini-3.1-flash-tts-preview.", + "Our low-latency, music generation model optimized for high-fidelity audio clips and precise rhythmic control.", + "Our advanced, full-song generative model with deep compositional understanding, optimized for precise structural control and complex transitions across diverse musical styles.", + "Gemini Robotics-ER 1.6 Preview", + "Gemini Robotics Embodied Reasoning 2 Preview", + "Our flagship music generation model, optimized for full-length songs with complex structural coherence.", + "Our high-performance multimodal model designed for fast, conversational video generation, editing, and cinematic control.", + "Preview release of our multimodal model for conversational video generation, editing, and cinematic control." + ], + "x-speakeasy-enum-format": "union", + "x-speakeasy-exports": [ + { + "name": "model", + "group": "interactions" + }, + { + "name": "model_param", + "group": "interactions" + } + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "Model", + "x-speakeasy-unknown-values": "allow" + }, + "ModelOutputStep": { + "description": "Output generated by the model.", + "type": "object", + "properties": { + "type": { + "const": "model_output" + }, + "content": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Content" + } + } + }, + "required": [ + "type" + ], + "examples": [ + { + "model_output": { + "summary": "ModelOutputStep", + "value": { + "type": "model_output", + "content": [ + { + "type": "text", + "text": "The capital of France is Paris." + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "OAuth2Config": { + "description": "Configuration for OAuth2 credentials with automatic token refresh.", + "type": "object", + "properties": { + "client_id": { + "description": "Required. OAuth2 client ID.", + "type": "string" + }, + "client_secret": { + "description": "Required. Input only. OAuth2 client secret. Write-only; never returned in responses.", + "type": "string", + "writeOnly": true + }, + "refresh_token": { + "description": "Required. Input only. OAuth2 refresh token. Write-only; never returned in responses.", + "type": "string", + "writeOnly": true + }, + "token_url": { + "description": "Required. OAuth2 token endpoint URL for refreshing access tokens.", + "type": "string" + }, + "id": { + "type": "string" + }, + "type": { + "const": "oauth2" + }, + "scopes": { + "description": "Optional. List of OAuth2 scopes.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "client_id", + "client_secret", + "id", + "refresh_token", + "token_url", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "OAuth2UpdateConfig": { + "description": "Configuration for updating OAuth2 credentials.", + "type": "object", + "properties": { + "type": { + "const": "oauth2" + }, + "client_id": { + "description": "Optional. OAuth2 client ID.", + "type": "string" + }, + "client_secret": { + "description": "Optional. Input only. OAuth2 client secret. Write-only; never returned in responses.", + "type": "string", + "writeOnly": true + }, + "refresh_token": { + "description": "Optional. Input only. OAuth2 refresh token. Write-only; never returned in responses.", + "type": "string", + "writeOnly": true + }, + "token_url": { + "description": "Optional. OAuth2 token endpoint URL for refreshing access tokens.", + "type": "string" + }, + "scopes": { + "description": "Optional. List of OAuth2 scopes.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "PingWebhookRequest": { + "description": "Request message for WebhookService.PingWebhook.", + "type": "object", + "properties": { + "id": { + "description": "Required. The ID of the webhook to ping.\nFormat: `{webhook_id}`", + "type": "string" + } + }, + "required": [ + "id" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "PingWebhookResponse": { + "description": "Response message for WebhookService.PingWebhook.", + "type": "object", + "x-speakeasy-model-namespace": "interactions" + }, + "Pitch": { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ], + "x-google-enum-descriptions": [ + "Lower pitch voice.", + "Medium pitch voice.", + "Higher pitch voice." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "PlaceCitation": { + "description": "A place citation annotation.", + "type": "object", + "properties": { + "type": { + "const": "place_citation" + }, + "place_id": { + "description": "The ID of the place, in `places/{place_id}` format.", + "type": "string" + }, + "name": { + "description": "Title of the place.", + "type": "string" + }, + "url": { + "description": "URI reference of the place.", + "type": "string" + }, + "review_snippets": { + "description": "Snippets of reviews that are used to generate answers about the\nfeatures of a given place in Google Maps.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ReviewSnippet" + } + }, + "start_index": { + "description": "Start of segment of the response that is attributed to this source.\n\nIndex indicates the start of the segment, measured in bytes.", + "type": "integer", + "format": "int32" + }, + "end_index": { + "description": "End of the attributed segment, exclusive.", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Places": { + "type": "object", + "properties": { + "place_id": { + "description": "The ID of the place, in `places/{place_id}` format.", + "type": "string" + }, + "name": { + "description": "Title of the place.", + "type": "string" + }, + "url": { + "description": "URI reference of the place.", + "type": "string" + }, + "review_snippets": { + "description": "Snippets of reviews that are used to generate answers about the\nfeatures of a given place in Google Maps.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ReviewSnippet" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "ProcessingCallDelta": { + "description": "Streaming delta for a server-initiated media processing step.", + "type": "object", + "properties": { + "type": { + "const": "processing_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ProcessingCallStep": { + "description": "A server-initiated processing step for media analysis (e.g. video\nunderstanding).", + "type": "object", + "properties": { + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "processing_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "id", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ProcessingResultDelta": { + "description": "Streaming delta for the result of a server-initiated media processing step.", + "type": "object", + "properties": { + "type": { + "const": "processing_result" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ProcessingResultStep": { + "description": "The result of a server-initiated media processing step.", + "type": "object", + "properties": { + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "processing_result" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "call_id", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "PromptedVoice": { + "description": "Parameters for prompted voice generation.\nRequired in `CreateVoice` when `type` is `\"prompted\"`. Returned in\n`CreateVoice`, `GetVoice`, and `ListVoices` responses for prompted voices.", + "type": "object", + "properties": { + "input": { + "description": "Required. The natural-language prompt describing the desired voice, e.g.\n\"A deep, booming male voice of a massive evil ogre in his middle years.\"", + "type": "string" + }, + "base_voice": { + "description": "Optional. The name or ID of an existing base voice to edit or remix.\nUse the voice name for prebuilt voices (e.g., `Puck`) or the voice ID for\npreviously created voices (e.g., `voice_abc123`). This can be any\nprompted or replicated voice that the user has access to.\nWhen present, `input` describes relative alterations to make to the base\nvoice.", + "type": "string" + } + }, + "required": [ + "input" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ReplicatedVoice": { + "description": "Input only. Parameters for replicated voice generation.\nRequired on input when `type` is `\"replicated\"`. Not returned in\nresponses.", + "type": "object", + "properties": { + "source_audio": { + "description": "Required. The reference audio sample of the speaker's voice to replicate.\nMust be an unmodified recording of an adult human voice.", + "$ref": "#/components/schemas/AudioData" + }, + "consent_audio": { + "description": "Required. Audio recording of the same speaker reading the required\nconsent phrase, confirming authorization to replicate the voice.", + "$ref": "#/components/schemas/AudioData" + } + }, + "required": [ + "consent_audio", + "source_audio" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ResponseFormat": { + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/AudioResponseFormat" + }, + { + "$ref": "#/components/schemas/ImageResponseFormat" + }, + { + "$ref": "#/components/schemas/TextResponseFormat" + }, + { + "$ref": "#/components/schemas/VideoResponseFormat" + }, + { + "type": "object", + "additionalProperties": true + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ResponseModality": { + "type": "string", + "enum": [ + "text", + "image", + "audio", + "video", + "document" + ], + "x-google-enum-descriptions": [ + "Indicates the model should return text.", + "Indicates the model should return images.", + "Indicates the model should return audio.", + "Indicates the model should return video.", + "Indicates the model should return documents." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ReviewSnippet": { + "description": "Encapsulates a snippet of a user review that answers a question about\nthe features of a specific place in Google Maps.", + "type": "object", + "properties": { + "title": { + "description": "Title of the review.", + "type": "string" + }, + "url": { + "description": "A link that corresponds to the user review on Google Maps.", + "type": "string" + }, + "review_id": { + "description": "The ID of the review snippet.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "RotateSigningSecretRequest": { + "description": "Request message for WebhookService.RotateSigningSecret.", + "type": "object", + "properties": { + "id": { + "description": "Required. The ID of the webhook for which to generate a signing secret.\nFormat: `{webhook_id}`", + "type": "string" + }, + "revocation_behavior": { + "description": "Optional. The revocation behavior for previous signing secrets.", + "type": "string", + "enum": [ + "revoke_previous_secrets_after_h24", + "revoke_previous_secrets_immediately" + ], + "x-google-enum-descriptions": [ + "Generate a new signing secret and revoke all previous secrets after 24\nhours. Default and safest option for migrations.", + "Revoke all previous secrets immediately. Use with caution as this can\ninterrupt ongoing notifications." + ] + } + }, + "required": [ + "id" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "RotateSigningSecretResponse": { + "description": "Response message for WebhookService.RotateSigningSecret.", + "type": "object", + "properties": { + "secret": { + "description": "Output only. The newly generated signing secret.", + "type": "string", + "readOnly": true + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "RunTriggerRequest": { + "description": "Request message for TriggerService.RunTrigger.", + "type": "object", + "properties": { + "id": { + "description": "Required. The ID of the trigger to run immediately.", + "type": "string" + } + }, + "required": [ + "id" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "SafetySetting": { + "description": "A safety setting that affects the safety-blocking behavior.\n\nA SafetySetting consists of a\nharm category and a\nthreshold for that\ncategory.", + "type": "object", + "properties": { + "type": { + "description": "Required. The type of harm category to be blocked.", + "$ref": "#/components/schemas/HarmCategory" + }, + "threshold": { + "description": "Required. The threshold for blocking content. If the harm probability\nexceeds this threshold, the content will be blocked.", + "type": "string", + "enum": [ + "block_low_and_above", + "block_medium_and_above", + "block_only_high", + "off" + ], + "x-google-enum-descriptions": [ + "Block content with a low harm probability or higher.", + "Block content with a medium harm probability or higher.", + "Block content with a high harm probability.", + "Turn off the safety filter entirely." + ] + }, + "method": { + "description": "Optional. The method for blocking content. If not specified, the default\nbehavior is to use the probability score.", + "type": "string", + "enum": [ + "severity", + "probability" + ], + "x-google-enum-descriptions": [ + "The harm block method uses both probability and severity scores.", + "The harm block method uses the probability score." + ] + } + }, + "required": [ + "threshold", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ServiceTier": { + "type": "string", + "enum": [ + "flex", + "standard", + "priority" + ], + "x-google-enum-descriptions": [ + "Flex service tier.", + "Standard service tier.", + "Priority service tier." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "SigningSecret": { + "description": "Represents a signing secret used to verify webhook payloads.", + "type": "object", + "properties": { + "truncated_secret": { + "description": "Output only. The truncated version of the signing secret.", + "type": "string", + "readOnly": true + }, + "expire_time": { + "description": "Output only. The expiration date of the signing secret.", + "type": "string", + "format": "date-time", + "readOnly": true + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "SmartTranscriptionMode": { + "description": "Configuration for smart transcription mode.", + "type": "object", + "properties": { + "type": { + "const": "smart" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Source": { + "description": "A source to be mounted into the environment.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gcs", + "inline", + "repository" + ], + "x-google-enum-descriptions": [ + "A Cloud Storage bucket.", + "Inline content.", + "A generic repository. The protocol prefix in the source URL\nidentifies the provider (e.g., github://, gcs://)." + ] + }, + "source": { + "description": "The source of the environment.\nFor Cloud Storage, this is the Cloud Storage path.\nFor GitHub, this is the GitHub path.", + "type": "string" + }, + "target": { + "description": "Where the source should appear in the environment.", + "type": "string" + }, + "content": { + "description": "The inline content if `type` is `INLINE`.", + "type": "string" + }, + "encoding": { + "description": "Optional encoding for inline content (e.g. `base64`).", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "SpeakerConfig": { + "description": "Configuration for multi-speaker and speech generation.", + "type": "object", + "properties": { + "speakers": { + "description": "Individual speaker configurations.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SpeechConfig" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "SpeechAnnotation": { + "description": "Speech annotation for text content.", + "type": "object", + "properties": { + "type": { + "const": "speech_metadata" + }, + "speaker": { + "description": "The speaker to associate with this turn.", + "type": "string" + }, + "style": { + "description": "Style instruction for the speech synthesis.", + "type": "string" + }, + "start_index": { + "description": "Start of segment of the response that is attributed to this source.\n\nIndex indicates the start of the segment, measured in bytes.", + "type": "integer", + "format": "int32" + }, + "end_index": { + "description": "End of the attributed segment, exclusive.", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "SpeechConfig": { + "description": "The configuration for speech interaction.", + "type": "object", + "properties": { + "voice": { + "description": "The voice of the speaker.", + "type": "string" + }, + "language": { + "description": "The language of the speech.", + "type": "string" + }, + "speaker": { + "description": "The speaker's name, it should match the speaker name given in the prompt.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "StaticMediaProcessing": { + "type": "object", + "properties": { + "type": { + "const": "static" + }, + "start_offset": { + "description": "Optional. Segment start time. Specified as a decimal number of seconds followed\nby an 's' suffix, e.g., \"10.5s\". Must be non-negative.", + "type": "string", + "format": "google-duration" + }, + "end_offset": { + "description": "Optional. Segment end time. Specified as a decimal number of seconds followed\nby an 's' suffix, e.g., \"30s\". Must be non-negative and greater than\n`start_offset` if `start_offset` is set.", + "type": "string", + "format": "google-duration" + }, + "fps": { + "description": "Optional. Video frame-rate sampling density.", + "type": "number", + "format": "double" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Step": { + "description": "A step in the interaction.", + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/CodeExecutionCallStep" + }, + { + "$ref": "#/components/schemas/CodeExecutionResultStep" + }, + { + "$ref": "#/components/schemas/FileSearchCallStep" + }, + { + "$ref": "#/components/schemas/FileSearchResultStep" + }, + { + "$ref": "#/components/schemas/FunctionCallStep" + }, + { + "$ref": "#/components/schemas/FunctionResultStep" + }, + { + "$ref": "#/components/schemas/GoogleMapsCallStep" + }, + { + "$ref": "#/components/schemas/GoogleMapsResultStep" + }, + { + "$ref": "#/components/schemas/GoogleSearchCallStep" + }, + { + "$ref": "#/components/schemas/GoogleSearchResultStep" + }, + { + "$ref": "#/components/schemas/McpServerToolCallStep" + }, + { + "$ref": "#/components/schemas/McpServerToolResultStep" + }, + { + "$ref": "#/components/schemas/ModelOutputStep" + }, + { + "$ref": "#/components/schemas/ProcessingCallStep" + }, + { + "$ref": "#/components/schemas/ProcessingResultStep" + }, + { + "$ref": "#/components/schemas/ThoughtStep" + }, + { + "$ref": "#/components/schemas/UrlContextCallStep" + }, + { + "$ref": "#/components/schemas/UrlContextResultStep" + }, + { + "$ref": "#/components/schemas/UserInputStep" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "StepDelta": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "format": "int32" + }, + "delta": { + "$ref": "#/components/schemas/StepDeltaData" + }, + "event_type": { + "const": "step.delta" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + }, + "metadata": { + "$ref": "#/components/schemas/StepDeltaMetadata" + } + }, + "required": [ + "delta", + "event_type", + "index" + ], + "examples": [ + { + "step_delta": { + "summary": "Step Delta", + "value": { + "delta": { + "type": "text", + "text": "Hello" + }, + "event_type": "step.delta", + "index": 0 + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "StepDeltaData": { + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/ArgumentsDelta" + }, + { + "$ref": "#/components/schemas/AudioDelta" + }, + { + "$ref": "#/components/schemas/CodeExecutionCallDelta" + }, + { + "$ref": "#/components/schemas/CodeExecutionResultDelta" + }, + { + "$ref": "#/components/schemas/DocumentDelta" + }, + { + "$ref": "#/components/schemas/FileSearchCallDelta" + }, + { + "$ref": "#/components/schemas/FileSearchResultDelta" + }, + { + "$ref": "#/components/schemas/FunctionResultDelta" + }, + { + "$ref": "#/components/schemas/GoogleMapsCallDelta" + }, + { + "$ref": "#/components/schemas/GoogleMapsResultDelta" + }, + { + "$ref": "#/components/schemas/GoogleSearchCallDelta" + }, + { + "$ref": "#/components/schemas/GoogleSearchResultDelta" + }, + { + "$ref": "#/components/schemas/ImageDelta" + }, + { + "$ref": "#/components/schemas/McpServerToolCallDelta" + }, + { + "$ref": "#/components/schemas/McpServerToolResultDelta" + }, + { + "$ref": "#/components/schemas/ProcessingCallDelta" + }, + { + "$ref": "#/components/schemas/ProcessingResultDelta" + }, + { + "$ref": "#/components/schemas/TextAnnotationDelta" + }, + { + "$ref": "#/components/schemas/TextDelta" + }, + { + "$ref": "#/components/schemas/ThoughtSignatureDelta" + }, + { + "$ref": "#/components/schemas/ThoughtSummaryDelta" + }, + { + "$ref": "#/components/schemas/UrlContextCallDelta" + }, + { + "$ref": "#/components/schemas/UrlContextResultDelta" + }, + { + "$ref": "#/components/schemas/VideoDelta" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "StepDeltaMetadata": { + "description": "Optional metadata accompanying ANY streamed event.", + "type": "object", + "properties": { + "total_usage": { + "description": "Statistics on the interaction request's token usage.", + "$ref": "#/components/schemas/Usage" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "StepStart": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "format": "int32" + }, + "step": { + "$ref": "#/components/schemas/Step" + }, + "event_type": { + "const": "step.start" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + } + }, + "required": [ + "event_type", + "index", + "step" + ], + "examples": [ + { + "step_start": { + "summary": "Step Start", + "value": { + "event_type": "step.start", + "index": 0, + "step": { + "type": "model_output" + } + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "StepStop": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "format": "int32" + }, + "event_type": { + "const": "step.stop" + }, + "usage": { + "description": "Cumulative model usage stats from the start of the session.", + "$ref": "#/components/schemas/Usage" + }, + "step_usage": { + "description": "Model usage stats for this specific step.", + "$ref": "#/components/schemas/Usage" + }, + "event_id": { + "description": "The event_id token to be used to resume the interaction stream, from\nthis event.", + "type": "string" + } + }, + "required": [ + "event_type", + "index" + ], + "examples": [ + { + "step_stop": { + "summary": "Step Stop", + "value": { + "event_type": "step.stop", + "index": 0 + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "TextAnnotationDelta": { + "type": "object", + "properties": { + "type": { + "const": "text_annotation_delta" + }, + "annotations": { + "description": "Citation information for model-generated content.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Annotation" + } + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "TextContent": { + "description": "A text content block.", + "type": "object", + "properties": { + "text": { + "description": "Required. The text content.", + "type": "string" + }, + "type": { + "const": "text" + }, + "annotations": { + "description": "Citation information for model-generated content.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Annotation" + } + } + }, + "required": [ + "text", + "type" + ], + "examples": [ + { + "text": { + "summary": "Text", + "value": { + "type": "text", + "text": "Hello, how are you?" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "TextDelta": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "type": { + "const": "text" + } + }, + "required": [ + "text", + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "TextResponseFormat": { + "description": "Configuration for text output format.", + "type": "object", + "properties": { + "type": { + "const": "text" + }, + "mime_type": { + "description": "The MIME type of the text output.", + "type": "string", + "enum": [ + "application/json", + "text/plain" + ], + "x-google-enum-descriptions": [ + "JSON output format.", + "Plain text output format." + ] + }, + "schema": { + "description": "The JSON schema that the output should conform to. Only applicable when\nmime_type is application/json.\n\nSupports a subset of JSON Schema. `minLength`, `maxLength`,\n`minProperties`, `maxProperties`, `uniqueItems`, `contains`,\n`minContains`, and `maxContains` are ignored and not enforced; validate\nthese in your application. `minItems` and `maxItems` are enforced. Schemas\nthat are too complex to enforce (for example, large `maxItems` on nested\narrays) return a 400 INVALID_ARGUMENT error.", + "type": "object", + "additionalProperties": { + "description": "Properties of the object." + } + } + }, + "required": [ + "type" + ], + "examples": [ + { + "text_response_format": { + "summary": "Text Output (JSON Schema)", + "value": { + "type": "text", + "mime_type": "application/json", + "schema": { + "type": "object", + "properties": { + "ingredients": { + "type": "array", + "items": { + "type": "string" + } + }, + "recipe_name": { + "type": "string" + } + }, + "required": [ + "ingredients", + "recipe_name" + ] + } + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ThinkingLevel": { + "type": "string", + "enum": [ + "minimal", + "low", + "medium", + "high" + ], + "x-google-enum-descriptions": [ + "Little to no thinking.", + "Low thinking level.", + "Medium thinking level.", + "High thinking level." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ThinkingSummaries": { + "type": "string", + "enum": [ + "auto", + "none" + ], + "x-google-enum-descriptions": [ + "Auto thinking summaries.", + "No thinking summaries." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ThoughtSignatureDelta": { + "type": "object", + "properties": { + "type": { + "const": "thought_signature" + }, + "signature": { + "description": "Signature to match the backend source to be part of the generation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ThoughtStep": { + "description": "A thought step.", + "type": "object", + "properties": { + "type": { + "const": "thought" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "summary": { + "description": "A summary of the thought.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Content" + } + } + }, + "required": [ + "type" + ], + "examples": [ + { + "thought": { + "summary": "ThoughtStep", + "value": { + "type": "thought", + "signature": "thought_sig_abcd1234", + "summary": [ + { + "type": "text", + "text": "The model is searching Google for the capital of France." + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ThoughtSummaryContent": { + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/ImageContent" + }, + { + "$ref": "#/components/schemas/TextContent" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ThoughtSummaryDelta": { + "type": "object", + "properties": { + "type": { + "const": "thought_summary" + }, + "content": { + "description": "A new summary item to be added to the thought.", + "$ref": "#/components/schemas/Content" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Tool": { + "description": "A tool that can be used by the model.", + "type": "object", + "oneOf": [ + { + "$ref": "#/components/schemas/CodeExecution" + }, + { + "$ref": "#/components/schemas/ComputerUse" + }, + { + "$ref": "#/components/schemas/FileSearch" + }, + { + "$ref": "#/components/schemas/Function" + }, + { + "$ref": "#/components/schemas/GoogleMaps" + }, + { + "$ref": "#/components/schemas/GoogleSearch" + }, + { + "$ref": "#/components/schemas/McpServer" + }, + { + "$ref": "#/components/schemas/UrlContext" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "ToolChoiceConfig": { + "description": "The tool choice configuration containing allowed tools.", + "type": "object", + "properties": { + "allowed_tools": { + "description": "The allowed tools.", + "$ref": "#/components/schemas/AllowedTools" + } + }, + "example": { + "allowed_tools": { + "mode": "any", + "tools": [ + "my_tool" + ] + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "TranscriptionConfig": { + "description": "Configuration for speech recognition (transcription).", + "type": "object", + "properties": { + "mode": { + "description": "Discriminated transcription mode options or enum.", + "oneOf": [ + { + "$ref": "#/components/schemas/TranscriptionMode" + }, + { + "type": "string", + "enum": [ + "verbatim", + "smart" + ], + "x-google-enum-descriptions": [ + "Verbatim transcription mode.", + "Smart transcription mode." + ] + } + ] + }, + "language_codes": { + "description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.", + "type": "array", + "items": { + "type": "string" + } + }, + "custom_vocabulary": { + "description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.", + "type": "array", + "items": { + "type": "string" + } + }, + "timestamp_granularities": { + "description": "Optional. The granularity of timestamps to include in the transcription output.\nSupported values: \"word\". If empty, no timestamps are generated.", + "type": "array", + "deprecated": true, + "items": { + "type": "string" + } + }, + "diarization_mode": { + "description": "Optional. Configures speaker diarization. Supported values: \"speaker\".", + "type": "string", + "deprecated": true + }, + "adaptation_phrases": { + "description": "Optional. A list of phrases to bias the ASR model towards.", + "type": "array", + "deprecated": true, + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "TranscriptionMode": { + "description": "Configuration for transcription mode.", + "oneOf": [ + { + "$ref": "#/components/schemas/SmartTranscriptionMode" + }, + { + "$ref": "#/components/schemas/VerbatimTranscriptionMode" + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Trigger": { + "description": "A trigger configuration that is scheduled to run an agent.", + "type": "object", + "properties": { + "schedule": { + "description": "Required. The cron schedule on which the trigger should run.\nStandard cron format.", + "type": "string" + }, + "time_zone": { + "description": "Required. Time zone in which the schedule should be interpreted.", + "type": "string" + }, + "interaction": { + "description": "Required. The interaction request template to be executed.", + "$ref": "#/components/schemas/CreateAgentInteractionParams" + }, + "id": { + "description": "Required. Output only. Identifier. The ID of the trigger.", + "type": "string", + "readOnly": true, + "x-google-identifier": true + }, + "display_name": { + "description": "Optional. The display name of the trigger.", + "type": "string" + }, + "environment_id": { + "description": "Optional. The environment ID for the trigger execution.", + "type": "string" + }, + "max_consecutive_failures": { + "description": "Optional. The maximum number of consecutive failures allowed before\nthe trigger is automatically paused (status becomes ERROR).", + "type": "integer", + "format": "int32" + }, + "execution_timeout_seconds": { + "description": "Optional. The execution timeout for the triggered interaction.", + "type": "integer", + "format": "int32" + }, + "consecutive_failure_count": { + "description": "Output only. The number of consecutive failures that have occurred\nsince the last successful execution.", + "type": "integer", + "format": "int32", + "readOnly": true + }, + "create_time": { + "description": "Output only. The time when the trigger was created.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "update_time": { + "description": "Output only. The time when the trigger was last updated.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "last_run_time": { + "description": "Output only. The time when the trigger was last run.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "next_run_time": { + "description": "Output only. The time when the trigger is scheduled to run next.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "last_pause_time": { + "description": "Output only. The time when the trigger was last paused.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "last_resume_time": { + "description": "Output only. The time when the trigger was last resumed.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "previous_interaction_id": { + "description": "Output only. The ID of the last interaction created by this trigger.", + "type": "string", + "readOnly": true + }, + "status": { + "description": "Output only. The current status of the trigger.", + "type": "string", + "enum": [ + "active", + "paused", + "error" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "The trigger is active and will fire on schedule.", + "The trigger is paused and will not fire.", + "The trigger has entered an error state due to consecutive failures." + ] + } + }, + "required": [ + "id", + "interaction", + "schedule", + "time_zone" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "TriggerExecution": { + "description": "An execution instance of a trigger.", + "type": "object", + "properties": { + "id": { + "description": "Required. Output only. Identifier. The ID of the trigger execution.", + "type": "string", + "readOnly": true, + "x-google-identifier": true + }, + "trigger_id": { + "description": "Required. Output only. Identifier. The ID of the trigger that created this execution.", + "type": "string", + "readOnly": true, + "x-google-identifier": true + }, + "scheduled_time": { + "description": "Output only. The time when the execution was scheduled to run.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "start_time": { + "description": "Output only. The time when the execution started.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "end_time": { + "description": "Output only. The time when the execution finished.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "status": { + "description": "Output only. The status of the execution.", + "type": "string", + "enum": [ + "in_progress", + "completed", + "failed", + "skipped", + "timed_out" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "The execution is currently in progress.", + "The execution completed successfully.", + "The execution failed.", + "The execution was skipped (e.g., previous execution still running).", + "The execution timed out." + ] + }, + "interaction_id": { + "description": "Output only. The ID of the interaction created by this execution, if any.", + "type": "string", + "readOnly": true + }, + "environment_id": { + "description": "Output only. The environment ID used for the execution.", + "type": "string", + "readOnly": true + }, + "error": { + "description": "Output only. The error message if the execution failed.", + "type": "string", + "readOnly": true + } + }, + "required": [ + "id", + "trigger_id" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "TriggerUpdate": { + "description": "Represents the fields of a Trigger that can be updated.", + "type": "object", + "properties": { + "status": { + "description": "Optional. The status of the trigger.", + "type": "string", + "enum": [ + "active", + "paused", + "error" + ], + "x-google-enum-descriptions": [ + "The trigger is active and will fire on schedule.", + "The trigger is paused and will not fire.", + "The trigger has entered an error state due to consecutive failures." + ] + }, + "display_name": { + "description": "Optional. The display name of the trigger.", + "type": "string" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "UpdateTriggerRequest": { + "description": "Request message for TriggerService.UpdateTrigger.", + "type": "object", + "properties": { + "trigger": { + "description": "Required. The trigger to update.", + "$ref": "#/components/schemas/TriggerUpdate" + }, + "id": { + "description": "Required. The ID of the trigger to update.", + "type": "string" + }, + "update_mask": { + "description": "Optional. The update mask applies to the resource.", + "type": "string", + "format": "google-fieldmask", + "pattern": "^(\\s*[^,\\s.]+(\\s*[,.]\\s*[^,\\s.]+)*)?$" + } + }, + "required": [ + "id", + "trigger" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "UpdateWebhookRequest": { + "description": "Request message for WebhookService.UpdateWebhook.", + "type": "object", + "properties": { + "webhook": { + "description": "Required. The webhook to update.", + "$ref": "#/components/schemas/WebhookUpdate" + }, + "id": { + "description": "Required. The ID of the webhook to update.", + "type": "string" + }, + "update_mask": { + "description": "Optional. The list of fields to update.", + "type": "string", + "format": "google-fieldmask", + "pattern": "^(\\s*[^,\\s.]+(\\s*[,.]\\s*[^,\\s.]+)*)?$" + } + }, + "required": [ + "id", + "webhook" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "UploadEnvironmentFileRequest": { + "description": "Request for `UploadEnvironmentFile`.", + "type": "object", + "properties": { + "overwrite": { + "description": "Optional. Whether to overwrite the destination file if it already exists.", + "type": "boolean" + }, + "extract": { + "description": "Optional. If true, treats the uploaded file as a tar/tar.gz archive and\nunpacks it into `path`.", + "type": "boolean" + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "UploadEnvironmentFileResponse": { + "description": "Response for `UploadEnvironmentFile`.", + "type": "object", + "properties": { + "files": { + "description": "List of files created or extracted in the environment.", + "type": "array", + "items": { + "$ref": "#/components/schemas/EnvironmentFile" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "UrlCitation": { + "description": "A URL citation annotation.", + "type": "object", + "properties": { + "type": { + "const": "url_citation" + }, + "url": { + "description": "The URL.", + "type": "string" + }, + "title": { + "description": "The title of the URL.", + "type": "string" + }, + "start_index": { + "description": "Start of segment of the response that is attributed to this source.\n\nIndex indicates the start of the segment, measured in bytes.", + "type": "integer", + "format": "int32" + }, + "end_index": { + "description": "End of the attributed segment, exclusive.", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLCitation" + }, + "UrlContext": { + "description": "A tool that can be used by the model to fetch URL context.", + "type": "object", + "properties": { + "type": { + "const": "url_context" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContext", + "x-codeSamples": [ + { + "label": "url_context", + "lang": "sh", + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"gemini-flash-latest\",\n \"tools\": [{\n \"type\": \"url_context\"\n }],\n \"input\": \"Summarize https://www.example.com\"\n }'\n" + }, + { + "label": "url_context", + "lang": "python", + "source": "from google import genai\n\nclient = genai.Client()\nresponse = client.interactions.create(\n model=\"gemini-flash-latest\",\n tools=[{\"type\": \"url_context\"}],\n input=\"Summarize https://www.example.com\",\n)\nprint(response.output_text)\n" + }, + { + "label": "url_context", + "lang": "javascript", + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n model: 'gemini-flash-latest',\n tools: [{type: 'url_context'}],\n input: 'Summarize https://www.example.com',\n});\nconsole.log(interaction.output_text);\n" + }, + { + "label": "url_context", + "lang": "java", + "source": "import com.google.genai.Client;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.CreateModelInteraction;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\nimport com.google.genai.gaos.models.interactions.URLContext;\n\nClient client = new Client();\nCreateModelInteraction params =\n CreateModelInteraction.builder()\n .model(\"gemini-flash-latest\")\n .tools(List.of(new URLContext()))\n .input(InteractionsInput.of(\"Summarize https://www.example.com\"))\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.outputText().orElse(\"\"));\n" + }, + { + "label": "url_context", + "lang": "go", + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n tool := gaos_interactions.NewTool(gaos_interactions.URLContext{})\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateModelInteraction{\n Model: gaos_interactions.Model(\"gemini-flash-latest\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"Summarize https://www.example.com\")),\n Tools: []gaos_interactions.Tool{tool},\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil && res.Interaction.OutputText != nil {\n fmt.Println(*res.Interaction.OutputText)\n }\n}\n" + } + ] + }, + "UrlContextCallArguments": { + "description": "The arguments to pass to the URL context.", + "type": "object", + "properties": { + "urls": { + "description": "The URLs to fetch.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContextCallArguments" + }, + "UrlContextCallDelta": { + "type": "object", + "properties": { + "arguments": { + "$ref": "#/components/schemas/UrlContextCallArguments" + }, + "type": { + "const": "url_context_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "arguments", + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContextCallDelta" + }, + "UrlContextCallStep": { + "description": "URL context call step.", + "type": "object", + "properties": { + "arguments": { + "description": "Required. The arguments to pass to the URL context.", + "$ref": "#/components/schemas/UrlContextCallArguments" + }, + "id": { + "description": "Required. A unique ID for this specific tool call.", + "type": "string" + }, + "type": { + "const": "url_context_call" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "arguments", + "id", + "type" + ], + "examples": [ + { + "url_context_call": { + "summary": "UrlContextCallStep", + "value": { + "type": "url_context_call", + "arguments": { + "urls": [ + "https://www.example.com" + ] + }, + "id": "url_call_10219" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContextCallStep" + }, + "UrlContextResult": { + "description": "The result of the URL context.", + "type": "object", + "properties": { + "url": { + "description": "The URL that was fetched.", + "type": "string" + }, + "status": { + "description": "The status of the URL retrieval.", + "type": "string", + "enum": [ + "success", + "error", + "paywall", + "unsafe" + ], + "x-google-enum-descriptions": [ + "Url retrieval is successful.", + "Url retrieval is failed due to error.", + "Url retrieval is failed because the content is behind paywall.", + "Url retrieval is failed because the content is unsafe." + ] + } + }, + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContextResult" + }, + "UrlContextResultDelta": { + "type": "object", + "properties": { + "result": { + "type": "array", + "items": { + "$ref": "#/components/schemas/UrlContextResult" + } + }, + "type": { + "const": "url_context_result" + }, + "is_error": { + "type": "boolean" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "result", + "type" + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContextResultDelta" + }, + "UrlContextResultStep": { + "description": "URL context result step.", + "type": "object", + "properties": { + "result": { + "description": "Required. The results of the URL context.", + "type": "array", + "items": { + "$ref": "#/components/schemas/UrlContextResult" + } + }, + "call_id": { + "description": "Required. ID to match the ID from the function call block.", + "type": "string" + }, + "type": { + "const": "url_context_result" + }, + "is_error": { + "description": "Whether the URL context resulted in an error.", + "type": "boolean" + }, + "signature": { + "description": "A signature hash for backend validation.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + } + }, + "required": [ + "call_id", + "result", + "type" + ], + "examples": [ + { + "url_context_result": { + "summary": "UrlContextResultStep", + "value": { + "type": "url_context_result", + "call_id": "url_call_10219", + "result": [ + { + "title": "Example Domain", + "url": "https://www.example.com", + "snippet": "This domain is for use in illustrative examples in documents." + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions", + "x-speakeasy-name-override": "URLContextResultStep" + }, + "Usage": { + "description": "Statistics on the interaction request's token usage.", + "type": "object", + "properties": { + "total_input_tokens": { + "description": "Number of tokens in the prompt (context).", + "type": "integer", + "format": "int32" + }, + "input_tokens_by_modality": { + "description": "A breakdown of input token usage by modality.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ModalityTokens" + } + }, + "total_cached_tokens": { + "description": "Number of tokens in the cached part of the prompt (the cached content).", + "type": "integer", + "format": "int32" + }, + "cached_tokens_by_modality": { + "description": "A breakdown of cached token usage by modality.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ModalityTokens" + } + }, + "total_output_tokens": { + "description": "Total number of tokens across all the generated responses.", + "type": "integer", + "format": "int32" + }, + "output_tokens_by_modality": { + "description": "A breakdown of output token usage by modality.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ModalityTokens" + } + }, + "total_tool_use_tokens": { + "description": "Number of tokens present in tool-use prompt(s).", + "type": "integer", + "format": "int32" + }, + "tool_use_tokens_by_modality": { + "description": "A breakdown of tool-use token usage by modality.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ModalityTokens" + } + }, + "total_thought_tokens": { + "description": "Number of tokens of thoughts for thinking models.", + "type": "integer", + "format": "int32" + }, + "total_tokens": { + "description": "Total token count for the interaction request (prompt + responses + other\ninternal tokens).", + "type": "integer", + "format": "int32" + }, + "grounding_tool_count": { + "description": "Grounding tool count.", + "type": "array", + "items": { + "$ref": "#/components/schemas/GroundingToolCount" + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "UserInputStep": { + "description": "Input provided by the user.", + "type": "object", + "properties": { + "type": { + "const": "user_input" + }, + "content": { + "title": "ContentList", + "type": "array", + "items": { + "$ref": "#/components/schemas/Content" + } + } + }, + "required": [ + "type" + ], + "examples": [ + { + "user_input": { + "summary": "UserInputStep", + "value": { + "type": "user_input", + "content": [ + { + "type": "text", + "text": "What is the capital of France?" + } + ] + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "VerbatimTranscriptionMode": { + "description": "Configuration for verbatim transcription mode.", + "type": "object", + "properties": { + "type": { + "const": "verbatim" + }, + "timestamp_granularities": { + "description": "Optional. The granularity of timestamps to include in the transcription output.\nSupported values: \"word\". If empty, no timestamps are generated.", + "type": "array", + "items": { + "type": "string" + } + }, + "diarization_mode": { + "description": "Optional. Configures speaker diarization. Supported values: \"speaker\".", + "type": "string" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "VideoConfig": { + "description": "Configuration options for video generation.", + "type": "object", + "properties": { + "task": { + "description": "Optional task mode for video generation. If not specified, the model\nautomatically determines the appropriate mode based on the provided text\nprompt and input media.", + "type": "string", + "enum": [ + "text_to_video", + "image_to_video", + "reference_to_video", + "edit", + "extend" + ], + "x-google-enum-descriptions": [ + "Generates video solely from a text prompt.", + "Generates video from one or two source images. The first image defines\nthe starting frame, and the optional second image defines the ending\nframe.", + "Generates video using reference media (such as images, audio, or video).", + "Modifies an existing input video.", + "Extends an existing input video." + ] + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "VideoContent": { + "description": "A video content block.", + "type": "object", + "properties": { + "type": { + "const": "video" + }, + "data": { + "description": "The video content.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "description": "The URI of the video.", + "type": "string" + }, + "processing": { + "description": "How the model processes this video for understanding.", + "oneOf": [ + { + "$ref": "#/components/schemas/MediaProcessing" + }, + { + "type": "string", + "enum": [ + "static", + "agentic" + ], + "x-google-enum-descriptions": [ + "Fixed-rate frame extraction. All frames placed in context.", + "Model-driven dynamic navigation." + ] + } + ] + }, + "resolution": { + "description": "The resolution of the media.", + "$ref": "#/components/schemas/MediaResolution" + }, + "name": { + "description": "A user-defined name for this content block. Can be referenced by the model\nin the final response.", + "type": "string" + }, + "mime_type": { + "description": "The mime type of the video.", + "type": "string", + "enum": [ + "video/mp4", + "video/mpeg", + "video/mpg", + "video/mov", + "video/avi", + "video/x-flv", + "video/webm", + "video/wmv", + "video/3gpp" + ], + "x-google-enum-descriptions": [ + "MP4 video format", + "MPEG video format", + "MPG video format", + "MOV video format", + "AVI video format", + "FLV video format", + "WebM video format", + "WMV video format", + "3GPP video format" + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "video": { + "summary": "Video", + "value": { + "type": "video", + "uri": "https://www.youtube.com/watch?v=9hE5-98ZeCg" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "VideoDelta": { + "type": "object", + "properties": { + "type": { + "const": "video" + }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, + "mime_type": { + "type": "string", + "enum": [ + "video/mp4", + "video/mpeg", + "video/mpg", + "video/mov", + "video/avi", + "video/x-flv", + "video/webm", + "video/wmv", + "video/3gpp", + "video/jpeg2000" + ], + "x-google-enum-descriptions": [ + "MP4 video format", + "MPEG video format", + "MPG video format", + "MOV video format", + "AVI video format", + "FLV video format", + "WebM video format", + "WMV video format", + "3GPP video format", + "JPEG 2000 video format" + ] + }, + "resolution": { + "description": "The resolution of the media.", + "$ref": "#/components/schemas/MediaResolution" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "VideoResponseFormat": { + "description": "Configuration for video output format.", + "type": "object", + "properties": { + "type": { + "const": "video" + }, + "delivery": { + "description": "The delivery mode for the video output.", + "type": "string", + "enum": [ + "inline", + "uri" + ], + "x-google-enum-descriptions": [ + "Video data is returned inline in the response.", + "Video data is returned as a URI." + ] + }, + "aspect_ratio": { + "description": "The aspect ratio for the video output.", + "type": "string", + "enum": [ + "16:9", + "9:16" + ], + "x-google-enum-descriptions": [ + "16:9 aspect ratio.", + "9:16 aspect ratio." + ] + }, + "duration": { + "description": "The duration for the video output.", + "type": "string", + "format": "google-duration" + }, + "resolution": { + "description": "The video output resolution. Defaults to 720p.", + "type": "string", + "enum": [ + "360p", + "720p", + "1080p", + "4k" + ], + "x-google-enum-descriptions": [ + "360p resolution.", + "720p resolution.", + "1080p resolution.", + "4K resolution." + ] + } + }, + "required": [ + "type" + ], + "examples": [ + { + "video_response_format": { + "summary": "Video Output", + "value": { + "type": "video", + "aspect_ratio": "16:9", + "delivery": "inline" + } + } + } + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Voice": { + "description": "A voice resource representing either a custom voice (created via\n`CreateVoice`) or a prebuilt system voice (returned by `ListVoices`).", + "type": "object", + "properties": { + "type": { + "description": "Required. The type of the voice. In `CreateVoice`, must be `\"replicated\"`\nor `\"prompted\"`. In `ListVoices` responses, may also be `\"prebuilt\"`.", + "$ref": "#/components/schemas/VoiceType" + }, + "id": { + "description": "Output only. The unique identifier of the voice.\n\n* For Google-managed custom voices (`store = true`), this is a generated ID\n with prefix `voice_` (for example, `voice_abc123def456`). Pass\n `voices/{id}` as the `name` in `GetVoice` and `DeleteVoice`, and pass\n `{id}` directly to `SpeechConfig.voice_config.voice` (or\n `SpeechConfig.voice`) during speech synthesis.\n* For prebuilt catalog voices (`\"prebuilt\"` returned by `ListVoices`), this\n is the speaker name (for example, `Puck` or `Charon`).\n* Unset when `CreateVoice` is called with `store = false`.", + "type": "string", + "readOnly": true + }, + "model": { + "description": "Optional. The model used to design or replicate the voice.\nIf omitted in `CreateVoice`, defaults to the latest supported voice design\nmodel. Returned in `CreateVoice`, `GetVoice`, and `ListVoices` responses\nfor custom voices (`\"prompted\"` and `\"replicated\"`); unset for `\"prebuilt\"`\nvoices. Created voices can be synthesized across any supported TTS\nsynthesis model.", + "type": "string" + }, + "key": { + "description": "Output only. The client-managed voice replication key (with prefix\n`voicekey_`). Returned only by `CreateVoice` when `type` is `\"replicated\"`\nand `store` is `false`. Pass this key to `SpeechConfig.voice_config.voice`\n(or `SpeechConfig.voice`) during speech synthesis.", + "type": "string", + "readOnly": true + }, + "replicated": { + "description": "Input only. Parameters for replicated voice generation.\nRequired on input when `type` is `\"replicated\"`. Not returned in\nresponses.", + "$ref": "#/components/schemas/ReplicatedVoice", + "writeOnly": true + }, + "expire_time": { + "description": "Output only. The timestamp at which a custom stored voice (`store = true`)\nor replicated voice key (`store = false`) expires. For custom stored voices\n(`store = true`), this expiration time is extended when the voice is used\nfor speech synthesis or as a `base_voice` in `CreateVoice`. Unset for\nprebuilt catalog voices (`\"prebuilt\"`), which do not expire.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "display_name": { + "description": "Optional. User-provided display name for a stored voice (`store = true`),\nor the catalog name for a prebuilt voice.", + "type": "string" + }, + "prompted": { + "description": "Parameters for prompted voice generation.\nRequired in `CreateVoice` when `type` is `\"prompted\"`. Returned in\n`CreateVoice`, `GetVoice`, and `ListVoices` responses for prompted voices.", + "$ref": "#/components/schemas/PromptedVoice" + }, + "language_code": { + "description": "Optional. Primary BCP-47 language tag (e.g. \"en-US\", \"fr-FR\").", + "type": "string" + }, + "region_code": { + "description": "Optional. ISO 3166-1 alpha-2 or UN M.49 geographic region code (e.g. \"US\",\n\"GB\", \"001\").", + "type": "string" + }, + "accent": { + "description": "Optional. Regional accent descriptor (e.g. \"American\", \"British\").", + "type": "string" + }, + "persona": { + "description": "Optional. Intended persona or character archetype (e.g. \"Warm, Friendly\",\n\"Narrator\").", + "type": "string" + }, + "context": { + "description": "Optional. Optimal usage context or domain (e.g. \"Conversational\",\n\"Audiobook\", \"News\").", + "type": "string" + }, + "gender": { + "description": "Optional. Perceived voice gender presentation (e.g. \"female\", \"male\",\n\"neutral\").", + "type": "string" + }, + "pitch": { + "description": "Optional. Voice pitch classification.", + "$ref": "#/components/schemas/Pitch" + }, + "description": { + "description": "Optional. Descriptive summary of vocal timbre, personality, and tone.", + "type": "string" + }, + "usage": { + "description": "Output only. Token usage statistics for the voice creation request.\nPopulated only in the response of `CreateVoice` when `type` is\n`\"prompted\"`; unset for `\"replicated\"` and in `GetVoice` and `ListVoices`\nresponses.", + "$ref": "#/components/schemas/Usage", + "readOnly": true + }, + "sample_audio": { + "description": "Output only. Sample audio (synthesizer prompt audio) generated for a\nprompted voice.\nPopulated only in `CreateVoice` and `GetVoice` responses when `type` is\n`\"prompted\"`; unset in `ListVoices` responses and for `\"replicated\"` or\n`\"prebuilt\"` voices.", + "$ref": "#/components/schemas/AudioData", + "readOnly": true + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "VoiceType": { + "type": "string", + "enum": [ + "replicated", + "prompted", + "prebuilt" + ], + "x-google-enum-descriptions": [ + "A custom voice replicated from a reference audio sample and speaker consent\nrecording.", + "A custom voice generated from a natural-language text prompt (optionally\nediting or remixing a `base_voice`).", + "A built-in system voice from Google's voice catalog (e.g., `Puck`,\n`Charon`). Returned in responses; cannot be specified as the type in\n`CreateVoice`." + ], + "x-speakeasy-model-namespace": "interactions" + }, + "Webhook": { + "description": "A Webhook resource.", + "type": "object", + "properties": { + "uri": { + "description": "Required. The URI to which webhook events will be sent.", + "type": "string" + }, + "subscribed_events": { + "description": "Required. The events that the webhook is subscribed to.\nAvailable events:\n- batch.succeeded\n- batch.expired\n- batch.failed\n- interaction.requires_action\n- interaction.completed\n- interaction.failed\n- video.generated", + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "description": "Optional. The user-provided name of the webhook.", + "type": "string" + }, + "create_time": { + "description": "Output only. The timestamp when the webhook was created.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "update_time": { + "description": "Output only. The timestamp when the webhook was last updated.", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "signing_secrets": { + "description": "Output only. The signing secrets associated with this webhook.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SigningSecret" + }, + "readOnly": true + }, + "state": { + "description": "Output only. The state of the webhook.", + "type": "string", + "enum": [ + "enabled", + "disabled", + "disabled_due_to_failed_deliveries" + ], + "readOnly": true, + "x-google-enum-descriptions": [ + "The webhook is enabled.", + "The webhook is disabled by the user.", + "The webhook is disabled due to failed deliveries." + ] + }, + "new_signing_secret": { + "description": "Output only. The new signing secret for the webhook. Only populated on create.", + "type": "string", + "readOnly": true + }, + "id": { + "description": "Output only. The ID of the webhook.", + "type": "string", + "readOnly": true + } + }, + "required": [ + "subscribed_events", + "uri" + ], + "x-speakeasy-model-namespace": "interactions" + }, + "WebhookConfig": { + "description": "Message for configuring webhook events for a request.", + "type": "object", + "properties": { + "uris": { + "description": "Optional. If set, these webhook URIs will be used for webhook events instead of the\nregistered webhooks.", + "type": "array", + "items": { + "type": "string" + } + }, + "user_metadata": { + "description": "Optional. The user metadata that will be returned on each event emission to the\nwebhooks.", + "type": "object", + "additionalProperties": { + "description": "Properties of the object." + } + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "WebhookUpdate": { + "type": "object", + "properties": { + "name": { + "description": "Optional. The user-provided name of the webhook.", + "type": "string" + }, + "uri": { + "description": "Optional. The URI to which webhook events will be sent.", + "type": "string" + }, + "subscribed_events": { + "description": "Optional. The events that the webhook is subscribed to.\nAvailable events:\n- batch.succeeded\n- batch.expired\n- batch.failed\n- interaction.requires_action\n- interaction.completed\n- interaction.failed\n- video.generated", + "type": "array", + "items": { + "type": "string" + } + }, + "state": { + "description": "Optional. The state of the webhook.", + "type": "string", + "enum": [ + "enabled", + "disabled", + "disabled_due_to_failed_deliveries" + ], + "x-google-enum-descriptions": [ + "The webhook is enabled.", + "The webhook is disabled by the user.", + "The webhook is disabled due to failed deliveries." + ] + } + }, + "x-speakeasy-model-namespace": "interactions" + }, + "WordInfo": { + "description": "Word-level ASR annotation for transcription output.\nCarries the word text, optional timing, and optional speaker attribution.", + "type": "object", + "properties": { + "type": { + "const": "word_info" + }, + "text": { + "description": "The transcribed word.", + "type": "string" + }, + "start_offset": { + "description": "Start offset in time of the word relative to the start of the audio.\nPresent when timestamp_granularities contains \"word\".", + "type": "string", + "format": "google-duration" + }, + "end_offset": { + "description": "End offset in time of the word relative to the start of the audio.\nPresent when timestamp_granularities contains \"word\".", + "type": "string", + "format": "google-duration" + }, + "speaker": { + "description": "Optional. Speaker label for this word (e.g. \"spk_1\", \"spk_2\").\nPresent when diarization_mode is set in TranscriptionConfig.", + "type": "string" + }, + "start_index": { + "description": "Start of segment of the response that is attributed to this source.\n\nIndex indicates the start of the segment, measured in bytes.", + "type": "integer", + "format": "int32" + }, + "end_index": { + "description": "End of the attributed segment, exclusive.", + "type": "integer", + "format": "int32" + } + }, + "required": [ + "type" + ], + "x-speakeasy-model-namespace": "interactions" + } + }, + "securitySchemes": { + "googleGenAIAuth": { + "type": "http", + "scheme": "custom", + "x-speakeasy-custom-security-scheme": { + "schema": { + "type": "object", + "properties": { + "accessToken": { + "description": "OAuth access token sent as a bearer Authorization header.", + "type": "string" + }, + "apiKey": { + "description": "Gemini API key sent as x-goog-api-key.", + "type": "string" + }, + "defaultHeaders": { + "description": "Additional default headers to apply before request-specific headers and auth.", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [] + } + } + } + } + }, + "security": [ + { + "googleGenAIAuth": [] + } + ], + "x-speakeasy-globals": { + "parameters": [ + { + "$ref": "#/components/parameters/api_version" + }, + { + "$ref": "#/components/parameters/google_genai_user_project" + } + ] + }, + "x-speakeasy-retries": { + "backoff": { + "exponent": 2, + "initialInterval": 500, + "maxElapsedTime": 30000, + "maxInterval": 8000 + }, + "retryConnectionErrors": true, + "statusCodes": [ + 408, + 409, + 429, + "5XX" + ], + "strategy": "backoff" + } +} diff --git a/tests/unit/interactions/test_openapi_compliance.py b/tests/unit/interactions/test_openapi_compliance.py index 247d02298aa..b6cc92e0adc 100644 --- a/tests/unit/interactions/test_openapi_compliance.py +++ b/tests/unit/interactions/test_openapi_compliance.py @@ -1,41 +1,56 @@ """ OpenAPI compliance tests for Google Interactions API. -Validates that our SDK requests/responses match the OpenAPI spec at: -https://ai.google.dev/static/api/interactions.openapi.json +Validates that our SDK requests/responses match the OpenAPI spec snapshot in +interactions.openapi.json, taken from https://ai.google.dev/static/api/interactions.openapi.json +(litellm/types/interactions/README.md says how to refresh it). The scheduled +interactions-spec-canary workflow runs the same tests against the live spec by passing +--interactions-spec-url, so a change Google makes shows up there, not in every PR. Run with: pytest tests/unit/interactions/test_openapi_compliance.py -v """ import json -import os import re -from typing import Any, Dict +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, Dict, Final, get_args from unittest.mock import MagicMock, patch import httpx import pytest from openapi_core import OpenAPI +from pydantic import BaseModel, TypeAdapter OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json" +OPENAPI_SPEC_SNAPSHOT: Final = Path(__file__).with_name("interactions.openapi.json") +SPEC_URL_OPTION: Final = TypeAdapter(str | None) -def _load_openapi_spec_dict() -> Dict[str, Any]: - """ - Load the OpenAPI spec JSON. +@dataclass(frozen=True, slots=True) +class LoadedSpec: + source: str + spec: Dict[str, Any] - In CI or offline environments, network access may not be available. - In that case, gracefully skip these tests instead of erroring. - """ - try: - response = httpx.get(OPENAPI_SPEC_URL, timeout=5.0) - response.raise_for_status() - return response.json() - except Exception as e: # pragma: no cover - defensive, env-dependent - pytest.skip( - f"Skipping Google Interactions OpenAPI compliance tests - " - f"unable to load spec from {OPENAPI_SPEC_URL}: {e}" - ) + +def _status_enum_of(model: type[BaseModel]) -> type[Enum]: + annotation: Final = model.model_fields["status"].annotation + candidates: Final = (annotation, *get_args(annotation)) + return next(candidate for candidate in candidates if isinstance(candidate, type) and issubclass(candidate, Enum)) + + +def _requested_spec_url(request: pytest.FixtureRequest) -> str | None: + return SPEC_URL_OPTION.validate_python(request.config.getoption("interactions_spec_url")) + + +def _load_openapi_spec(live_url: str | None) -> LoadedSpec: + if live_url is None: + return LoadedSpec(source=str(OPENAPI_SPEC_SNAPSHOT), spec=json.loads(OPENAPI_SPEC_SNAPSHOT.read_text())) + with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0) as client: + response: Final = client.get(live_url) + response.raise_for_status() + return LoadedSpec(source=str(response.url), spec=response.json()) def _model_create_request_schema(spec_dict: Dict[str, Any]) -> Dict[str, Any]: @@ -65,9 +80,13 @@ def _declared_type_value(variant_schema: Dict[str, Any]) -> Any: @pytest.fixture(scope="module") -def spec_dict() -> Dict[str, Any]: - """Load raw spec dict for manual validation.""" - return _load_openapi_spec_dict() +def loaded_spec(request: pytest.FixtureRequest) -> LoadedSpec: + return _load_openapi_spec(_requested_spec_url(request)) + + +@pytest.fixture(scope="module") +def spec_dict(loaded_spec: LoadedSpec) -> Dict[str, Any]: + return loaded_spec.spec @pytest.fixture(scope="module") @@ -76,6 +95,18 @@ def openapi_spec(spec_dict: Dict[str, Any]) -> OpenAPI: return OpenAPI.from_dict(spec_dict) +def test_spec_comes_from_the_requested_source(loaded_spec: LoadedSpec, request: pytest.FixtureRequest) -> None: + assert loaded_spec.source == (_requested_spec_url(request) or str(OPENAPI_SPEC_SNAPSHOT)) + + +def test_interaction_status_export_carries_the_interaction_models_statuses() -> None: + from litellm.types.interactions import InteractionStatus, generated + + exported: Final = frozenset(member.value for member in InteractionStatus) + modelled: Final = frozenset(member.value for member in _status_enum_of(generated.Interaction)) + assert exported == modelled, sorted(exported ^ modelled) + + class TestRequestCompliance: """Tests that our request bodies match the OpenAPI spec.""" @@ -249,27 +280,24 @@ class TestResponseCompliance: assert field in schema["properties"], f"Output field '{field}' not in spec" print(f"✓ Output field '{field}' exists in spec") - def test_status_enum_values(self, spec_dict): - """Verify status enum values match spec.""" - # `status` is an output-only field; validate against the response schema. - schema = spec_dict["components"]["schemas"]["Interaction"] - status_prop = schema["properties"]["status"] - # Google Interactions API uses lowercase status values (updated Feb 2026). - # Keep this an exact match: this test intentionally breaks CI when - # Google changes the live spec — that breakage is how we get notified - # to review the change. - expected_statuses = [ - "in_progress", - "requires_action", - "completed", - "failed", - "cancelled", - "incomplete", - "budget_exceeded", - "queued", - ] - assert status_prop["enum"] == expected_statuses - print(f"✓ Status enum values: {expected_statuses}") + @pytest.mark.parametrize( + "schema_name", + [ + "Interaction", + "InteractionStatusUpdate", + "CreateModelInteractionParams", + "CreateAgentInteractionParams", + "UrlContextResult", + ], + ) + def test_status_enum_values(self, spec_dict, schema_name): + from litellm.types.interactions import generated + + schema = spec_dict["components"]["schemas"][schema_name] + spec_statuses: Final = frozenset(schema["properties"]["status"]["enum"]) + parsed_statuses: Final = frozenset(member.value for member in _status_enum_of(getattr(generated, schema_name))) + assert spec_statuses + assert spec_statuses <= parsed_statuses, sorted(spec_statuses - parsed_statuses) def test_usage_schema(self, spec_dict): """Verify Usage schema fields.""" diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 5ee14ff530d..9e8e9cf2a80 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -160,6 +160,12 @@ def test_api_base_line_keeps_an_at_sign_in_the_path(api_base: str) -> None: # Tuple format: (error_message, expected_result) context_window_test_cases = [ # Positive cases (should return True) + ( + '"{\n "error": {\n "message": "Your input exceeds the context window of this model. ' + 'Please adjust your input and try again.",\n "type": "invalid_request_error",\n' + ' "param": "input",\n "code": "context_length_exceeded"\n }\n}"', + True, + ), ( "An error occurred: The input exceeds the model's maximum context limit of 8192 tokens.", True, diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index d7474936425..57e66805678 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -8,6 +8,7 @@ from litellm import CustomLLM from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, + get_provider_affinity_header, inferred_provider, is_registered_custom_provider, ) @@ -706,3 +707,11 @@ class TestClaudeModelPatternMatching: set_fallback_generalizations([]) with pytest.raises(litellm.BadRequestError): litellm.get_llm_provider(model="claude-opus-4-9") + + +@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"]) +def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None: + api_base: Final = "https://opencode.ai/zen/go/v1" + model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base) + assert (model, provider, destination) == ("test-model", prefix, api_base) + assert get_provider_affinity_header(destination) == "x-opencode-session" diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index 84e785ac99a..d44169ae28a 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -399,9 +399,10 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): @pytest.mark.asyncio -async def test_refetch_respects_local_env_override(monkeypatch): - """LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"]) +async def test_refetch_respects_local_env_override(monkeypatch, flag): + """LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True short-circuits to the bundled backup, zero HTTP.""" + monkeypatch.setenv(flag, "True") def _fail(request): raise AssertionError("no HTTP request should be made when local map is forced") @@ -708,9 +709,10 @@ def test_boot_load_falls_back_when_urlsplit_rejects_the_url(): assert "invalid URL" in (source["fallback_reason"] or "") -def test_boot_load_respects_local_env_override(monkeypatch): - """LITELLM_LOCAL_MODEL_COST_MAP=True still short-circuits to the backup with zero HTTP.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"]) +def test_boot_load_respects_local_env_override(monkeypatch, flag): + """LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True still short-circuits to the backup with zero HTTP.""" + monkeypatch.setenv(flag, "True") def _fail(request): raise AssertionError("no HTTP request should be made when local map is forced") diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index bbfae2229ce..8509b1d6f7d 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2047,7 +2047,9 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key( ) -> None: """The exporter must post to Langtrace's complete /api/trace path with the key in x-api-key, without leaking it into the process-wide OTEL_EXPORTER_OTLP_TRACES_HEADERS.""" + import requests_mock from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.trace.export import SpanExportResult from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils import litellm_logging as logging_module @@ -2069,8 +2071,12 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key( assert type(logger) is OpenTelemetry and logger.callback_name == "langtrace" exporter: Final = logger._get_span_processor().span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._endpoint == expected_endpoint - assert exporter._headers == {"x-api-key": api_key} + with requests_mock.Mocker() as collector: + collector.post(expected_endpoint, status_code=200) + assert exporter.export(()) is SpanExportResult.SUCCESS + assert [(request.url, request.headers.get("x-api-key")) for request in collector.request_history] == [ + (expected_endpoint, api_key) + ] assert "OTEL_EXPORTER_OTLP_TRACES_HEADERS" not in os.environ finally: logging_module._in_memory_loggers.clear() diff --git a/tests/unit/litellm_core_utils/test_offline_mode.py b/tests/unit/litellm_core_utils/test_offline_mode.py new file mode 100644 index 00000000000..f5a4786837d --- /dev/null +++ b/tests/unit/litellm_core_utils/test_offline_mode.py @@ -0,0 +1,17 @@ +import pytest + +from litellm.litellm_core_utils.offline_mode import is_offline_mode + + +@pytest.mark.parametrize( + ("value", "expected"), + [("true", True), ("True", True), (" TRUE ", True), ("false", False), ("", False), ("1", False), ("yes", False)], +) +def test_is_offline_mode_reads_true_case_insensitively(monkeypatch, value, expected): + monkeypatch.setenv("LITELLM_OFFLINE_MODE", value) + assert is_offline_mode() is expected + + +def test_is_offline_mode_is_off_when_unset(monkeypatch): + monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) + assert is_offline_mode() is False diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 194d7995735..88af656ad2e 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1173,6 +1173,27 @@ def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch: assert counted == litellm.token_counter(model="", text=_CODE_SAMPLE) +@pytest.mark.parametrize("model", ["llama-2-7b", "meta-llama/llama-3-8b-instruct"]) +def test_offline_mode_counts_downloaded_tokenizer_models_with_tiktoken(monkeypatch: pytest.MonkeyPatch, model: str): + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + + with patch("litellm.utils.tokenizer_dispatch.from_pretrained") as from_pretrained: + result: Final = _select_tokenizer_helper(model) + + from_pretrained.assert_not_called() + assert result["type"] == "openai_tokenizer" + + +def test_offline_mode_keeps_deepseek_v4_on_its_bundled_tokenizer(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + + assert litellm.token_counter(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) == _deepseek_token_count( + _CODE_SAMPLE + ) + + def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer(): _load_huggingface_tokenizer.cache_clear() _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 1f02b3aaa5a..f4920505722 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -2033,6 +2033,7 @@ async def test_anthropic_messages_with_extra_headers(): ], "model": "claude-haiku-4-5-20251001", "stop_reason": "end_turn", + "stop_sequence": None, "usage": {"input_tokens": 10, "output_tokens": 20}, } @@ -2096,6 +2097,7 @@ async def test_anthropic_messages_with_thinking(): ], "model": "claude-haiku-4-5-20251001", "stop_reason": "end_turn", + "stop_sequence": None, "usage": {"input_tokens": 10, "output_tokens": 20}, } @@ -2210,6 +2212,7 @@ async def test_anthropic_messages_bedrock_dynamic_region(): "content": [{"type": "text", "text": "This is a mock response"}], "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "stop_reason": "end_turn", + "stop_sequence": None, "usage": {"input_tokens": 10, "output_tokens": 20}, } diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py index 7fbec3ba12e..457d5d5e1c8 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py @@ -1,17 +1,23 @@ from collections.abc import Mapping +from datetime import datetime, timezone from pathlib import Path from typing import Final from unittest.mock import patch +import httpx import pytest +from pydantic import JsonValue import litellm +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.anthropic.wif import get_anthropic_wif_token from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig +from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig +from litellm.types.llms.anthropic import AnthropicFinishReason from litellm.types.router import GenericLiteLLMParams from tests.unit.llms.anthropic.test_anthropic_wif import ( ScriptedPoster, @@ -27,6 +33,52 @@ _WIF_PARAMS: Final[dict] = { } +@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type) +@pytest.mark.parametrize( + ("stop_reason", "stop_fields", "expected_stop_sequence"), + ( + pytest.param("end_turn", {}, None, id="end_turn_missing"), + pytest.param("max_tokens", {}, None, id="max_tokens_missing"), + pytest.param("tool_use", {}, None, id="tool_use_missing"), + pytest.param("end_turn", {"stop_sequence": None}, None, id="explicit_null"), + pytest.param("stop_sequence", {"stop_sequence": ", 7"}, ", 7", id="matched_stop"), + ), +) +def test_native_response_defaults_stop_sequence_without_changing_other_fields( + config: AnthropicMessagesConfig, + stop_reason: AnthropicFinishReason, + stop_fields: Mapping[str, str | None], + expected_stop_sequence: str | None, +) -> None: + upstream: Final[dict[str, JsonValue]] = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "test-model", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": stop_reason, + "usage": {"input_tokens": 19, "output_tokens": 42}, + **stop_fields, + } + logging_obj: Final = Logging( + model="test-model", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-call", + function_id="test-function", + ) + response: Final = config.transform_anthropic_messages_response( + model="test-model", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj + ) + + assert response == { + **upstream, + "stop_sequence": expected_stop_sequence, + } + + def test_workload_identity_allowed_for_anthropic() -> None: assert AnthropicMessagesConfig()._allows_workload_identity is True diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py index fd9aa75be74..4ed9054f3d3 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py @@ -26,7 +26,9 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( is_provider_error_chunk, parse_anthropic_error_event, ) +from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator +from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig from litellm.types.utils import Choices, Message, ModelResponse @@ -1733,6 +1735,49 @@ async def _drained(stream: AsyncIterator[bytes]) -> list[bytes]: return [chunk async for chunk in stream] +@pytest.mark.asyncio +@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type) +@pytest.mark.parametrize("stop_sequence", (None, ", 7"), ids=("null", "matched_stop")) +async def test_native_stream_preserves_nullable_stop_fields_across_chunk_boundaries( + config: AnthropicMessagesConfig, stop_sequence: str | None +) -> None: + logging_obj: Final = _make_logging_obj("native_stream_stop_fields") + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": logging_obj.model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 0}, + }, + }, + { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn" if stop_sequence is None else "stop_sequence", + "stop_sequence": stop_sequence, + }, + "usage": {"output_tokens": 0}, + }, + {"type": "message_stop"}, + ) + upstream: Final = b"".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events) + response: Final = httpx.Response(200, content=_chunked(upstream, 7)) + stream: Final = config.get_async_streaming_response_iterator( + model=logging_obj.model, + httpx_response=response, + request_body={"model": logging_obj.model}, + litellm_logging_obj=logging_obj, + ) + + assert b"".join(await _drained(stream)) == upstream + + def _data_events(frames: Sequence[bytes]) -> list[Mapping[str, object]]: return [ json.loads(line[len(b"data:") :]) for frame in frames for line in frame.splitlines() if line.startswith(b"data:") diff --git a/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py b/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py index 3826b22af34..ace82e517ad 100644 --- a/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py +++ b/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py @@ -1,6 +1,10 @@ -import litellm +import json +from unittest.mock import MagicMock, patch + +import httpx import pytest -from unittest.mock import MagicMock + +import litellm @pytest.mark.asyncio @@ -134,6 +138,34 @@ async def test_aws_polly_tts_with_native_voice(): assert request_body.get("TextType", "text") == "text" +@pytest.mark.asyncio +@pytest.mark.usefixtures("fake_provider_credentials") +@pytest.mark.parametrize("engine", ["standard", "neural", "long-form", "generative"]) +async def test_aws_polly_tts_sends_engine_from_model(engine: str) -> None: + """ + The engine in the model name must reach Polly. speech() strips the + provider prefix before the config reads it, which used to turn every + engine into the neural default. + """ + mock_httpx_response = MagicMock(spec=httpx.Response) + mock_httpx_response.content = b"fake_audio_data" + mock_httpx_response.status_code = 200 + mock_httpx_response.headers = {"content-type": "audio/mpeg"} + + with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post: + mock_post.return_value = mock_httpx_response + + await litellm.aspeech( + model=f"aws_polly/{engine}", + voice="Joanna", + input="Hello", + aws_region_name="us-east-1", + ) + + request_body = json.loads(mock_post.call_args.kwargs["data"]) + assert request_body["Engine"] == engine + + @pytest.mark.asyncio @pytest.mark.usefixtures("fake_provider_credentials") async def test_aws_polly_tts_with_openai_voice_mapping(): diff --git a/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py b/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py index 4778b6296fc..5f195b7a478 100644 --- a/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py +++ b/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py @@ -1,6 +1,8 @@ import json from typing import Final +import pytest + from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig @@ -17,3 +19,17 @@ def test_installed_botocore_signs_the_speech_request() -> None: authorization: Final = headers["Authorization"] assert "/us-west-2/polly/aws4_request" in authorization assert json.loads(body) == {"Text": "ping", "VoiceId": "Joanna"} + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("standard", "standard"), + ("Long-Form", "long-form"), + ("aws_polly/generative", "generative"), + ("aws_polly", "neural"), + ("unknown", "neural"), + ], +) +def test_extract_engine_from_model(model: str, expected: str) -> None: + assert AWSPollyTextToSpeechConfig()._extract_engine_from_model(model) == expected diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 7ed98f6c705..d46242fc5f7 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -21,6 +21,7 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +from tests.unit.proxy.conftest import httpx_transport @pytest.mark.serial @@ -100,6 +101,69 @@ def test_get_complete_url(): assert result == expected +@pytest.mark.parametrize( + ("base_path", "expected_path"), + [ + ("/foundry/openai/openai/v1", "/foundry/openai/openai/v1/responses?api-version=v1"), + ("/foundry/openai/openai/v1/", "/foundry/openai/openai/v1/responses?api-version=v1"), + ("/foundry/openai/openai/v1/responses", "/foundry/openai/openai/v1/responses?api-version=v1"), + ("/foundry/openai/openai/v1/responses/", "/foundry/openai/openai/v1/responses?api-version=v1"), + ( + "/openai/v1?redirect=/openai/v1/responses", + "/openai/v1/responses?redirect=%2Fopenai%2Fv1%2Fresponses&api-version=v1", + ), + ( + "/openai/v1?tenant=example&api-version=preview", + "/openai/v1/responses?tenant=example&api-version=preview", + ), + ( + "/openai/v1/responses?api-version=2025-04-01-preview", + "/openai/v1/responses?api-version=2025-04-01-preview", + ), + ( + "/openai/v1/responses?redirect=/openai/responses", + "/openai/v1/responses?redirect=%2Fopenai%2Fresponses&api-version=v1", + ), + ("/foo%2Fbar/openai/v1?tenant=example", "/foo%2Fbar/openai/v1/responses?tenant=example&api-version=v1"), + ( + "/foo%3Fbar/openai/v1/responses?tenant=example", + "/foo%3Fbar/openai/v1/responses?tenant=example&api-version=v1", + ), + ("/openai", "/openai/v1/responses?api-version=v1"), + ("/foundry/openai", "/foundry/openai/v1/responses?api-version=v1"), + ], +) +def test_explicit_v1_api_base_preserves_the_gateway_prefix_and_query(base_path: str, expected_path: str) -> None: + result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url( + api_base=f"https://gateway.example.com{base_path}", litellm_params={"api_version": "v1"} + ) + + assert result == f"https://gateway.example.com{expected_path}" + + +@pytest.mark.parametrize( + ("litellm_params", "expected_version"), + [ + ({}, AZURE_DEFAULT_RESPONSES_API_VERSION), + ({"api_version": "v1"}, "v1"), + ({"api_version": "latest"}, "latest"), + ({"api_version": "preview"}, "preview"), + ({"api_version": "2025-04-01-preview"}, "2025-04-01-preview"), + ({"api_version": "2024-10-21"}, AZURE_DEFAULT_RESPONSES_API_VERSION), + ({"api_version": "2024-10-21", "responses_api_version": "preview"}, "preview"), + ], +) +def test_explicit_v1_api_base_keeps_responses_api_version_selection( + litellm_params: dict[str, str], expected_version: str +) -> None: + result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url( + api_base="https://gateway.example.com/foundry/openai/openai/v1", + litellm_params=litellm_params, + ) + + assert result == f"https://gateway.example.com/foundry/openai/openai/v1/responses?api-version={expected_version}" + + @pytest.mark.serial def test_response_id_path_requests_encode_response_id(): config = AzureOpenAIResponsesAPIConfig() @@ -947,6 +1011,119 @@ def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url( assert route.calls[0].request.url == _unset_api_version_responses_url() +_COMPLETED_RESPONSE_SSE: Final = "".join( + f"data: {json.dumps(event)}\n\n" + for event in ( + { + "type": "response.output_text.delta", + "sequence_number": 0, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": "Done.", + }, + {"type": "response.completed", "sequence_number": 1, "response": _COMPLETED_RESPONSE}, + ) +) + + +@pytest.fixture +def gateway_responses(respx_mock: respx.MockRouter, stream: bool) -> respx.Route: + body: Final = _COMPLETED_RESPONSE_SSE if stream else json.dumps(_COMPLETED_RESPONSE) + content_type: Final = "text/event-stream" if stream else "application/json" + return respx_mock.post( + "https://gateway.example.com/foundry/openai/openai/v1/responses?api-version=v1" + ).mock(return_value=httpx.Response(200, text=body, headers={"content-type": content_type})) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_explicit_v1_base_sends_responses_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None: + response: Final = litellm.responses( + model="azure/gpt-6.1-sol", + input="hi", + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join(event.delta for event in response if event.type == "response.output_text.delta") + if stream + else response.output[0].content[0].text + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + +@pytest.mark.parametrize("stream", [False, True]) +def test_explicit_v1_base_sends_bridged_chat_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None: + response: Final = litellm.completion( + model="azure/responses/gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join(chunk.choices[0].delta.content or "" for chunk in response) + if stream + else response.choices[0].message.content + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +async def test_explicit_v1_base_sends_async_responses_to_the_gateway( + gateway_responses: respx.Route, stream: bool +) -> None: + response: Final = await litellm.aresponses( + model="azure/gpt-6.1-sol", + input="hi", + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join([event.delta async for event in response if event.type == "response.output_text.delta"]) + if stream + else response.output[0].content[0].text + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +async def test_explicit_v1_base_sends_async_bridged_chat_to_the_gateway( + gateway_responses: respx.Route, stream: bool +) -> None: + response: Final = await litellm.acompletion( + model="azure/responses/gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join([chunk.choices[0].delta.content or "" async for chunk in response]) + if stream + else response.choices[0].message.content + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + _LIFECYCLE_BODIES: Final = { "get": _COMPLETED_RESPONSE, "delete": {"id": "resp_dated", "object": "response", "deleted": True}, diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 5186541a62b..ccac6ffc309 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -879,6 +879,32 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages(): } +def test_azure_ai_reference_only_tool_result_is_sent_as_empty_text(): + messages: Final = [ + {"role": "user", "content": "load the WebFetch tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_reference", "tool_name": "WebFetch"}], + }, + ] + + request: Final = AzureAIStudioConfig().transform_request( + model="fw-glm-5.2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""} + + @pytest.mark.parametrize( "api_base, expected_url", [ diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py index ffddb09cca7..8cf69ba4cd3 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py @@ -1,5 +1,6 @@ import asyncio import base64 +import contextlib import hashlib import json import logging @@ -18,6 +19,9 @@ import tiktoken import litellm from litellm._logging import verbose_logger from litellm.llms.bedrock.count_tokens.kimi_k3 import ( + HUGGING_FACE_SOURCE, + KIMI_K3_RANKS_FILE, + KIMI_K3_REVISION, KimiK3Encodings, KimiK3TokenizerSource, kimi_k3_anthropic_tokens, @@ -522,3 +526,40 @@ async def test_tokenizer_load_still_running_after_the_wait_is_logged( assert loaded is None assert any("Kimi K3 tokenizer still loading" in record.getMessage() for record in caplog.records) + + +@pytest.mark.parametrize( + ("offline_mode", "disable_download", "asks_hub"), + [("true", None, False), ("false", True, False), ("false", None, True)], +) +def test_offline_mode_and_disabled_downloads_never_ask_hugging_face_for_the_tokenizer( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, offline_mode: str, disable_download: bool | None, asks_hub: bool +): + import huggingface_hub.constants + import huggingface_hub.utils._detect_agent + + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path / "hub")) + monkeypatch.setattr(huggingface_hub.constants, "AGENT_HARNESSES_PATH", str(tmp_path / "agent_harnesses.json")) + monkeypatch.setattr(huggingface_hub.utils._detect_agent, "_registry", None) + monkeypatch.setenv("LITELLM_OFFLINE_MODE", offline_mode) + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", disable_download) + with respx.mock(assert_all_called=False) as router: + hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503)) + with contextlib.suppress(Exception): + HUGGING_FACE_SOURCE.ranks_path() + assert hub.called is asks_hub + + +def test_offline_mode_reads_the_tokenizer_from_the_hugging_face_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + import huggingface_hub.constants + + cached: Final = tmp_path / "models--moonshotai--Kimi-K3" / "snapshots" / KIMI_K3_REVISION / KIMI_K3_RANKS_FILE + cached.parent.mkdir(parents=True) + cached.write_bytes(b"cached ranks") + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + with respx.mock(assert_all_called=False) as router: + hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503)) + path: Final = HUGGING_FACE_SOURCE.ranks_path() + assert Path(path).read_bytes() == b"cached ranks" + assert not hub.called diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 63cc45b29df..8b32ab2e6b8 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5542,3 +5542,85 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_ sent: Final = received[0] assert "safeguards" not in json.loads(sent.content) assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +@pytest.mark.parametrize( + ("api_base", "explicit_header", "expected_session"), + [ + ("https://opencode.ai/inference/go/anthropic", None, "conversation-123"), + ("https://opencode.ai/inference/go/anthropic", "caller-override", "caller-override"), + ("https://other.example", None, None), + ], +) +async def test_messages_opencode_session_header( + stream: bool, + environment_base: str | None, + api_base: str, + explicit_header: str | None, + expected_session: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + if environment_base: + monkeypatch.setenv(environment_base, "https://opencode.ai/inference/go/anthropic") + received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + payload: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "OK"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def respond(request: httpx.Request) -> httpx.Response: + received.put_nowait(request) + return ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': payload})}\n\n" + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ) + if stream + else httpx.Response(200, json=payload) + ) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + try: + result: Final = await litellm.anthropic.messages.acreate( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": "hello"}], + max_tokens=128000, + api_base=None if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" else api_base, + api_key="test-key", + stream=stream, + litellm_session_id="conversation-123", + client=upstream, + extra_headers={"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"}, + ) + if stream: + chunks: Final = b"".join([chunk async for chunk in result]) + assert b'"text": "OK"' in chunks + assert b"message_stop" in chunks + else: + assert result["content"] == [{"type": "text", "text": "OK"}] + finally: + await upstream.close() + sent: Final = received.get_nowait() + assert str(sent.url) == f"{api_base}/v1/messages" + assert sent.headers.get("x-opencode-session") == expected_session + assert sent.headers["x-api-key"] == "test-key" + assert json.loads(sent.content) == { + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 128000, + "stream": stream, + } + assert received.empty() diff --git a/tests/unit/llms/opencode/test_transformation.py b/tests/unit/llms/opencode/test_transformation.py new file mode 100644 index 00000000000..96203c23122 --- /dev/null +++ b/tests/unit/llms/opencode/test_transformation.py @@ -0,0 +1,90 @@ +from typing import Final + +import pytest + +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header +from litellm.llms.opencode.transformation import OpenCodeProviderAdapter + + +@pytest.mark.parametrize( + "api_base", + [ + "https://opencode.ai/zen/v1", + "https://opencode.ai/zen/go/v1/", + "https://opencode.ai/inference/openai/v1", + "https://opencode.ai/inference/go/openai/v1", + "https://opencode.ai/inference/go/anthropic/v1", + "https://opencode.ai/inference/anthropic/v1", + ], +) +def test_opencode_defaults_affinity_header_to_the_callers_session(api_base: str) -> None: + headers: Final = {"Authorization": "Bearer test-key"} + params: Final = {"metadata": {"session_id": "conversation-123"}} + assert add_provider_affinity_header( + headers, params, OpenCodeProviderAdapter().get_provider_affinity_header(api_base) + ) == { + **headers, + "x-opencode-session": "conversation-123", + } + assert headers == {"Authorization": "Bearer test-key"} + assert params == {"metadata": {"session_id": "conversation-123"}} + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.openai.com/v1", + "https://opencode.ai.example/zen/go/v1", + "https://opencode.ai@example.com/zen/go/v1", + "https://example.com/opencode.ai/zen/go/v1", + "https://example.com/v1?upstream=https://opencode.ai/zen/go/v1", + "https://opencode.ai/console/v1", + "http://opencode.ai/zen/go/v1", + ], +) +def test_other_destinations_do_not_receive_an_automatic_session_header(api_base: str | None) -> None: + headers: Final = {"Authorization": "Bearer test-key", "x-existing": "unchanged"} + assert ( + add_provider_affinity_header( + headers, + {"litellm_session_id": "conversation-123"}, + OpenCodeProviderAdapter().get_provider_affinity_header(api_base), + ) + == headers + ) + + +@pytest.mark.parametrize( + "params", + [ + {}, + {"litellm_trace_id": "request-123"}, + {"metadata": {"trace_id": "request-123"}}, + { + "litellm_session_id": "request-123", + "litellm_metadata": {"session_id": "request-123", SESSION_ID_GENERATED_METADATA_KEY: True}, + }, + ], +) +def test_opencode_does_not_invent_a_conversation_id(params: dict[str, object]) -> None: + assert ( + add_provider_affinity_header( + {}, params, OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1") + ) + == {} + ) + + +def test_opencode_preserves_explicit_header_and_affinity_configuration() -> None: + assert add_provider_affinity_header( + {"X-OpenCode-Session": "explicit"}, + {"litellm_session_id": "conversation-123"}, + OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"), + ) == {"X-OpenCode-Session": "explicit"} + assert add_provider_affinity_header( + {}, + {"litellm_session_id": "conversation-123", "provider_affinity_header": "x-custom-session"}, + OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"), + ) == {"x-custom-session": "conversation-123"} diff --git a/tests/unit/proxy/auth/test_password_policy.py b/tests/unit/proxy/auth/test_password_policy.py index f9f5025b57f..0ad1083ebe0 100644 --- a/tests/unit/proxy/auth/test_password_policy.py +++ b/tests/unit/proxy/auth/test_password_policy.py @@ -180,6 +180,18 @@ async def test_breach_check_skipped_when_disabled(): assert result is None +@pytest.mark.asyncio +async def test_breach_check_skipped_in_offline_mode(monkeypatch): + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + password = "correct horse battery staple" + result = await validate_password_not_breached( + password=password, + general_settings={}, + client=_client_returning(f"{_sha1_upper(password)[5:]}:42"), + ) + assert result is None + + @pytest.mark.asyncio async def test_rejects_breached_password(): password = "correct horse battery staple" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 4cdd7fefd28..108f7ce2983 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -11688,6 +11688,124 @@ def _alias_request(route: str, data: dict, content_type: str = "application/json return request +def _viewer_inference_key(monkeypatch: pytest.MonkeyPatch, team_id: str | None, *, spend: float = 0.25) -> str: + from litellm.proxy import proxy_server + from litellm.proxy.utils import ProxyLogging, hash_token + + key: Final = "sk-viewer-inference" + hashed_key: Final = hash_token(key) + cache: Final = UserApiKeyCache() + cache.set_cache( + key=hashed_key, + value=UserAPIKeyAuth( + token=hashed_key, + user_id="viewer", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + team_id=team_id, + models=["gpt-6.1-sol"], + max_budget=1.0, + spend=spend, + ), + ) + cache.set_cache( + key="viewer", + value=LiteLLM_UserTable( + user_id="viewer", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + user_email="viewer@example.com", + models=["gpt-6.1-sol"], + ), + ) + if team_id is not None: + cache.set_cache( + key=team_membership_reservation_cache_key(user_id="viewer", team_id=team_id), + value=NO_TEAM_MEMBERSHIP_SENTINEL, + ) + for name, value in { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": MagicMock(), + "user_api_key_cache": cache, + "spend_counter_cache": DualCache(), + "proxy_logging_obj": ProxyLogging(user_api_key_cache=cache), + "llm_model_list": [], + }.items(): + monkeypatch.setattr(proxy_server, name, value) + return key + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +@pytest.mark.parametrize("route", ["/chat/completions", "/v1/chat/completions"]) +async def test_viewer_inference_uses_the_normal_key_authorization_path( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, route: str +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id) + request: Final = _alias_request(route, {"model": "gpt-6.1-sol", "messages": [{"role": "user", "content": "usage"}]}) + + result: Final = await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert (result.user_id, result.user_role, result.team_id, result.models, result.max_budget) == ( + "viewer", + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + team_id, + ["gpt-6.1-sol"], + 1.0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +async def test_viewer_inference_rejects_a_model_outside_the_key_allowlist( + monkeypatch: pytest.MonkeyPatch, team_id: str | None +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id) + request: Final = _alias_request("/chat/completions", {"model": "blocked-model"}) + + with pytest.raises(ProxyException) as failure: + await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert (failure.value.code, failure.value.type, failure.value.param) == ( + "403", + ProxyErrorTypes.key_model_access_denied, + "model", + ) + assert failure.value.message == ( + "The requested model 'blocked-model' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +@pytest.mark.parametrize("spend", [1.0, 1.25], ids=["at-limit", "over-limit"]) +async def test_viewer_inference_rejects_an_exhausted_key_budget( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, spend: float +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id, spend=spend) + request: Final = _alias_request("/chat/completions", {"model": "gpt-6.1-sol"}) + + with pytest.raises(ProxyException) as failure: + await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert failure.value.type == ProxyErrorTypes.budget_exceeded + assert f"Current cost: {spend}, Max budget: 1.0" in failure.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +async def test_viewer_inference_access_does_not_allow_management_writes( + monkeypatch: pytest.MonkeyPatch, team_id: str | None +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id) + request: Final = _alias_request("/team/new", {"team_alias": "forbidden"}) + + with pytest.raises(ProxyException) as failure: + await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert failure.value.code == "403" + assert "proxy_admin_viewer" in failure.value.message + + async def _enforce_alias_access(token: UserAPIKeyAuth, data: dict, route: str, request, router: litellm.Router): from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access @@ -12480,11 +12598,14 @@ async def test_lens_oauth2_validates_identity_without_inference_user_field( patch("litellm.proxy.proxy_server.master_key", "sk-fixture-master"), patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), - patch.dict(os.environ, { - "OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect", - "OAUTH_CLIENT_ID": "fixture-client", - "OAUTH_CLIENT_SECRET": "fixture-client-secret", - }), + patch.dict( + os.environ, + { + "OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect", + "OAUTH_CLIENT_ID": "fixture-client", + "OAUTH_CLIENT_SECRET": "fixture-client-secret", + }, + ), patch("httpx.AsyncClient.send", side_effect=token_info) as network, ): if active: @@ -12611,6 +12732,4 @@ def test_should_skip_budget_checks_spares_only_zero_spend_mcp_discovery( ) -> None: from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks - assert ( - _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected - ) + assert _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 7437042e4bd..9b2a96bbc58 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -36,6 +36,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL +from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -2158,46 +2159,122 @@ async def test_commit_spend_updates_to_db_increments_key_total_spend_alongside_s @pytest.mark.asyncio -async def test_update_database_creates_single_task(): +async def test_shutdown_drain_finishes_the_rollups_update_database_started(): """ - Test that update_database() fires exactly 1 asyncio.create_task() call - (the batched task) instead of the previous 11. + Regression for LIT-6027. The key, team and daily rollups ran in an unreferenced asyncio task, + so the shutdown sequence (flush, drain_logging_tasks, flush) never waited for them and spend + logs landed without matching daily rows. Once update_database returns, drain_logging_tasks + must leave every rollup for that request in its queue. """ - db_writer = DBSpendUpdateWriter() - - # Mock all helpers so nothing real runs + db_writer: Final = DBSpendUpdateWriter() db_writer._insert_spend_log_to_db = AsyncMock() - db_writer._batch_database_updates = AsyncMock() - + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + fake_payload: Final = { + "request_id": "req-lit-6027", + "api_key": "hashed-key", + "user": "test-user", + "team_id": "test-team", + "agent_id": None, + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:00:01", + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "model_group": "gpt-4o-mini", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.0, + "metadata": '{"usage_object": {}}', + } with ( patch("litellm.proxy.proxy_server.disable_spend_logs", False), - patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), - patch("litellm.proxy.db.db_spend_update_writer.asyncio.create_task") as mock_create_task, + patch("litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", return_value=fake_payload), ): await db_writer.update_database( token="test-token", user_id="test-user", - end_user_id="test-end-user", + end_user_id=None, + team_id="test-team", + org_id=None, + kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), start_time=datetime.now(), end_time=datetime.now(), - team_id="test-team", - org_id="test-org", - completion_response=MagicMock(), response_cost=0.1, - kwargs={"model": "gpt-4", "custom_llm_provider": "openai"}, ) + await drain_logging_tasks() - # Exactly 1 create_task call (the batch), not 11 - assert mock_create_task.call_count == 1 + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + assert spend["key_list_transactions"] == {"test-token": 0.1} + assert spend["team_list_transactions"] == {"test-team": 0.1} + daily_user: Final = ( + await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + daily_team: Final = ( + await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + assert daily_user is not None and daily_team is not None + (user_row,) = daily_user.values() + (team_row,) = daily_team.values() + assert (user_row["api_requests"], user_row["spend"]) == (1, 0.1) + assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1) @pytest.mark.asyncio -async def test_batch_database_updates_isolation_on_failure(): +async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue(): + """ + Regression for LIT-6027. A rollup enqueue that is still in flight when the shutdown drain + starts (here: blocked on a full spend queue) must be finished by the time drain_logging_tasks + returns, so the second shutdown flush commits it. + """ + enqueue_started: Final = asyncio.Event() + release_enqueue: Final = asyncio.Event() + + class _BlockedSpendQueue(SpendUpdateQueue): + async def add_update(self, update: SpendUpdateQueueItem): + enqueue_started.set() + await release_enqueue.wait() + await super().add_update(update) + + db_writer: Final = DBSpendUpdateWriter() + db_writer.spend_update_queue = _BlockedSpendQueue() + db_writer._insert_spend_log_to_db = AsyncMock() + with ( + patch("litellm.proxy.proxy_server.disable_spend_logs", True), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), + ): + await db_writer.update_database( + token="t1", + user_id=None, + end_user_id=None, + team_id="team-1", + org_id=None, + kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), + start_time=datetime.now(), + end_time=datetime.now(), + response_cost=0.1, + ) + await enqueue_started.wait() + drain: Final = asyncio.create_task(drain_logging_tasks()) + release_enqueue.set() + await drain + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + + assert spend["key_list_transactions"] == {"t1": 0.1} + assert spend["team_list_transactions"] == {"team-1": 0.1} + + +@pytest.mark.asyncio +async def test_batch_database_updates_isolation_on_failure(caplog: pytest.LogCaptureFixture): """ Test that if one helper inside _batch_database_updates raises, - all other helpers still execute. + all other helpers still execute and the failure is reported at ERROR with the request id. """ db_writer = DBSpendUpdateWriter() @@ -2217,6 +2294,7 @@ async def test_batch_database_updates_isolation_on_failure(): db_writer.add_spend_log_transaction_to_daily_org_transaction = AsyncMock() db_writer.add_spend_log_transaction_to_daily_tag_transaction = AsyncMock() + caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name) await db_writer._batch_database_updates( response_cost=0.1, user_id="u1", @@ -2226,7 +2304,7 @@ async def test_batch_database_updates_isolation_on_failure(): end_user_id="eu1", prisma_client=MagicMock(), litellm_proxy_budget_name="budget", - payload={"key": "value"}, + payload={"request_id": "req-iso", "key": "value"}, ) # _update_key_db raised, but all others should still have been called @@ -2242,6 +2320,48 @@ async def test_batch_database_updates_isolation_on_failure(): db_writer.add_spend_log_transaction_to_daily_team_transaction.assert_awaited_once() db_writer.add_spend_log_transaction_to_daily_org_transaction.assert_awaited_once() db_writer.add_spend_log_transaction_to_daily_tag_transaction.assert_awaited_once() + assert any( + "_update_key_db" in record.getMessage() and "req-iso" in record.getMessage() + for record in caplog.records + if record.levelno >= logging.ERROR + ) + + +@pytest.mark.asyncio +async def test_malformed_metadata_keeps_the_daily_rollup_row_and_is_reported(caplog: pytest.LogCaptureFixture): + """ + Regression for LIT-6027. A spend log whose metadata is not JSON used to vanish from every + daily rollup while the spend log itself persisted, with nothing above debug in the logs. + """ + writer: Final = DBSpendUpdateWriter() + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + payload: Final = { + "request_id": "req-bad-metadata", + "user": "test-user", + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:00:01", + "api_key": "test-key", + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "model_group": "gpt-4o-mini", + "prompt_tokens": 100, + "completion_tokens": 50, + "spend": 0.05, + "metadata": "{not-json", + } + caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name) + + await writer.add_spend_log_transaction_to_daily_user_transaction( # pyright: ignore[reportUnknownMemberType] # prisma_client param is untyped on the writer + payload=payload, prisma_client=prisma + ) + + daily: Final = await writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + (row,) = daily.values() + assert (row["api_requests"], row["spend"], row["prompt_tokens"]) == (1, 0.05, 100) + assert any( + "req-bad-metadata" in record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR + ) @pytest.mark.asyncio @@ -2775,12 +2895,12 @@ async def test_update_database_does_not_deepcopy_on_request_path(): """ Regression for LIT-4088: copy.deepcopy must not run while the caller awaits update_database(). The deepcopy used to isolate the daily-spend helpers is - relocated into the _batch_database_updates background task, and the spend-log + relocated into the _batch_database_updates logging task, and the spend-log insert receives the payload directly (all consumers are read-only). Asserts: - zero copy.deepcopy calls happen on the awaited request path - - the batch background task still hands the daily helpers an isolated copy + - the tracked rollup task still hands the daily helpers an isolated copy (mutating the original after the task ran does not bleed into it) - the spend-log insert receives the payload on the request path with the correct content @@ -2856,16 +2976,13 @@ async def test_update_database_does_not_deepcopy_on_request_path(): response_cost=0.1, ) - # Request path is clean: nothing was deepcopied while the caller awaited. assert len(deepcopy_calls) == 0 # The spend-log insert ran inline on the request path with the real payload. assert captured_spend_log["ref"] is fake_payload assert captured_spend_log["model_at_call"] == "gpt-4" assert fake_payload["spend"] == 0.1 - - # Now let the batch background task run; the deepcopy happens here. - await asyncio.sleep(0) + await drain_logging_tasks() assert len(deepcopy_calls) >= 1 assert len(captured_batch_payloads) == 1 diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 021eb289ad7..4384507e841 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -126,6 +126,51 @@ def test_proxy_decisions_route_returns_answers_and_cost( assert upstream.calls[0].request.headers["authorization"] == "Bearer test-key" +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone")) +@pytest.mark.parametrize("question_count", (128, 129, 154)) +def test_proxy_systemone_preserves_large_question_batches( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + question_count: int, +) -> None: + questions: Final = { + f"q{index}": {"type": "noul", "instructions": f"Question {index}?"} for index in range(question_count) + } + answers: Final = {f"q{index}": {"type": "noul", "noul": index / question_count} for index in range(question_count)} + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={**_RESPONSE, "answers": answers} + ) + + response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions}) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + **_REQUEST, + "model": "pplx-decider-v1-27b", + "questions": questions, + } + assert response.json()["answers"] == answers + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone")) +@pytest.mark.parametrize( + "questions", + ({}, {"": {"type": "noul", "instructions": "Is this a defect?"}}, {"is_defect": {"type": "noul"}}), + ids=("empty", "empty_name", "missing_instructions_or_criteria"), +) +def test_proxy_systemone_rejects_invalid_questions_before_http( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + questions: Mapping[str, object], +) -> None: + response: Final = client.post(endpoint, json={**_REQUEST, "questions": questions}) + + assert response.status_code == 400, response.text + assert len(respx_mock.calls) == 0 + + def test_proxy_decisions_dispatches_typesafe_deployment( client: TestClient, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py b/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py index 2b438a9d370..4b42e6404a3 100644 --- a/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py +++ b/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py @@ -52,7 +52,7 @@ app.include_router(router) client = TestClient(app) BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets" -SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"] +SORTABLE = ["budget_duration", "budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"] def _row(budget_id: str, **overrides: Any) -> dict[str, Any]: @@ -172,22 +172,12 @@ def test_appends_the_tiebreaker_to_an_explicit_sort(query_raw, as_proxy_admin): _get("sort=-max_budget") - assert 'ORDER BY "max_budget" DESC NULLS LAST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0] + assert 'ORDER BY "max_budget" DESC NULLS FIRST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0] -def test_refuses_to_sort_on_budget_duration(query_raw, as_proxy_admin): - """The column holds "7d"/"30d", so a lexicographic ORDER BY would put "30d" - before "7d" and silently mis-order the page.""" - _serve(query_raw, []) - +def test_accepts_duration_sorting(query_raw, as_proxy_admin): response = _get("sort=budget_duration") - - assert response.status_code == 400 - assert response.headers["content-type"].startswith("application/problem+json") - body = response.json() - assert "budget_duration" in body["detail"] - assert body["allowed"] == SORTABLE - query_raw.assert_not_called() + assert response.status_code == 200 def test_the_advertised_sort_fields_are_the_ones_that_work(query_raw, as_proxy_admin): @@ -346,7 +336,7 @@ def test_q_does_not_search_any_other_column(query_raw, as_proxy_admin): def test_is_null_selects_the_unlimited_budgets(query_raw, as_proxy_admin): - """"Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap.""" + """ "Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap.""" _serve(query_raw, [_row("b-unlimited", max_budget=None)]) body = _get("filter[max_budget][is_null]=true").json() diff --git a/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py b/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py index 62863163b8e..4b62cf79094 100644 --- a/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py +++ b/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py @@ -5,14 +5,20 @@ Tests _test_guardrail_definitions and _compute_overall_action directly without needing a running proxy. """ +import json +from pathlib import Path +from unittest.mock import patch + import pytest from fastapi import Request +import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.policy_endpoints.endpoints import ( GuardrailTestResultEntry, _compute_overall_action, _test_guardrail_definitions, + get_policy_templates, list_policies, ) from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -368,3 +374,19 @@ async def test_list_policies_reports_each_loaded_policy_with_its_resolved_guardr assert response == PolicyListResponse(policies=expected_policies, total_count=0) assert list(response.policies) == list(expected_policies) + + +@pytest.mark.parametrize("flag", ["LITELLM_OFFLINE_MODE", "LITELLM_LOCAL_POLICY_TEMPLATES"]) +@pytest.mark.asyncio +async def test_policy_templates_serve_the_bundled_copy_without_fetching(monkeypatch: pytest.MonkeyPatch, flag: str): + monkeypatch.setenv(flag, "true") + bundled = json.loads((Path(litellm.__file__).parent / "policy_templates_backup.json").read_text(encoding="utf-8")) + + with patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + side_effect=AssertionError("policy templates were fetched"), + ) as get_client: + response = await get_policy_templates(request=_policy_list_request(), user_api_key_dict=UserAPIKeyAuth()) + + get_client.assert_not_called() + assert response == bundled diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index 479391e54fe..e1a939273ae 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -9198,58 +9198,6 @@ async def test_saml_callback_blocked_when_admin_ui_disabled(): assert "Admin UI is Disabled" in response.body.decode() -@pytest.mark.asyncio -async def test_saml_callback_enforces_free_sso_user_limit_after_validation(): - """An IdP-initiated assertion must not bypass the >5 free-SSO-user Enterprise gate - that /sso/key/generate enforces; the ACS re-checks it after validating the assertion, - so the entitlement DB query never runs on unvalidated input.""" - from litellm.proxy._types import ProxyException - from litellm.proxy.management_endpoints.types import CustomOpenID - from litellm.proxy.management_endpoints.ui_sso import saml_callback - - call_order: list[str] = [] - - async def _fake_handle_acs(**kwargs): - call_order.append("validate") - return (CustomOpenID( - id="dana@litellm.ai", - email="dana@litellm.ai", - first_name=None, - last_name=None, - display_name="dana", - picture=None, - provider="saml", - team_ids=[], - user_role=None, - ), None) - - async def _fake_count_billable_users(): - call_order.append("count") - return 6 - - async def _stream(): - yield b"SAMLResponse=signed-response" - - request_double = SimpleNamespace(cookies={}, headers={}, stream=_stream) - - with patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}), patch( - "litellm.proxy.proxy_server.premium_user", False - ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch( - "litellm.proxy.proxy_server.master_key", MASTER_KEY - ), patch( - "litellm.proxy.management_endpoints.sso.saml_sso.SAMLAuthHandler.handle_acs", - new=_fake_handle_acs, - ), patch( - "litellm.repositories.user_repository.UserRepository.count_billable_users", - new=AsyncMock(side_effect=_fake_count_billable_users), - ): - with pytest.raises(ProxyException) as exc: - await saml_callback(request_double) - - assert str(exc.value.code) == "403" - assert call_order == ["validate", "count"] - - @pytest.mark.asyncio async def test_cli_poll_key_tolerates_missing_user_row(): """The CLI poll must still mint the JWT when the user lookup raises, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py deleted file mode 100644 index 4daddb89350..00000000000 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ /dev/null @@ -1,657 +0,0 @@ -""" -Tests for AI Usage Chat module. -""" - -import json -from collections.abc import AsyncIterator -from typing import Final -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from fastapi import HTTPException - -from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - TOOL_HANDLERS, - TOOLS_ADMIN, - TOOLS_BASE, - _build_system_prompt, - _summarise_entity_data, - _summarise_usage_data, - stream_usage_ai_chat, -) - - -SAMPLE_AGGREGATED_RESPONSE = { - "results": [ - { - "date": "2025-01-15", - "metrics": { - "spend": 50.25, - "prompt_tokens": 20000, - "completion_tokens": 10000, - "total_tokens": 30000, - "api_requests": 500, - "successful_requests": 480, - "failed_requests": 20, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - }, - "breakdown": { - "models": { - "gpt-4": { - "metrics": { - "spend": 40.0, - "api_requests": 300, - "total_tokens": 25000, - }, - "metadata": {}, - "api_key_breakdown": {}, - }, - }, - "providers": { - "openai": { - "metrics": {"spend": 50.25, "api_requests": 500}, - "metadata": {}, - "api_key_breakdown": {}, - }, - }, - "api_keys": { - "sk-test123": { - "metrics": {"spend": 50.25}, - "metadata": {"key_alias": "Production Key"}, - }, - }, - "model_groups": {}, - "mcp_servers": {}, - "entities": {}, - }, - }, - ], - "metadata": { - "total_spend": 50.25, - "total_api_requests": 500, - "total_successful_requests": 480, - "total_failed_requests": 20, - "total_tokens": 30000, - }, -} - -SAMPLE_TEAM_RESPONSE = { - "results": [ - { - "date": "2025-01-15", - "metrics": {"spend": 100.0, "api_requests": 1000, "total_tokens": 50000}, - "breakdown": { - "entities": { - "team-1": { - "metrics": { - "spend": 60.0, - "api_requests": 600, - "total_tokens": 30000, - }, - "metadata": {"alias": "Engineering"}, - "api_key_breakdown": {}, - }, - "team-2": { - "metrics": { - "spend": 40.0, - "api_requests": 400, - "total_tokens": 20000, - }, - "metadata": {"alias": "Marketing"}, - "api_key_breakdown": {}, - }, - }, - "models": {}, - "providers": {}, - "api_keys": {}, - "model_groups": {}, - "mcp_servers": {}, - }, - }, - ], - "metadata": {"total_spend": 100.0, "total_api_requests": 1000}, -} - - -class TestToolSchemas: - def test_admin_tools_include_all(self): - assert len(TOOLS_ADMIN) == 3 - names = {t["function"]["name"] for t in TOOLS_ADMIN} - assert "get_usage_data" in names - assert "get_team_usage_data" in names - assert "get_tag_usage_data" in names - - def test_base_tools_restricted_to_usage_only(self): - assert len(TOOLS_BASE) == 1 - assert TOOLS_BASE[0]["function"]["name"] == "get_usage_data" - - def test_admin_prompt_mentions_all_tools(self): - prompt = _build_system_prompt(is_admin=True) - assert "get_usage_data" in prompt - assert "get_team_usage_data" in prompt - assert "get_tag_usage_data" in prompt - - def test_non_admin_prompt_only_mentions_usage_tool(self): - prompt = _build_system_prompt(is_admin=False) - assert "get_usage_data" in prompt - assert "get_team_usage_data" not in prompt - assert "get_tag_usage_data" not in prompt - - def test_system_prompt_includes_todays_date(self): - from datetime import date - - prompt = _build_system_prompt(is_admin=True) - assert date.today().isoformat() in prompt - - -class TestSummariseUsageData: - def test_summarise_includes_totals(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "$50.25" in summary - assert "500" in summary - - def test_summarise_includes_models(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "gpt-4" in summary - - def test_summarise_includes_providers(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "openai" in summary - - def test_summarise_handles_empty_data(self): - empty = {"results": [], "metadata": {}} - summary = _summarise_usage_data(empty) - assert "no data" in summary.lower() - - -class TestSummariseEntityData: - def test_team_summary_includes_teams(self): - summary = _summarise_entity_data(SAMPLE_TEAM_RESPONSE, "Team") - assert "Engineering" in summary - assert "Marketing" in summary - assert "$60.0" in summary - assert "$40.0" in summary - - def test_team_summary_empty(self): - empty = {"results": [], "metadata": {}} - summary = _summarise_entity_data(empty, "Team") - assert "No Team usage data" in summary - - -class TestStreamUsageAiChat: - @pytest.mark.asyncio - async def test_stream_emits_status_events(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_123" - mock_tool_call.function.name = "get_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Total spend is $50.25" - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_usage_data", - new_callable=AsyncMock, - ) as mock_fetch, - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - mock_fetch.return_value = SAMPLE_AGGREGATED_RESPONSE - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "What is my total spend?"}], - model="gpt-4o-mini", - user_id="user-123", - is_admin=True, - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - status_events = [e for e in events if e["type"] == "status"] - tool_call_events = [e for e in events if e["type"] == "tool_call"] - chunk_events = [e for e in events if e["type"] == "chunk"] - done_events = [e for e in events if e["type"] == "done"] - - assert len(status_events) >= 1 - assert "Thinking" in status_events[0]["message"] - assert len(tool_call_events) >= 1 - assert tool_call_events[0]["tool_name"] == "get_usage_data" - assert tool_call_events[0]["status"] in ("running", "complete") - assert len(chunk_events) >= 1 - assert len(done_events) == 1 - - @pytest.mark.asyncio - async def test_stream_handles_team_tool(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_team" - mock_tool_call.function.name = "get_team_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_team", - "type": "function", - "function": { - "name": "get_team_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Engineering is the top team." - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_team_usage_data", - new_callable=AsyncMock, - ) as mock_fetch, - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - mock_fetch.return_value = SAMPLE_TEAM_RESPONSE - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "Which team spends the most?"}], - model="gpt-4o-mini", - is_admin=True, - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - chunk_events = [e for e in events if e["type"] == "chunk"] - assert len(chunk_events) >= 1 - assert "Engineering" in chunk_events[0]["content"] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "reason", - [ - "end_date must be on or after start_date", - "start_date and end_date must be valid YYYY-MM-DD dates", - ], - ids=["reversed", "invalid-format"], - ) - @pytest.mark.parametrize( - "tool_name", ["get_usage_data", "get_team_usage_data", "get_tag_usage_data"] - ) - async def test_invalid_tool_dates_are_reported_to_the_model( - self, tool_name: str, reason: str - ) -> None: - fetch: Final = AsyncMock( - side_effect=HTTPException(status_code=400, detail={"error": reason}) - ) - - tool_message, statuses = await _run_one_tool( - tool_name, "2025-01-31", "2025-01-01", fetch - ) - - fetch.assert_awaited_once() - assert tool_message == f"Invalid date range: {reason}" - assert statuses == ["running", "complete"] - - @pytest.mark.asyncio - async def test_tool_dates_the_query_layer_accepts_are_queried_as_sent( - self, - ) -> None: - fetch: Final = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE) - - tool_message, statuses = await _run_one_tool( - "get_usage_data", "2025/01/01", "2025-01-31T00:00:00", fetch - ) - - fetch.assert_awaited_once_with( - start_date="2025/01/01", end_date="2025-01-31T00:00:00" - ) - assert tool_message.startswith("Total Spend: $") - assert statuses == ["running", "complete"] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "error", - [ - HTTPException(status_code=500, detail={"error": "db down"}), - HTTPException(status_code=400, detail="page and page_size must be >= 1"), - ], - ids=["server-error", "other-400"], - ) - async def test_other_tool_errors_stay_a_generic_fetch_error( - self, error: HTTPException - ) -> None: - tool_message, statuses = await _run_one_tool( - "get_team_usage_data", - "2025-01-01", - "2025-01-31", - AsyncMock(side_effect=error), - ) - - assert tool_message == "Error fetching team usage data. Please try again." - assert statuses == ["running", "error"] - - @pytest.mark.asyncio - async def test_stream_handles_error(self): - with patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm: - mock_litellm.acompletion = AsyncMock(side_effect=Exception("LLM error")) - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "test"}], - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - error_events = [e for e in events if e["type"] == "error"] - assert len(error_events) == 1 - assert "internal error" in error_events[0]["message"].lower() - - @pytest.mark.asyncio - async def test_non_admin_enforces_user_id(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_456" - mock_tool_call.function.name = "get_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - "user_id": "other-user", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_456", - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31","user_id":"other-user"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Data." - yield chunk - - mock_fetch = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE) - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch.dict( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS", - { - "get_usage_data": { - "fetch": mock_fetch, - "summarise": _summarise_usage_data, - "label": "global usage data", - } - }, - ), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "Show data"}], - model="gpt-4o-mini", - user_id="my-user-id", - is_admin=False, - ): - events.append(event) - - mock_fetch.assert_called_once_with( - start_date="2025-01-01", - end_date="2025-01-31", - user_id="my-user-id", - ) - - -class TestUsageAiChatServiceAccountGuard: - """ - Security regression: a non-admin caller with user_id=None (service-account - key) must be rejected at the endpoint boundary, before any tool dispatch. - """ - - @pytest.mark.asyncio - async def test_non_admin_with_user_id_none_is_rejected(self): - from fastapi import HTTPException - - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( - ChatMessage, - UsageAIChatRequest, - usage_ai_chat, - ) - - service_account_key = UserAPIKeyAuth( - user_id=None, - user_role=LitellmUserRoles.INTERNAL_USER, - ) - request = MagicMock() - body = UsageAIChatRequest( - messages=[ChatMessage(role="user", content="hi")], - model="gpt-4o-mini", - ) - - with pytest.raises(HTTPException) as exc_info: - await usage_ai_chat( - data=body, - request=request, - user_api_key_dict=service_account_key, - ) - - assert exc_info.value.status_code == 403 - assert "Service-account keys" in str(exc_info.value.detail) - - def test_resolve_fetch_kwargs_tripwire_fires_on_none_user_id(self): - """ - Defense-in-depth: if a future endpoint forgets the entry guard and - a non-admin caller with user_id=None reaches _resolve_fetch_kwargs, - the tripwire must fire rather than issuing an unscoped query. - """ - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - _resolve_fetch_kwargs, - ) - - with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info: - _resolve_fetch_kwargs( - fn_name="get_usage_data", - fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"}, - user_id=None, - is_admin=False, - ) - assert "Endpoint-level guard missing" in str(exc_info.value) - - -class TestUsageAiChatKeepalive: - async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: - import asyncio - - import litellm - from fastapi.responses import StreamingResponse - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( - ChatMessage, - UsageAIChatRequest, - usage_ai_chat, - ) - - monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) - - async def slow_acompletion(**kwargs): - await asyncio.sleep(delay) - response = MagicMock() - response.choices = [MagicMock()] - response.choices[0].message.tool_calls = None - response.choices[0].message.content = "Total spend is $50.25" - return response - - with patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", - new=AsyncMock(side_effect=slow_acompletion), - ): - response = await usage_ai_chat( - data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="gpt-4o-mini"), - request=MagicMock(), - user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), - ) - assert isinstance(response, StreamingResponse) - chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] - return chunks, dict(response.headers) - - @pytest.mark.asyncio - async def test_endpoint_pings_while_the_planning_completion_is_still_running(self, monkeypatch): - chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05) - - assert headers["content-type"].startswith("text/event-stream") - assert headers["cache-control"] == "no-cache" - assert headers["x-accel-buffering"] == "no" - assert chunks[0].startswith(b'data: {"type": "status"') - assert chunks[1] == b": ping\n\n" - assert chunks.count(b": ping\n\n") >= 3 - assert b'"content": "Total spend is $50.25"' in b"".join(chunks) - assert chunks[-1] == b'data: {"type": "done"}\n\n' - - @pytest.mark.asyncio - async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch): - chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15) - - assert b": ping\n\n" not in chunks - assert chunks[0].startswith(b'data: {"type": "status"') - assert chunks[-1] == b'data: {"type": "done"}\n\n' - - -async def _run_one_tool( - tool_name: str, start_date: str, end_date: str, fetch: AsyncMock -) -> tuple[str, list[object]]: - """Runs one admin chat turn whose model calls `tool_name` with these dates; returns the - tool message the follow-up model call carried and the tool statuses streamed.""" - arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date}) - mock_tool_call: Final = MagicMock() - mock_tool_call.id = "call_dates" - mock_tool_call.function.name = tool_name - mock_tool_call.function.arguments = arguments - - mock_first_response: Final = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_dates", - "type": "function", - "function": {"name": tool_name, "arguments": arguments}, - } - ], - } - - async def mock_stream() -> AsyncIterator[MagicMock]: - chunk: Final = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Done." - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch.dict(TOOL_HANDLERS[tool_name], {"fetch": fetch}), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[mock_first_response, mock_stream()] - ) - events: Final[tuple[dict[str, object], ...]] = tuple( - [ - json.loads(event.replace("data: ", "").strip()) - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "How much did we spend?"}], - model="gpt-4o-mini", - is_admin=True, - ) - ] - ) - final_messages: Final = mock_litellm.acompletion.call_args_list[1].kwargs[ - "messages" - ] - - tool_messages: Final = [m for m in final_messages if m["role"] == "tool"] - assert len(tool_messages) == 1, tool_messages - assert tool_messages[0]["tool_call_id"] == "call_dates" - return str(tool_messages[0]["content"]), [ - e["status"] for e in events if e["type"] == "tool_call" - ] diff --git a/tests/unit/proxy/management_helpers/test_team_list_sorting.py b/tests/unit/proxy/management_helpers/test_team_list_sorting.py new file mode 100644 index 00000000000..bc4c77cc6a4 --- /dev/null +++ b/tests/unit/proxy/management_helpers/test_team_list_sorting.py @@ -0,0 +1,85 @@ +import json +import re +import sqlite3 +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from pydantic import JsonValue + +from litellm.proxy.management_helpers.team_list_sorting import UnsupportedTeamFilter, _where_sql, sorted_team_ids + + +@pytest.mark.parametrize( + ("where", "expected"), + ( + ({}, ("a", "b", "c")), + ({"OR": []}, ()), + ({"AND": []}, ("a", "b", "c")), + ({"team_id": "a"}, ("a",)), + ({"team_alias": {"contains": "ALPHA", "mode": "insensitive"}}, ("a", "c")), + ({"team_id": {"startsWith": "b"}}, ("b",)), + ({"organization_id": {"in": ["org-1"]}}, ("a", "b")), + ( + { + "OR": [{"team_id": "b"}, {"team_alias": {"contains": "alpha", "mode": "insensitive"}}], + "AND": [{"OR": [{"organization_id": "org-1"}, {"team_id": "c"}]}], + "team_id": {"in": ["a", "c"]}, + }, + ("a", "c"), + ), + ({"team_id": "a' OR 1=1 --"}, ()), + ), +) +def test_team_filters_preserve_search_and_authorization_intersection( + where: dict[str, JsonValue], expected: tuple[str, ...] +) -> None: + result: Final = _where_sql(where) + assert not isinstance(result, UnsupportedTeamFilter) + predicate, params = result + sqlite_predicate: Final = re.sub( + r"= ANY\((\$\d+)::text\[\]\)", r"IN (SELECT value FROM json_each(\1))", predicate + ).replace(" ILIKE ", " LIKE ") + with sqlite3.connect(":memory:") as db: + db.execute("CREATE TABLE teams (team_id TEXT, team_alias TEXT, organization_id TEXT)") + db.executemany( + "INSERT INTO teams VALUES (?, ?, ?)", + (("a", "Alpha", "org-1"), ("b", "Beta", "org-1"), ("c", "Alpha", "org-2")), + ) + rows: Final = db.execute( + f"SELECT t.team_id FROM teams t WHERE {sqlite_predicate} ORDER BY t.team_id", + { + str(index): json.dumps(value) if isinstance(value, list) else value + for index, value in enumerate(params, 1) + }, + ).fetchall() + assert tuple(row[0] for row in rows) == expected + + +@pytest.mark.parametrize("where", ({"unknown": "x"}, {"team_id": {"not": "x"}}, {"AND": [False]})) +@pytest.mark.asyncio +async def test_unsupported_filter_fails_without_reading_unscoped_teams(where: dict[str, JsonValue]) -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock() + result: Final = await sorted_team_ids(prisma, where=where, sort_by="keys_count", descending=False, skip=0, take=25) + assert isinstance(result, UnsupportedTeamFilter) + prisma.db.query_raw.assert_not_awaited() + + +@pytest.mark.parametrize("sort_by", ("keys_count", "members", "models", "organization_alias")) +@pytest.mark.parametrize("descending", (False, True)) +@pytest.mark.asyncio +async def test_sorted_ids_bind_search_and_pagination_without_interpolating_input( + sort_by: str, descending: bool +) -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + search: Final = "x' OR TRUE --" + result: Final = await sorted_team_ids( + prisma, where={"team_id": search}, sort_by=sort_by, descending=descending, skip=25, take=25 + ) + assert result == () + sql, *bindings = prisma.db.query_raw.await_args.args + assert search not in sql + assert bindings == [search, 25, 25] + assert ("DESC" if descending else "ASC") + " NULLS LAST, t.team_id ASC" in sql diff --git a/tests/unit/proxy/management_helpers/test_user_list_sorting.py b/tests/unit/proxy/management_helpers/test_user_list_sorting.py new file mode 100644 index 00000000000..263c1691b23 --- /dev/null +++ b/tests/unit/proxy/management_helpers/test_user_list_sorting.py @@ -0,0 +1,71 @@ +from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids + + +@pytest.mark.parametrize("sort_by", ("status", "key_count")) +@pytest.mark.parametrize("descending", (False, True)) +@pytest.mark.asyncio +async def test_user_sort_binds_scope_search_and_page_and_excludes_ui_sessions( + sort_by: Literal["status", "key_count"], descending: bool +) -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + search: Final = "x' OR TRUE --" + result: Final = await sorted_user_ids( + prisma, + sort_by=sort_by, + descending=descending, + skip=25, + take=25, + role="internal_user", + user_ids=" a, b ", + user_email=search, + team="team-1", + sso_user_ids="sso-1, sso-2", + organization_ids="org-1, org-2", + search=search, + ) + assert result == () + sql, *bindings = prisma.db.query_raw.await_args.args + assert search not in sql + assert bindings == [ + "internal_user", + ("a", "b"), + search, + "team-1", + False, + ("sso-1", "sso-2"), + ("org-1", "org-2"), + search, + 25, + 25, + ] + ([UI_SESSION_TOKEN_TEAM_ID] if sort_by == "key_count" else []) + assert ("DESC" if descending else "ASC") + ", u.user_id ASC" in sql + assert ("k.team_id IS DISTINCT FROM $11::text" in sql) == (sort_by == "key_count") + + +@pytest.mark.asyncio +async def test_missing_user_filters_preserve_unfiltered_query() -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + result: Final = await sorted_user_ids( + prisma, + sort_by="status", + descending=False, + skip=0, + take=25, + role=None, + user_ids=None, + user_email=None, + team=None, + sso_user_ids=None, + organization_ids=None, + search=None, + ) + assert result == () + assert prisma.db.query_raw.await_args.args[1:] == (None, None, None, None, True, (), (), None, 25, 0) diff --git a/tests/unit/proxy/proxy_server/test_routes_config.py b/tests/unit/proxy/proxy_server/test_routes_config.py index a92986392e9..7d0a7f3f261 100644 --- a/tests/unit/proxy/proxy_server/test_routes_config.py +++ b/tests/unit/proxy/proxy_server/test_routes_config.py @@ -716,6 +716,53 @@ def test_config_list_exposes_config_reload_interval(client, auth_as, mock_prisma assert entry["field_default_value"] == 30 +@pytest.mark.parametrize("enabled", (True, False)) +def test_config_list_exposes_and_saves_wildcard_routes_toggle( + client: TestClient, + auth_as: Callable[..., contextlib.AbstractContextManager[object]], + mock_prisma: MagicMock, + monkeypatch: pytest.MonkeyPatch, + enabled: bool, +) -> None: + from pydantic import TypeAdapter + + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import ConfigList, LitellmUserRoles + + table: Final = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _seed_settings_store(monkeypatch, {}) + monkeypatch.setattr(ps, "general_settings", ps.proxy_config.settings) + entries: Final = TypeAdapter(list[ConfigList]) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + before: Final = client.get("/config/list", params={"config_type": "general_settings"}) + assert before.status_code == 200, before.text + toggle: Final = next( + entry for entry in entries.validate_json(before.content) if entry.field_name == "return_wildcard_routes" + ) + assert toggle.field_type == "Boolean" + assert toggle.field_value is None + assert toggle.editable is True + + saved: Final = client.post( + "/config/field/update", + json={"field_name": "return_wildcard_routes", "field_value": enabled, "config_type": "general_settings"}, + ) + assert saved.status_code == 200, saved.text + persisted: Final = TypeAdapter(dict[str, bool]).validate_json( + table.upsert.call_args.kwargs["data"]["create"]["param_value"] + ) + assert persisted == {"return_wildcard_routes": enabled} + + after: Final = client.get("/config/list", params={"config_type": "general_settings"}) + assert after.status_code == 200, after.text + updated: Final = next( + entry for entry in entries.validate_json(after.content) if entry.field_name == "return_wildcard_routes" + ) + assert updated.field_value is enabled + + def test_config_field_update_accepts_config_reload_interval(client, auth_as, mock_prisma, monkeypatch): """POST /config/field/update accepts proxy_config_reload_interval_seconds and persists it to the DB general_settings row for all pods to pick up.""" diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 8f27365fcc1..cc61b6f2fcf 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1430,6 +1430,30 @@ async def test_get_autorouter_presets_fetches_once_per_process(monkeypatch, rese assert calls == ["https://example.test/presets.json"] +@pytest.mark.asyncio +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_AUTOROUTER_PRESETS", "LITELLM_OFFLINE_MODE"]) +async def test_get_autorouter_presets_local_flags_serve_bundled_without_fetching( + monkeypatch, reset_autorouter_presets_cache, flag +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _load_bundled_autorouter_presets, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + monkeypatch.setenv(flag, "True") + calls = [] + + async def fake_fetch(url): + calls.append(url) + raise AssertionError("no fetch when the bundled catalog is forced") + + presets = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + + assert presets == _load_bundled_autorouter_presets() + assert calls == [] + + @pytest.mark.asyncio async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start( monkeypatch, reset_autorouter_presets_cache diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 23bbe521670..8d53817f51e 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5,19 +5,21 @@ import hashlib import json import re import sqlite3 +from collections.abc import Mapping from datetime import timezone -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import prisma import pytest -from fastapi import HTTPException +from fastapi import HTTPException, Request from fastapi.testclient import TestClient import litellm import litellm.proxy.proxy_server as ps from litellm.proxy.auth.authorization import OwnedRows from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup, load_permitted_log_team_ids +from litellm.proxy.auth.route_checks import RouteChecks def _default_date_range(): @@ -268,6 +270,8 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.proxy._types import ( + LiteLLM_TeamTable, + LiteLLM_UserTable, LitellmUserRoles, Member, ProxyException, @@ -1894,6 +1898,91 @@ async def test_ui_view_spend_logs_user_filter_intersects_permitted_team_scope(cl app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.parametrize( + ("team_role", "selected_user", "expected_ids"), + ( + ("admin", None, ("member-in-team", "second-member-in-team")), + ("admin", "member", ("member-in-team",)), + ("admin", "caller", ()), + ("admin", "outsider", ()), + ("user", None, None), + (None, "member", None), + ), +) +def test_ui_client_usage_enforces_selected_team_and_member_scope( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + team_role: Literal["admin", "user"] | None, + selected_user: str | None, + expected_ids: tuple[str, ...] | None, +) -> None: + logs: Final = tuple( + { + "request_id": request_id, + "user": user_id, + "team_id": team_id, + "api_key": "shared-key", + "spend": 0.1, + "startTime": "2026-01-02T12:00:00+00:00", + "model": "test-model", + } + for request_id, user_id, team_id in ( + ("member-in-team", "member", "selected-team"), + ("same-member-other-team", "member", "other-team"), + ("second-member-in-team", "second-member", "selected-team"), + ("caller-personal", "caller", None), + ("outsider-request", "outsider", "other-team"), + ) + ) + team: Final = LiteLLM_TeamTable( + team_id="selected-team", + members_with_roles=([Member(user_id="caller", role=team_role)] if team_role else []) + + [Member(user_id="member", role="user"), Member(user_id="second-member", role="user")], + team_member_permissions=[], + ) + + async def team_lookup(where: Mapping[str, object]) -> LiteLLM_TeamTable | None: + return team if where.get("team_id") == team.team_id else None + + def filter_logs(where: Mapping[str, object]) -> tuple[dict[str, str | float | None], ...]: + return tuple( + row + for row in logs + if (where.get("team_id") is None or row["team_id"] == where["team_id"]) + and (where.get("user") is None or row["user"] == where["user"]) + ) + + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="caller") + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="caller", user_role=LitellmUserRoles.INTERNAL_USER.value), + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/spend/logs/ui", + request=Request({"type": "http", "method": "GET", "path": "/spend/logs/ui", "query_string": b""}), + valid_token=auth, + request_data={}, + ) + monkeypatch.setattr(ps, "prisma_client", make_ui_spend_logs_mock_prisma(logs, filter_logs, team_lookup)) + monkeypatch.setitem(app.dependency_overrides, ps.user_api_key_auth, lambda: auth) + response: Final = client.get( + "/spend/logs/ui", + params={ + "start_date": "2026-01-01 00:00:00", + "end_date": "2026-01-03 00:00:00", + "team_id": "selected-team", + "page_size": 1000, + "group_by_session": "false", + **({"user_id": selected_user} if selected_user else {}), + }, + headers={"Authorization": "Bearer sk-test"}, + ) + if expected_ids is None: + assert response.status_code == 403 + return + assert response.status_code == 200 + assert tuple(row["request_id"] for row in response.json()["data"]) == expected_ids + assert response.json()["total"] == len(expected_ids) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_pagination(client, monkeypatch): mock_spend_logs = [ @@ -6524,7 +6613,6 @@ def test_ui_view_request_response_reads_from_cold_storage(client, monkeypatch): from litellm.proxy._types import ( LiteLLM_OrganizationMembershipTable, - LiteLLM_UserTable, LiteLLMRoutes, hash_token, ) diff --git a/tests/unit/proxy/telemetry/test_endpoints.py b/tests/unit/proxy/telemetry/test_endpoints.py index 3b5f66da0ed..c962c8df6a5 100644 --- a/tests/unit/proxy/telemetry/test_endpoints.py +++ b/tests/unit/proxy/telemetry/test_endpoints.py @@ -1,13 +1,12 @@ -from dataclasses import dataclass, field from collections.abc import Mapping +from dataclasses import dataclass, field from typing import Final -from typing_extensions import LiteralString - import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from httpx import ASGITransport, AsyncClient, Response +from typing_extensions import LiteralString from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth diff --git a/tests/unit/proxy/telemetry/test_settings.py b/tests/unit/proxy/telemetry/test_settings.py index 0e9c180da4f..b88e6811113 100644 --- a/tests/unit/proxy/telemetry/test_settings.py +++ b/tests/unit/proxy/telemetry/test_settings.py @@ -1,7 +1,6 @@ from typing import Final import pytest - from pydantic import ValidationError from litellm.proxy.telemetry.settings import EnvPolicy, TelemetrySettings, describe_errors, env_policy, load_settings diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index adc4de8c8fa..0749fc12ea9 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -93,6 +93,20 @@ def test_retired_roi_calculator_cannot_be_loaded_or_called( assert response.status_code == 404, f"{method} {path}: {response.text}" +@pytest.mark.parametrize("disable_lazy_routes", ("false", "true")) +@pytest.mark.parametrize("path", ("/lazy/warm/usage_ai", "/usage/ai/chat")) +def test_retired_usage_chat_cannot_be_loaded_or_called( + monkeypatch: pytest.MonkeyPatch, disable_lazy_routes: str, path: str +) -> None: + monkeypatch.setenv(FLAG, disable_lazy_routes) + app: Final = FastAPI() + attach_lazy_features(app) + + with TestClient(app) as client: + response: Final = client.post(path) + assert response.status_code == 404, f"{path}: {response.text}" + + @pytest.mark.parametrize("value", ("1", "true", "TRUE", "yes", "on")) def test_flag_registers_every_feature_at_startup(monkeypatch: pytest.MonkeyPatch, value: str) -> None: monkeypatch.setenv(FLAG, value) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index d9def23bde8..64ff8392c02 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -106,6 +106,13 @@ class TestGetMetadataVariableName: request = self._make_request("/v1/files") assert get_metadata_variable_name(request) == "litellm_metadata" + @pytest.mark.parametrize( + "path", ["/v1beta/interactions", "/interactions", "/v1beta/interactions/v1_abc123", "/interactions/v1_abc123/cancel"] + ) + def test_returns_litellm_metadata_for_interactions_routes(self, path): + request = self._make_request(path) + assert get_metadata_variable_name(request) == "litellm_metadata" + def test_returns_metadata_for_chat_completions(self): request = self._make_request("/chat/completions") assert get_metadata_variable_name(request) == "metadata" @@ -708,6 +715,46 @@ async def test_add_litellm_data_to_request_parses_json_object_string_litellm_met assert updated["litellm_metadata"]["cost_centre"] == "research" +@pytest.mark.asyncio +async def test_interactions_spend_logs_metadata_survives_the_router_metadata_bucket(): + """GH#47841: on /v1beta/interactions the router plants its deployment fields in + `litellm_metadata`. The proxy must put the request's metadata there too: otherwise the + spend log reads only the router's bucket and `spend_logs_metadata` from the + `x-litellm-spend-logs-metadata` header never reaches LiteLLM_SpendLogs.""" + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1beta/interactions" + request_mock.url.path = "/v1beta/interactions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = { + "Content-Type": "application/json", + "x-litellm-spend-logs-metadata": '{"cost_centre": "research"}', + } + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + request_mock.state.parent_otel_span = None + + data = await add_litellm_data_to_request( + data={"model": "gemini-3.1-flash-lite", "input": "Reply OK.", "store": True}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + assert data["litellm_metadata"]["spend_logs_metadata"] == {"cost_centre": "research"} + + # The router then adds its deployment fields to the same bucket, and the spend log + # resolves that bucket. + data["litellm_metadata"].update({"model_group": "gemini-3.1-flash-lite", "deployment": "gemini/gemini-3.1-flash-lite"}) + resolved = get_litellm_metadata_from_kwargs( + {"litellm_params": {"metadata": data.get("metadata") or {}, "litellm_metadata": data["litellm_metadata"]}} + ) + assert resolved["spend_logs_metadata"] == {"cost_centre": "research"} + assert resolved["model_group"] == "gemini-3.1-flash-lite" + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_strip(): """Regression: proxy_server_request['body'] used to be snapshotted before @@ -8064,6 +8111,67 @@ def test_client_anthropic_api_headers_stay_off_openai_compatible_providers(): assert forwarded == {} +def test_client_anthropic_api_headers_stay_off_azure_ai_outside_native_messages(): + forwarded = _headers_forwarded_to({"anthropic-beta": "thinking-binding-controls-2026-08-01"}, "azure_ai") + + assert forwarded == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_client_thinking_binding_beta_reaches_azure_ai_foundry_claude(): + import httpx + + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + data: Final[dict] = {"model": "claude-opus-4-8"} # mutable-ok: the function under test writes into it + add_provider_specific_headers_to_request( + data=data, + headers={ + "content-type": "application/json", + "anthropic-beta": "thinking-binding-controls-2026-08-01", + "anthropic-version": "2026-08-01", + }, + ) + received: Final[list[httpx.Request]] = [] # mutable-ok: the fake Foundry endpoint records what the proxy sent + + def foundry_answers(request: httpx.Request) -> httpx.Response: + received.append(request) + return httpx.Response( + status_code=200, + json={ + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 6}, + }, + request=request, + ) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers)) + try: + await litellm.anthropic.messages.acreate( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + thinking={"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "drop_block"}}, + model="azure_ai/claude-opus-4-8", + api_key="foundry-key", + api_base="https://my-resource.services.ai.azure.com/anthropic", + client=upstream, + provider_specific_header=data["provider_specific_header"], + ) + finally: + await upstream.close() + + assert received[0].headers["anthropic-beta"] == "thinking-binding-controls-2026-08-01" + assert received[0].headers["anthropic-version"] == "2026-08-01" + assert json.loads(received[0].content)["thinking"]["block_binding"] == {"prefix_mismatch_behavior": "drop_block"} + + TYPESAFE_DANGER_ZONE_HEADERS = { "TypeSafe-Danger-Zone-Images": "true", "typesafe-danger-zone-long-context": "true", diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index f8ea1e5d075..a26a64c6fb0 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -3813,3 +3813,47 @@ def test_live_session_preserves_pretty_printed_start_frame( assert captured.route_type == "_arealtime" assert routed_data["model"] == "gpt-live-1" assert routed_data["live_session_start"] == expected_start + + +@pytest.mark.parametrize("scope", [None, "expand"]) +@pytest.mark.parametrize( + "general_settings, wildcard_query, wildcard_listed", + [ + ({}, None, False), + ({"return_wildcard_routes": False}, None, False), + ({"return_wildcard_routes": True}, None, True), + ({"return_wildcard_routes": "true"}, None, False), + ({"return_wildcard_routes": True}, False, False), + ({"return_wildcard_routes": False}, True, True), + ({}, True, True), + ], +) +@pytest.mark.asyncio +async def test_model_list_return_wildcard_routes_defaults_to_general_setting( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, object], + wildcard_query: bool | None, + wildcard_listed: bool, + scope: str | None, +) -> None: + router: Final = litellm.Router( + model_list=[ + {"model_name": "gpt-6.1-sol", "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-fake"}}, + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "sk-fake"}}, + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", None) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_model", None) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", general_settings) + + response: Final = await litellm.proxy.proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_role=LitellmUserRoles.PROXY_ADMIN), + return_wildcard_routes=wildcard_query, + scope=scope, + ) + listed: Final = [model["id"] for model in response["data"]] + + assert "gpt-6.1-sol" in listed, listed + assert ("openai/*" in listed) is wildcard_listed, listed diff --git a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py index 25f1301dd9d..b5a41c31810 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py @@ -12,10 +12,11 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import app from litellm.proxy.ui_crud_endpoints.latest_release_endpoints import ( + CHANGELOG_FEED_URL, + CHANGELOG_URL, LATEST_RELEASE_CACHE_KEY, LATEST_RELEASE_CACHE_TTL_SECONDS, LATEST_RELEASE_UNAVAILABLE_CACHE_TTL_SECONDS, - LATEST_RELEASE_URL, RELEASE_BY_TAG_URL, LatestReleaseInfo, LatestReleaseUnavailable, @@ -49,37 +50,68 @@ SAMPLE_RELEASE: Final = { "html_url": "https://github.com/BerriAI/litellm/releases/tag/v1.102.0", "body": SAMPLE_BODY, } + + +def _changelog_page(folder: str) -> str: + return f"{CHANGELOG_URL}{folder}/{folder.removeprefix('v').replace('.', '-')}" + + +def _feed(*folders: str) -> httpx.Response: + items: Final = "".join( + f"{folder}{_changelog_page(folder)}" for folder in folders + ) + return httpx.Response(200, content=f'{items}'.encode()) + + EXPECTED_INFO: Final = { "version": "1.102.0", "new_features": 2, "bug_fixes": 2, "other_updates": 4, - "release_url": SAMPLE_RELEASE["html_url"], + "release_url": _changelog_page("v1.102.0"), } -class _RecordingClient: - def __init__(self, outcomes: list[httpx.Response | Exception]) -> None: - self._outcomes = outcomes - self.calls: list[tuple[str, float | None]] = [] - - async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: - self.calls.append((url, timeout)) - outcome = self._outcomes[min(len(self.calls) - 1, len(self._outcomes) - 1)] - if isinstance(outcome, Exception): - raise outcome - return outcome - - def _github_response(status: int = 200, payload: object = SAMPLE_RELEASE) -> httpx.Response: return httpx.Response(status, content=json.dumps(payload).encode()) +def _tag_url(tag: str) -> str: + return RELEASE_BY_TAG_URL.format(tag=tag) + + +class _ByUrl: + """Answers each URL from a fixed map; unknown URLs are 404, like a tag with no release.""" + + def __init__(self, routes: dict[str, httpx.Response | Exception]) -> None: + self._routes = routes + self.calls: list[tuple[str, float | None]] = [] + + async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: + self.calls.append((url, timeout)) + outcome = self._routes.get(url, httpx.Response(404, content=b"{}")) + if isinstance(outcome, Exception): + raise outcome + return outcome + + @property + def urls(self) -> list[str]: + return [url for url, _ in self.calls] + + +def _minor_release_routes(**overrides: httpx.Response | Exception) -> dict[str, httpx.Response | Exception]: + return {CHANGELOG_FEED_URL: _feed("v1.102.0"), _tag_url("v1.102.0"): _github_response(), **overrides} + + +def _minor_release(**overrides: httpx.Response | Exception) -> _ByUrl: + return _ByUrl(_minor_release_routes(**overrides)) + + def _fresh_cache() -> InMemoryCache: return InMemoryCache(max_size_in_memory=1, default_ttl=LATEST_RELEASE_CACHE_TTL_SECONDS) -def _override_dependencies(client: _RecordingClient, cache: InMemoryCache, role: LitellmUserRoles) -> None: +def _override_dependencies(client: _ByUrl, cache: InMemoryCache, role: LitellmUserRoles) -> None: async def auth() -> UserAPIKeyAuth: return UserAPIKeyAuth(api_key="sk-test", user_id="test-user", user_role=role) @@ -127,35 +159,68 @@ class TestCountReleaseBullets: class TestGetLatestReleaseInfo: @pytest.mark.asyncio - async def test_fetches_and_parses_github_release(self): - client = _RecordingClient([_github_response()]) + async def test_announces_the_changelog_release_and_links_its_changelog_page(self): + client = _minor_release() result = await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock()) assert isinstance(result, LatestReleaseInfo) assert result.model_dump() == EXPECTED_INFO - assert client.calls == [(LATEST_RELEASE_URL, 5)] + assert client.calls == [(CHANGELOG_FEED_URL, 5), (_tag_url("v1.102.0"), 5)] + + @pytest.mark.asyncio + async def test_github_release_missing_from_the_changelog_is_not_announced(self): + automated: Final = _github_response(payload=_release("v1.103.0", "* feat(proxy): automated release\n")) + result = await _info(_minor_release(**{_tag_url("v1.103.0"): automated})) + assert isinstance(result, LatestReleaseInfo) + assert result.model_dump() == EXPECTED_INFO + + @pytest.mark.asyncio + async def test_highest_stable_version_wins_regardless_of_feed_order(self): + client = _minor_release(**{CHANGELOG_FEED_URL: _feed("v1.101.6", "v1.103.0rc1", "v1.102.0", "v1.100.9")}) + result = await _info(client) + assert isinstance(result, LatestReleaseInfo) + assert result.model_dump() == EXPECTED_INFO + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "feed", + [ + httpx.Response(503, content=_feed("v1.102.0").content), + httpx.Response(200, content=b"not a feed"), + _feed("v1.103.0rc1", "v1.103.0-dev.1"), + httpx.Response( + 200, + content=f"https://example.com/?{_changelog_page('v9.0.0')}" + "https://example.com/release_notes/v9.0.0/x".encode(), + ), + ], + ids=["server_error", "not_xml", "only_prereleases", "foreign_links"], + ) + async def test_bad_changelog_feeds_are_unavailable(self, feed: httpx.Response): + client = _minor_release(**{CHANGELOG_FEED_URL: feed}) + result = await _info(client) + assert isinstance(result, LatestReleaseUnavailable) + assert client.urls == [CHANGELOG_FEED_URL] @pytest.mark.asyncio async def test_second_call_within_ttl_does_not_refetch(self): - client = _RecordingClient([_github_response()]) + client = _minor_release() cache = _fresh_cache() fetch_lock = asyncio.Lock() first = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) second = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) assert first == second - assert len(client.calls) == 1 + assert len(client.calls) == 2 @pytest.mark.asyncio async def test_success_is_cached_for_the_full_ttl(self): cache = _fresh_cache() - await get_latest_release_info( - client=_RecordingClient([_github_response()]), cache=cache, fetch_lock=asyncio.Lock() - ) + await get_latest_release_info(client=_minor_release(), cache=cache, fetch_lock=asyncio.Lock()) remaining = await cache.async_get_ttl(LATEST_RELEASE_CACHE_KEY) - time.time() assert LATEST_RELEASE_CACHE_TTL_SECONDS - 5 < remaining <= LATEST_RELEASE_CACHE_TTL_SECONDS @pytest.mark.asyncio async def test_failure_is_cached_briefly_so_github_is_not_hammered(self): - client = _RecordingClient([httpx.ConnectError("boom")]) + client = _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")}) cache = _fresh_cache() fetch_lock = asyncio.Lock() first = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) @@ -172,30 +237,29 @@ class TestGetLatestReleaseInfo: @pytest.mark.parametrize( "response", [ + _github_response(status=404, payload={"message": "Not Found"}), _github_response(status=403, payload={"message": "rate limited"}), _github_response(status=500, payload={}), _github_response(payload={"tag_name": "v1.0.0"}), httpx.Response(200, content=b"not json"), + httpx.ConnectError("boom"), ], - ids=["rate_limited", "server_error", "missing_fields", "not_json"], + ids=["no_release_for_tag", "rate_limited", "server_error", "missing_fields", "not_json", "unreachable"], ) - async def test_bad_github_responses_are_unavailable(self, response: httpx.Response): - result = await get_latest_release_info( - client=_RecordingClient([response]), cache=_fresh_cache(), fetch_lock=asyncio.Lock() - ) + async def test_bad_github_responses_are_unavailable(self, response: httpx.Response | Exception): + result = await _info(_minor_release(**{_tag_url("v1.102.0"): response})) assert isinstance(result, LatestReleaseUnavailable) @pytest.mark.asyncio async def test_concurrent_misses_share_one_fetch(self): event = asyncio.Event() - class _BlockingClient(_RecordingClient): + class _BlockingClient(_ByUrl): async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: - self.calls.append((url, timeout)) await event.wait() - return _github_response() + return await super().get(url, timeout=timeout) - client = _BlockingClient([]) + client = _BlockingClient(_minor_release_routes()) cache = _fresh_cache() fetch_lock = asyncio.Lock() tasks = [ @@ -208,19 +272,19 @@ class TestGetLatestReleaseInfo: results = await asyncio.gather(*tasks) expected: Final = LatestReleaseInfo.model_validate(EXPECTED_INFO) assert results == [expected] * 5 - assert len(client.calls) == 1 + assert len(client.calls) == 2 @pytest.mark.asyncio async def test_failure_under_lock_is_also_coalesced(self): event = asyncio.Event() - class _FailingBlockingClient(_RecordingClient): + class _FailingBlockingClient(_ByUrl): async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: self.calls.append((url, timeout)) await event.wait() raise httpx.ConnectError("boom") - client = _FailingBlockingClient([]) + client = _FailingBlockingClient({}) cache = _fresh_cache() fetch_lock = asyncio.Lock() tasks = [ @@ -245,40 +309,23 @@ def _release(tag: str, body: str | None, *, prerelease: bool = False) -> dict[st } -def _tag_url(tag: str) -> str: - return RELEASE_BY_TAG_URL.format(tag=tag) - - -class _GitHubByUrl: - """Answers each GitHub URL from a fixed map; unknown URLs are 404, like a tag with no release.""" - - def __init__(self, routes: dict[str, httpx.Response | Exception]) -> None: - self._routes = routes - self.urls: list[str] = [] - - async def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: - self.urls.append(url) - outcome = self._routes.get(url, httpx.Response(404, content=b"{}")) - if isinstance(outcome, Exception): - raise outcome - return outcome - - PATCH_RELEASE: Final = _release("v1.102.2", "* fix(proxy): second backport by @k in https://x/pull/10\n") FIRST_PATCH: Final = _release("v1.102.1", "* fix(proxy): first backport by @k in https://x/pull/9\n") -def _minor_line(**overrides: httpx.Response | Exception) -> dict[str, httpx.Response | Exception]: - routes: dict[str, httpx.Response | Exception] = { - LATEST_RELEASE_URL: _github_response(payload=PATCH_RELEASE), - _tag_url("v1.102.0"): _github_response(payload=SAMPLE_RELEASE), - _tag_url("v1.102.1"): _github_response(payload=FIRST_PATCH), - } - routes.update({_tag_url(tag): outcome for tag, outcome in overrides.items()}) - return routes +def _minor_line(**overrides: httpx.Response | Exception) -> _ByUrl: + return _ByUrl( + { + CHANGELOG_FEED_URL: _feed("v1.102.2", "v1.101.6"), + _tag_url("v1.102.2"): _github_response(payload=PATCH_RELEASE), + _tag_url("v1.102.0"): _github_response(payload=SAMPLE_RELEASE), + _tag_url("v1.102.1"): _github_response(payload=FIRST_PATCH), + **{_tag_url(tag): outcome for tag, outcome in overrides.items()}, + } + ) -async def _info(client: _GitHubByUrl) -> LatestReleaseInfo | LatestReleaseUnavailable: +async def _info(client: _ByUrl) -> LatestReleaseInfo | LatestReleaseUnavailable: return await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock()) @@ -289,7 +336,7 @@ class TestPatchReleaseAggregatesMinorLine: @pytest.mark.asyncio async def test_patch_release_counts_include_every_earlier_patch_on_its_line(self): - client = _GitHubByUrl(_minor_line()) + client = _minor_line() result = await _info(client) assert isinstance(result, LatestReleaseInfo) assert result.model_dump() == { @@ -297,23 +344,11 @@ class TestPatchReleaseAggregatesMinorLine: "new_features": 2, "bug_fixes": 4, "other_updates": 4, - "release_url": PATCH_RELEASE["html_url"], + "release_url": _changelog_page("v1.102.2"), } - assert sorted(client.urls) == sorted([LATEST_RELEASE_URL, _tag_url("v1.102.0"), _tag_url("v1.102.1")]) - - @pytest.mark.asyncio - async def test_minor_release_makes_a_single_request(self): - client = _RecordingClient([_github_response()]) - await get_latest_release_info(client=client, cache=_fresh_cache(), fetch_lock=asyncio.Lock()) - assert client.calls == [(LATEST_RELEASE_URL, 5)] - - @pytest.mark.asyncio - async def test_non_semver_latest_tag_counts_only_its_own_notes(self): - client = _GitHubByUrl({LATEST_RELEASE_URL: _github_response(payload=_release("v1.102.1-hotfix", "* fix: x\n"))}) - result = await _info(client) - assert isinstance(result, LatestReleaseInfo) - assert (result.new_features, result.bug_fixes, result.other_updates) == (0, 1, 0) - assert client.urls == [LATEST_RELEASE_URL] + assert sorted(client.urls) == sorted( + [CHANGELOG_FEED_URL, _tag_url("v1.102.2"), _tag_url("v1.102.0"), _tag_url("v1.102.1")] + ) @pytest.mark.asyncio @pytest.mark.parametrize( @@ -326,15 +361,13 @@ class TestPatchReleaseAggregatesMinorLine: ids=["never_published", "prerelease", "null_body"], ) async def test_missing_prerelease_or_empty_patch_adds_nothing(self, skipped: httpx.Response): - result = await _info(_GitHubByUrl(_minor_line(**{"v1.102.1": skipped}))) + result = await _info(_minor_line(**{"v1.102.1": skipped})) assert isinstance(result, LatestReleaseInfo) assert (result.new_features, result.bug_fixes, result.other_updates) == (2, 3, 4) @pytest.mark.asyncio async def test_latest_release_with_null_body_still_counts_its_line(self): - routes = _minor_line() - routes[LATEST_RELEASE_URL] = _github_response(payload=_release("v1.102.2", None)) - result = await _info(_GitHubByUrl(routes)) + result = await _info(_minor_line(**{"v1.102.2": _github_response(payload=_release("v1.102.2", None))})) assert isinstance(result, LatestReleaseInfo) assert (result.new_features, result.bug_fixes, result.other_updates) == (2, 3, 4) @@ -349,31 +382,51 @@ class TestPatchReleaseAggregatesMinorLine: ids=["unreachable", "rate_limited", "not_json"], ) async def test_earlier_patch_failure_is_unavailable(self, outcome: httpx.Response | Exception): - result = await _info(_GitHubByUrl(_minor_line(**{"v1.102.0": outcome}))) + result = await _info(_minor_line(**{"v1.102.0": outcome})) assert isinstance(result, LatestReleaseUnavailable) class TestLatestReleaseInfoEndpoint: def test_returns_release_stats_for_authenticated_user(self, http_client): - _override_dependencies(_RecordingClient([_github_response()]), _fresh_cache(), LitellmUserRoles.INTERNAL_USER) + _override_dependencies(_minor_release(), _fresh_cache(), LitellmUserRoles.INTERNAL_USER) response = http_client.get("/get/latest_release_info") assert response.status_code == 200 assert response.json() == EXPECTED_INFO - def test_returns_null_when_github_is_unreachable(self, http_client): + def test_returns_null_when_the_changelog_is_unreachable(self, http_client): _override_dependencies( - _RecordingClient([httpx.ConnectError("boom")]), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN + _ByUrl({CHANGELOG_FEED_URL: httpx.ConnectError("boom")}), _fresh_cache(), LitellmUserRoles.PROXY_ADMIN ) response = http_client.get("/get/latest_release_info") assert response.status_code == 200 assert response.json() is None + @pytest.mark.parametrize("name", ["LITELLM_DISABLE_LATEST_RELEASE_CHECK", "LITELLM_OFFLINE_MODE"]) + @pytest.mark.parametrize( + ("flag", "expected_body", "expected_urls"), + [ + ("true", None, []), + (" TRUE ", None, []), + ("false", EXPECTED_INFO, [CHANGELOG_FEED_URL, _tag_url("v1.102.0")]), + ], + ) + def test_disable_flag_skips_every_outbound_request( + self, http_client, monkeypatch, name, flag, expected_body, expected_urls + ): + monkeypatch.setenv(name, flag) + client = _minor_release() + _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) + response = http_client.get("/get/latest_release_info") + assert response.status_code == 200 + assert response.json() == expected_body + assert client.urls == expected_urls + def test_repeated_requests_reuse_cache(self, http_client): - client = _RecordingClient([_github_response()]) + client = _minor_release() _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO assert http_client.get("/get/latest_release_info").json() == EXPECTED_INFO - assert len(client.calls) == 1 + assert len(client.calls) == 2 def test_rejects_unauthenticated_requests(self, monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.master_key", MASTER_KEY) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py index 747e8da8043..e335b08f040 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py @@ -12,15 +12,118 @@ import asyncio import hashlib import json import logging +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from litellm._service_logger import ServiceTypes +from litellm.models.user import LiteLLM_UserTable from litellm.proxy.db.log_db_metrics import record_db_io from litellm.proxy.utils import PrismaClient +from litellm.repositories.user_repository import free_sso_user_creation + + +class _SSOAdmissionUsers: + def __init__(self, count: int) -> None: + self.rows: Mapping[str, LiteLLM_UserTable] = { + f"user-{index}": LiteLLM_UserTable(user_id=f"user-{index}") + for index in range(count) + } + + async def find_unique( + self, where: Mapping[str, object] + ) -> LiteLLM_UserTable | None: + return self.rows.get(str(where["user_id"])) + + async def count(self, where: Mapping[str, object] | None = None) -> int: + if where is None: + return len(self.rows) + return sum((row.metadata or {}).get("scim_active") is False for row in self.rows.values()) + + async def upsert( + self, where: Mapping[str, object], data: Mapping[str, object] + ) -> LiteLLM_UserTable: + existing: Final = await self.find_unique(where) + if existing is not None: + return existing + created: Final = LiteLLM_UserTable.model_validate(data["create"]) + self.rows = {**self.rows, created.user_id: created} + return created + + +class _SSOAdmissionTransaction(AbstractAsyncContextManager["_SSOAdmissionTransaction"]): + def __init__(self, users: _SSOAdmissionUsers) -> None: + self.litellm_usertable: Final = users + + async def __aenter__(self) -> _SSOAdmissionTransaction: + return self + + async def __aexit__(self, *exc_info: object) -> None: + return None + + async def execute_raw(self, query: str) -> int: + return 0 + + async def query_raw(self, query: str) -> object: + return ({"locked": 1},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("existing_count", "scope_id", "user_id", "enabled", "child_task", "refused"), + [ + (4, "new-user", "new-user", True, False, False), + (5, "new-user", "new-user", True, False, True), + (6, "user-0", "user-0", True, False, False), + (5, "sso-user", "new-user", True, False, False), + (5, "new-user", "new-user", True, True, False), + (5, None, "new-user", True, False, True), + (5, None, "new-user", False, False, False), + ], +) +async def test_user_insert_enforces_free_sso_admission_only_for_its_owner( + prisma_client: PrismaClient, + monkeypatch: pytest.MonkeyPatch, + existing_count: int, + scope_id: str | None, + user_id: str, + enabled: bool, + child_task: bool, + refused: bool, +) -> None: + users: Final = _SSOAdmissionUsers(existing_count) + prisma_client.db.litellm_usertable = users + prisma_client.db.tx.return_value = _SSOAdmissionTransaction(users) + monkeypatch.setattr("asyncio.sleep", AsyncMock(return_value=None)) + original: Final = users.rows.get(user_id) + + async def attempt() -> None: + if refused: + with pytest.raises(HTTPException) as error: + await prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + assert error.value.status_code == 403 + assert user_id not in users.rows + elif child_task: + result: Final = await asyncio.create_task( + prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + ) + assert result is users.rows[user_id] + else: + inserted: Final = await prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + assert inserted is users.rows[user_id] + + with free_sso_user_creation(scope_id, enabled=enabled): + for _ in range(2): + await attempt() + + assert len(users.rows) == existing_count + int(not refused and original is None) + if original is not None: + assert users.rows[user_id] is original @pytest.mark.asyncio diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 436367bd82d..bf571c74e22 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -4,10 +4,12 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st in expected_responses_api_request/. """ +import asyncio import copy import json import logging -from collections.abc import Awaitable, Callable +import math +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path from importlib import import_module from typing import Final @@ -16,10 +18,12 @@ from unittest.mock import AsyncMock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.llms.openai import ResponsesAPIResponse from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, @@ -332,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up( def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]: - return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False) + return tuple( + level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False + ) def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]: @@ -622,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -644,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs( from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -670,7 +678,9 @@ _ORIGINAL_WS_INPUT = [ @pytest.mark.asyncio @pytest.mark.parametrize("nested", [False, True]) -async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket +async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame( + nested: bool, +): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket from unittest.mock import MagicMock from litellm.responses.main import _aresponses_websocket @@ -681,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame ) with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -710,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}' with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -967,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped.""" monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com") injected_client = AsyncHTTPHandler() - mock_post = AsyncMock( - return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200) - ) + mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)) injected_client.post = mock_post await litellm.aresponses( @@ -1231,3 +1241,227 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) -> assert [event.response.output[0].content[0].text for event in response if event.type == "response.completed"] == [ "Done." ] + + +async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse: + response: Final = ( + litellm.responses(model=model, input="hi", **pricing) + if sync_mode + else await litellm.aresponses(model=model, input="hi", **pricing) + ) + assert isinstance(response, ResponsesAPIResponse) + return response + + +def _response_cost(response: ResponsesAPIResponse) -> float: + hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params") + return hidden_params["response_cost"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map")) +async def test_responses_bills_per_call_custom_pricing_from_usage( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1])) + ) + pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + + response: Final = await _invoke_responses(sync_mode, model, pricing) + + assert response.usage is not None + expected: Final = ( + response.usage.input_tokens * pricing["input_cost_per_token"] + + response.usage.output_tokens * pricing["output_cost_per_token"] + ) + assert math.isclose(_response_cost(response), expected) + + +@pytest.mark.asyncio +async def test_concurrent_responses_calls_keep_their_own_per_call_prices( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + model: Final = "openai/gpt-5.4-nano" + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + cost_map_row_before: Final = copy.deepcopy(model_cost.get(model)) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano")) + ) + rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5)) + + responses: Final = await asyncio.gather( + *( + _invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate}) + for in_rate, out_rate in rates + ) + ) + + for response, (in_rate, out_rate) in zip(responses, rates, strict=True): + assert response.usage is not None + assert math.isclose( + _response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate + ) + assert model_cost.get(model) == cost_map_row_before + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("api_base", "explicit_header", "expected_session"), + [ + ("https://opencode.ai/inference/go/openai/v1", None, "conversation-123"), + ("https://opencode.ai/inference/go/openai/v1", "caller-override", "caller-override"), + ("https://other.example/v1", None, None), + ], +) +@pytest.mark.asyncio +async def test_responses_opencode_session_header( + asynchronous: bool, stream: bool, api_base: str, explicit_header: str | None, expected_session: str | None +) -> None: + received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + payload: Final = _minimal_responses_api_payload("resp_test", "gpt-6-luna") + + def respond(request: httpx.Request) -> httpx.Response: + received.put_nowait(request) + return ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text="".join( + f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': payload})}\n\n" + for kind in ("response.created", "response.completed") + ), + ) + if stream + else httpx.Response(200, json=payload) + ) + + args: Final = { + "model": "openai/gpt-6-luna", + "input": "hello", + "api_base": api_base, + "api_key": "test-key", + "litellm_session_id": "conversation-123", + "stream": stream, + "extra_headers": {"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"}, + } + if asynchronous: + async_client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + try: + response: Final = await litellm.aresponses(**args, client=async_client) + if stream: + events: Final = tuple([event.type async for event in response]) + assert "response.completed" in events + else: + assert response.output[0].content[0].text == "Done." + finally: + await async_client.close() + else: + sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + result: Final = litellm.responses(**args, client=sync_client) + if stream: + assert "response.completed" in tuple(event.type for event in result) + else: + assert result.output[0].content[0].text == "Done." + finally: + sync_client.close() + sent: Final = received.get_nowait() + assert str(sent.url) == f"{api_base}/responses" + assert sent.headers.get("x-opencode-session") == expected_session + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello", "stream": stream} + assert received.empty() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("source", ["OPENAI_BASE_URL", "OPENAI_API_BASE", "litellm.api_base"]) +@pytest.mark.parametrize("explicit_base", [None, "https://other.example/v1"]) +@pytest.mark.asyncio +async def test_responses_opencode_session_uses_effective_destination( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + asynchronous: bool, + source: str, + explicit_base: str | None, +) -> None: + default_base: Final = "https://opencode.ai/inference/go/openai/v1" + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", default_base if source == "litellm.api_base" else None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + if source != "litellm.api_base": + monkeypatch.setenv(source, default_base) + endpoint: Final = respx_mock.post(f"{explicit_base or default_base}/responses").respond( + json=_minimal_responses_api_payload("resp_test", "gpt-6-luna") + ) + args: Final = { + "model": "openai/gpt-6-luna", + "input": "hello", + "api_base": explicit_base, + "api_key": "test-key", + "litellm_session_id": "conversation-123", + } + response: Final = await litellm.aresponses(**args) if asynchronous else litellm.responses(**args) + assert response.output[0].content[0].text == "Done." + assert endpoint.call_count == 1 + sent: Final = endpoint.calls[0].request + assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"} + + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.asyncio +async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "nano", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates}, + "model_info": {"id": "nano-deployment"}, + } + ] + ) + payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano") + usage: Final = _JSON_OBJECT.validate_python(payload["usage"]) + cached_tokens: Final = 8 + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}}, + ) + ) + + response: Final = await router.aresponses(model="nano", input="hi") + + assert isinstance(response, ResponsesAPIResponse) + assert response.usage is not None + uncached_input: Final = response.usage.input_tokens - cached_tokens + full_price: Final = ( + response.usage.input_tokens * rates["input_cost_per_token"] + + response.usage.output_tokens * rates["output_cost_per_token"] + ) + floor: Final = ( + uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"] + ) + assert floor < _response_cost(response) < full_price diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 726128386d5..78722458cf3 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -116,6 +116,9 @@ def _wrap_nested_level(value: object, depth: int) -> object: (responses_detectors.provider_affinity, {"provider_affinity_header": ""}, True), (responses_detectors.provider_affinity, {"provider_affinity_header": None, "session_id": "session-1"}, False), (responses_detectors.provider_affinity, {}, False), + (responses_detectors.provider_affinity, {"api_base": "https://opencode.ai/inference/go/openai/v1"}, True), + (messages_detectors.provider_affinity, {"api_base": "https://opencode.ai/inference/go/anthropic/v1"}, True), + (messages_detectors.provider_affinity, {"api_base": "https://other.example/v1"}, False), (detectors.streaming, {"stream": True}, True), (detectors.streaming, {"stream": False}, False), (detectors.timeout_variants, {"request_timeout": 5}, True), diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index 2a85122d91c..deba7fe9587 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -17,7 +17,9 @@ import pytest import litellm from litellm.anthropic_beta_headers_manager import ( + GetAnthropicBetaHeadersConfig, filter_and_transform_beta_headers, + get_beta_headers_config, update_headers_with_filtered_beta, update_request_with_filtered_beta, ) @@ -429,13 +431,13 @@ class TestAnthropicBetaHeadersFiltering: assert filtered == ["fine-grained-tool-streaming-2025-05-14"] @pytest.mark.parametrize( - "provider", ["anthropic", "bedrock", "bedrock_converse", "vertex_ai", "databricks"] + "provider", ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai", "databricks"] ) def test_thinking_binding_controls_forwarded(self, provider): """`thinking.block_binding` (preserved thinking, Claude Fable 5.1) is only accepted alongside thinking-binding-controls-2026-08-01. The body field is forwarded untouched, so stripping the header (previously unknown, hence - dropped) makes Bedrock and Vertex reject the request with + dropped) makes Bedrock, Vertex and Azure AI Foundry reject the request with "thinking.adaptive.block_binding: Extra inputs are not permitted".""" filtered = filter_and_transform_beta_headers( beta_headers=["thinking-binding-controls-2026-08-01"], @@ -558,3 +560,13 @@ class TestAnthropicBetaHeadersFiltering: headers = {"anthropic-version": "2023-06-01"} assert update_headers_with_filtered_beta(headers, "bedrock_mantle") == {"anthropic-version": "2023-06-01"} + + +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "LITELLM_OFFLINE_MODE"]) +def test_local_flags_load_bundled_beta_headers_without_fetching(monkeypatch, flag): + monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) + monkeypatch.setenv(flag, "True") + with patch("litellm.anthropic_beta_headers_manager.httpx.get") as remote_get: + config = get_beta_headers_config(url="https://example.test/anthropic_beta_headers_config.json") + remote_get.assert_not_called() + assert config == GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() diff --git a/tests/unit/test_elevenlabs_tts_pricing.py b/tests/unit/test_elevenlabs_tts_pricing.py new file mode 100644 index 00000000000..16f726b9d6c --- /dev/null +++ b/tests/unit/test_elevenlabs_tts_pricing.py @@ -0,0 +1,49 @@ +"""ElevenLabs text-to-speech prices match the published API rates. + +https://elevenlabs.io/pricing/api lists "v3" and "v2 Multilingual" at $0.08 and +"Flash / Turbo" at $0.04, each "Price per 1K characters", and says API usage is +billed in US dollars, not credits. The map had v3 and Multilingual v2 at a +Scale-plan credit estimate ($0.18 per 1K characters) and no Flash rows. +""" + +import json +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +PRICES = { + "elevenlabs/eleven_v3": 8e-05, + "elevenlabs/eleven_multilingual_v2": 8e-05, + "elevenlabs/eleven_flash_v2_5": 4e-05, + "elevenlabs/eleven_flash_v2": 4e-05, +} + + +@pytest.fixture(autouse=True) +def local_model_cost_map(monkeypatch): + """Never depend on the published cost map having these rows yet.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model, per_character", PRICES.items()) +def test_metadata_and_backup(model, per_character): + main = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text()) + backup = json.loads((REPO_ROOT / "litellm/model_prices_and_context_window_backup.json").read_text()) + assert main[model] == backup[model] + info = litellm.get_model_info(model) + assert info["litellm_provider"] == "elevenlabs" + assert info["mode"] == "audio_speech" + assert info["input_cost_per_character"] == per_character + + +@pytest.mark.parametrize("model, per_character", PRICES.items()) +def test_speech_cost_is_charged_per_character(model, per_character): + cost = litellm.completion_cost(model=model, call_type="speech", prompt="x" * 1000) + assert cost == pytest.approx(per_character * 1000) diff --git a/tests/unit/test_get_blog_posts.py b/tests/unit/test_get_blog_posts.py index 499dd9bcf42..d1de75a4500 100644 --- a/tests/unit/test_get_blog_posts.py +++ b/tests/unit/test_get_blog_posts.py @@ -190,8 +190,9 @@ def test_get_blog_posts_ttl_expired_refetches(): assert len(posts) == 2 -def test_get_blog_posts_local_env_var_skips_remote(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_BLOG_POSTS", "true") +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_BLOG_POSTS", "LITELLM_OFFLINE_MODE"]) +def test_get_blog_posts_local_env_var_skips_remote(monkeypatch, flag): + monkeypatch.setenv(flag, "true") with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get") as mock_get: posts = get_blog_posts(url=litellm.blog_posts_url) mock_get.assert_not_called() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index c3d3575b8b3..70b268bf2a2 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -20,7 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError, AsyncOpenAI +from openai import APITimeoutError, AsyncOpenAI, OpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion from pydantic import TypeAdapter @@ -95,6 +95,142 @@ def preserve_litellm_completion_state(monkeypatch: pytest.MonkeyPatch) -> None: WHITE_PNG: Final = (Path(__file__).parents[1] / "white_100x100.png").read_bytes() +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("client_base", "explicit_header", "expected_session"), + [ + ("https://opencode.ai/zen/go/v1", None, "conversation-123"), + ("https://opencode.ai/zen/go/v1", "caller-override", "caller-override"), + ("https://other.example/v1", None, None), + ], +) +@pytest.mark.asyncio +async def test_completion_sends_session_only_to_the_actual_opencode_destination( + asynchronous: bool, + stream: bool, + client_base: str, + explicit_header: str | None, + expected_session: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://opencode.ai/zen/go/v1") + monkeypatch.setenv("OPENAI_API_BASE", "https://opencode.ai/zen/go/v1") + monkeypatch.setattr(litellm, "api_base", "https://opencode.ai/zen/go/v1") + requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + requests.put_nowait(request) + payload: Final = { + "id": "chatcmpl-test", + "object": "chat.completion.chunk" if stream else "chat.completion", + "created": 1, + "model": "session-test", + "choices": [ + { + "index": 0, + "delta" if stream else "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + } + return ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(payload)}\n\ndata: [DONE]\n\n", + ) + if stream + else httpx.Response(200, json=payload) + ) + + args: Final = { + "model": "openai/session-test", + "api_base": "https://opencode.ai/zen/go/v1", + "messages": [{"role": "user", "content": "hello"}], + "litellm_session_id": "conversation-123", + "extra_headers": {"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"}, + "stream": stream, + } + if asynchronous: + async with AsyncOpenAI( + api_key="test-key", + base_url=client_base, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) as async_client: + response: Final = await acompletion(**args, client=async_client) + content: Final = ( + "".join([chunk.choices[0].delta.content or "" async for chunk in response]) + if stream + else response.choices[0].message.content + ) + assert content == "OK" + else: + with OpenAI( + api_key="test-key", base_url=client_base, http_client=httpx.Client(transport=httpx.MockTransport(respond)) + ) as sync_client: + result: Final = completion(**args, client=sync_client) + text: Final = ( + "".join(chunk.choices[0].delta.content or "" for chunk in result) + if stream + else result.choices[0].message.content + ) + assert text == "OK" + sent: Final = requests.get_nowait() + assert str(sent.url) == f"{client_base}/chat/completions" + assert sent.headers.get("x-opencode-session") == expected_session + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == { + "model": "session-test", + "messages": args["messages"], + **({"stream": True} if stream else {}), + } + assert requests.empty() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("source", ["OPENAI_BASE_URL", "OPENAI_API_BASE", "litellm.api_base"]) +@pytest.mark.parametrize("explicit_base", [None, "https://other.example/v1"]) +@pytest.mark.asyncio +async def test_completion_opencode_session_uses_effective_destination( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + asynchronous: bool, + source: str, + explicit_base: str | None, +) -> None: + default_base: Final = "https://opencode.ai/inference/go/openai/v1" + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", default_base if source == "litellm.api_base" else None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + if source != "litellm.api_base": + monkeypatch.setenv(source, default_base) + endpoint: Final = respx_mock.post(f"{explicit_base or default_base}/chat/completions").respond( + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "session-test", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"}, "finish_reason": "stop"}], + } + ) + args: Final = { + "model": "openai/session-test", + "messages": [{"role": "user", "content": "hello"}], + "api_base": explicit_base, + "api_key": "test-key", + "litellm_session_id": "conversation-123", + } + response: Final = await litellm.acompletion(**args) if asynchronous else litellm.completion(**args) + assert response.choices[0].message.content == "OK" + assert endpoint.call_count == 1 + sent: Final = endpoint.calls[0].request + assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == {"model": "session-test", "messages": args["messages"]} + + @pytest.fixture def openai_api_response(): mock_response_data = { diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 701b79d0ba9..9b9fec4c9a1 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -791,3 +791,26 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa sibling = rows["volcengine/doubao-seed-2-1-pro-260628"] assert isinstance(sibling, dict) assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")) + + +def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]: + return tuple( + key + for key, row in prices.items() + if "gpt-oss" in key and isinstance(row, dict) and str(row.get("litellm_provider", "")).startswith("bedrock") + ) + + +@pytest.mark.parametrize("path", [PRICES_PATH, BACKUP_PRICES_PATH], ids=["main", "backup"]) +def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblings(path: Path) -> None: + prices: Final = json.loads(path.read_text()) + rows: Final = _bedrock_gpt_oss_rows(prices) + assert rows, "no Bedrock gpt-oss rows found" + reference: Final = prices["openai.gpt-oss-safeguard-120b"]["supports_vision"] + undeclared: Final = tuple(key for key in rows if prices[key].get("supports_vision") is not reference) + assert undeclared == (), f"modality routing only skips rows that declare supports_vision: {undeclared}" + + +def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: + assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True + assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9cc1bf6ce79..f06f9e08f7e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -29871,3 +29871,105 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s wrapped.adopt_fallback_headers(fallback) assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens + + +_IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} +_IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}] +_IMAGE_RESPONSES_INPUT = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo="}, + ], + } +] +_VISION_LEG: Final[dict[str, object]] = {"litellm_params": {"model": "openai/gpt-5.6-terra"}, "model_info": {"id": "vision"}} +_DECLARED_TEXT_LEG: Final[dict[str, object]] = { + "litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, + "model_info": {"id": "declared-text", "supports_vision": False}, +} + + +def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": "mixed", **deployment} for deployment in deployments], + routing_strategy=routing_strategy, + enable_pre_call_checks=True, + ) + + +def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]: + return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)] + + +def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"] + + +def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [ + "vision", + "declared-text", + ] + + +def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids( + router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT} + ) == ["vision"] + + +def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only(): + other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}} + router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg) + assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [ + "declared-text", + "declared-text-2", + ] + + +def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests(): + custom_leg = { + "litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"}, + "model_info": {"id": "custom"}, + } + router = _pre_call_router(_VISION_LEG, custom_leg) + assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"] + + +def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model(): + catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}} + azure_text_leg = { + "litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"}, + "model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"}, + } + router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg) + assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"] + + +@pytest.mark.parametrize( + ("deployment", "expected"), + [ + (_DECLARED_TEXT_LEG, True), + ({"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, "model_info": {"supports_vision": True}}, False), + ({"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {}}, True), + ({"litellm_params": {"model": "azure/my-deploy"}, "model_info": {"base_model": "azure/gpt-audio"}}, True), + ({"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct"}, "model_info": {}}, False), + ({"litellm_params": {"model": "no-such-model-anywhere"}, "model_info": {}}, False), + ({"litellm_params": {}, "model_info": {}}, False), + ], + ids=["override-false", "override-true", "cost-map-false", "base-model-false", "undeclared", "unresolvable", "no-params"], +) +def test_deployment_declares_no_vision_reads_the_override_then_the_cost_map(deployment: dict[str, object], expected: bool): + assert litellm.Router._deployment_declares_no_vision(deployment, "mixed") is expected + + +def test_request_carries_image_sees_chat_parts_and_responses_input_items(): + assert litellm.Router._request_carries_image(_IMAGE_CHAT, {}) is True + assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True + assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False + assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cdff03af3b4..ad069aa987b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -90,7 +90,6 @@ from litellm.utils import ( get_llm_provider, get_optional_params_embeddings, ProviderConfigManager, - TextCompletionStreamWrapper, _check_provider_match, get_potential_model_names, _is_litellm_router_call, @@ -2528,41 +2527,6 @@ def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing(): assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.000001 -def test_reasoning_content_preserved_in_text_completion_wrapper(): - """Ensure reasoning_content is copied from delta to text_choices.""" - chunk = ModelResponseStream( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="Some answer text", - role="assistant", - reasoning_content="Here's my chain of thought...", - ), - ) - ], - ) - - wrapper = TextCompletionStreamWrapper( - completion_stream=None, # Not used in convert_to_text_completion_object - model="test-model", - stream_options=None, - ) - - transformed = wrapper.convert_to_text_completion_object(chunk) - - assert "choices" in transformed - assert len(transformed["choices"]) == 1 - choice = transformed["choices"][0] - assert choice["text"] == "Some answer text" - assert choice["reasoning_content"] == "Here's my chain of thought..." - - def test_anthropic_claude_4_invoke_chat_provider_config(): """Test that the Anthropic Claude 4 Invoke chat provider config is correct.""" from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( diff --git a/tests/unit/text_completions/__init__.py b/tests/unit/text_completions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/text_completions/test_streaming_iterator.py b/tests/unit/text_completions/test_streaming_iterator.py new file mode 100644 index 00000000000..b7edffc0753 --- /dev/null +++ b/tests/unit/text_completions/test_streaming_iterator.py @@ -0,0 +1,144 @@ +from collections.abc import AsyncIterator, Iterator +from typing import Final + +import pytest + +from litellm.exceptions import APIConnectionError +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper +from litellm.types.utils import Choices, Delta, ModelResponseStream, StreamingChoices, Usage + + +def test_reasoning_content_preserved_in_text_completion_wrapper(): + """Ensure reasoning_content is copied from delta to text_choices.""" + chunk = ModelResponseStream( + id="test-id", + created=1234567890, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + content="Some answer text", + role="assistant", + reasoning_content="Here's my chain of thought...", + ), + ) + ], + ) + + wrapper = TextCompletionStreamWrapper( + completion_stream=None, # Not used in convert_to_text_completion_object + model="test-model", + stream_options=None, + ) + + transformed = wrapper.convert_to_text_completion_object(chunk) + + assert "choices" in transformed + assert len(transformed["choices"]) == 1 + choice = transformed["choices"][0] + assert choice["text"] == "Some answer text" + assert choice["reasoning_content"] == "Here's my chain of thought..." + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_stream", (False, True)) +@pytest.mark.parametrize("empty_choices", (False, True)) +@pytest.mark.parametrize("stream_options", (None, {}, {"include_usage": False}, {"include_usage": True})) +@pytest.mark.parametrize("usage", (None, Usage(prompt_tokens=7, completion_tokens=2, total_tokens=9), Usage())) +async def test_text_completion_stream_usage_is_opt_in( + async_stream: bool, empty_choices: bool, stream_options: dict[str, bool] | None, usage: Usage | None +) -> None: + chunk: Final = ModelResponseStream( + id="test-id", + created=1234567890, + model="test-model", + choices=[] if empty_choices else [StreamingChoices(index=0, delta=Delta(content="Hello"))], + usage=usage, + ) + + async def chunks() -> AsyncIterator[ModelResponseStream]: + yield chunk + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks() if async_stream else iter((chunk,)), + model="test-model", + stream_options=stream_options, + ) + responses: Final = [response async for response in wrapper] if async_stream else list(wrapper) + expected_usage: Final = ( + {"usage": usage.model_dump(exclude_none=True)} + if stream_options and stream_options.get("include_usage") is True and usage is not None + else {} + ) + assert [response.model_dump(exclude_none=True) for response in responses] == [ + { + "id": "test-id", + "object": "text_completion", + "created": 1234567890, + "model": "test-model", + "choices": [] if empty_choices else [{"index": 0, "text": "Hello"}], + **expected_usage, + } + ] + + +def test_text_completion_conversion_rejects_non_streaming_choices() -> None: + chunk: Final = Choices() + wrapper: Final = TextCompletionStreamWrapper(completion_stream=None, model="test-model") + + with pytest.raises(Exception, match="Error occurred converting to text completion object") as error: + wrapper.convert_to_text_completion_object(chunk) + + assert str(chunk) in str(error.value) + + +@pytest.mark.parametrize("chunk", (None, "None")) +def test_sync_text_completion_stream_rejects_invalid_chunks(chunk: str | None) -> None: + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=iter((chunk,)), model="test-model", custom_llm_provider="openai" + ) + + with pytest.raises(APIConnectionError, match="OpenAIException") as error: + next(wrapper) + + assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunk", (None, "None")) +async def test_async_text_completion_stream_rejects_invalid_chunks(chunk: str | None) -> None: + async def chunks() -> AsyncIterator[str | None]: + yield chunk + + wrapper: Final = TextCompletionStreamWrapper(completion_stream=chunks(), model="test-model") + + with pytest.raises(Exception, match=r"^$") as error: + await anext(wrapper) + + assert not isinstance(error.value, StopAsyncIteration) + + +def test_sync_text_completion_stream_maps_upstream_failure_after_content() -> None: + chunk: Final = ModelResponseStream( + id="test-id", + created=1234567890, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta(content="Hello"))], + ) + + def chunks() -> Iterator[ModelResponseStream]: + yield chunk + raise RuntimeError("upstream disconnected") + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks(), model="test-model", custom_llm_provider="openai" + ) + assert next(wrapper).choices[0].text == "Hello" + + with pytest.raises(APIConnectionError, match="upstream disconnected") as error: + next(wrapper) + + assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 4802aa057c6..38d4c435e9a 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1098,14 +1098,6 @@ "count": 1 } }, - "src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx": { - "no-nested-ternary": { - "count": 1 - }, - "react-hooks/immutability": { - "count": 1 - } - }, "src/app/(dashboard)/usage/_components/components/UsagePageView.tsx": { "react-hooks/purity": { "count": 1 @@ -1767,16 +1759,16 @@ "count": 1 }, "max-params": { - "count": 15 + "count": 14 }, "no-nested-ternary": { "count": 5 }, "no-restricted-syntax": { - "count": 140 + "count": 139 }, "prefer-const": { - "count": 29 + "count": 28 } }, "src/components/object_permissions_view.tsx": { @@ -1798,9 +1790,6 @@ }, "prefer-const": { "count": 2 - }, - "react-hooks/set-state-in-effect": { - "count": 4 } }, "src/components/organization/organization_view.tsx": { diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 944455721cd..8704dc55c38 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -28,7 +28,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.8", + "next": "16.4.1", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "6.49.0", @@ -54,7 +54,7 @@ "tailwind-merge": "3.4.0", "usehooks-ts": "3.1.1", "uuid": "14.0.0", - "zod": "4.6.5" + "zod": "4.6.6" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -69,8 +69,8 @@ "@types/react-copy-to-clipboard": "5.0.7", "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "eslint": "9.39.2", "eslint-config-next": "16.3.3", "eslint-config-prettier": "10.1.8", @@ -87,7 +87,7 @@ "typescript": "5.9.3", "typescript-eslint": "8.60.1", "vite": "7.3.7", - "vitest": "4.1.11" + "vitest": "4.1.12" }, "engines": { "node": ">=24.14.1", @@ -2153,9 +2153,9 @@ } }, "node_modules/@next/env": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.8.tgz", - "integrity": "sha512-Al9zqHVV7TJv0eFuOU4U7Lvv74PTih4Ch63sk2xCIpSTkE3udFnaOcnzP2lQVymiL7yS9Cj2iClUXlR3EQ5sEw==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.4.1.tgz", + "integrity": "sha512-5sUUrrSvs/2/2LnLoesybMGx12EtPB46fdZ6RhAw0XDGSPxU2LFyQBB7yuEiOFFJPKmBY6+q2jTr1/AMbnp0/g==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -2170,9 +2170,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.8.tgz", - "integrity": "sha512-2JPRMh2nmQG5CiL7cXGL9AGwnPWJQ//cTtAUCT+w511QHk79SYz3LGv/pc5X643B/WEO0rvu3Yww0hqwt3kgeA==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.4.1.tgz", + "integrity": "sha512-U+hQ5Gj3fpzf4fveTIF3dpUFYhaSW8vHhdAnbiIomzXzg7boreYFgBqprepXFtW4sw+aTttiNmg0gew/iFNhbg==", "cpu": [ "arm64" ], @@ -2186,9 +2186,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.8.tgz", - "integrity": "sha512-GZtCCOBKJ4leVIT/Th0llWKhD1ca92lzbQiS5R5ON9QkoiFnilFsebDae1JU2a3HWoKMEmEZWGs1AGLavVM72Q==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.4.1.tgz", + "integrity": "sha512-/hqC3V1XLUjhMASf/WridKdO9FXoEgN3HBWmtKbj00MPgnltutfraWJ3bYTpCRIgEIpbjJyOVLpYgAhh/BkdcQ==", "cpu": [ "x64" ], @@ -2202,9 +2202,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.8.tgz", - "integrity": "sha512-O659ygeQYqneJ1fBKMpFxIFqYkYswu8IAS1OCKK/4f3ZgJJm1dRz4fVJZRi/kLLWjnBKnebOePA4WNv+sV1pVA==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.4.1.tgz", + "integrity": "sha512-GHzncHbcaS9g95kcvTNtKQW3aCLoR+xc3Jm7qrkpk1R+D2BjMHIPyvQGRPKCdMX95BsH/421jBUK06q9tEwzjw==", "cpu": [ "arm64" ], @@ -2221,9 +2221,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.8.tgz", - "integrity": "sha512-dSjKSyWpzxoO1d3DIZZcP4XJcNaKeLmxQMFOiYl5vuBRMmweIqnAhty8tAmRsvTss779cK1FtYnDMj40e4TQlg==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.4.1.tgz", + "integrity": "sha512-1SuJyDxW/PR9dHJat/ZRgcYLhuSZ+1psP4DlKb2tKIuXbjygqpLrJWRSq+onZQBv8LcmU67EIrmb6c3XTaJLjw==", "cpu": [ "arm64" ], @@ -2240,9 +2240,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.8.tgz", - "integrity": "sha512-lbqOuz3RPRcv+o9msNsJw5x4+Y1ZwPTs6vmL6DCf7i0fZfvng/F59wyeDwqHIvV0mK//RBy/jJkZ+nCKsSMXjQ==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.4.1.tgz", + "integrity": "sha512-Istng1hMGp9TGO4lUqeKLd70MPN/OOkucVTP2Nmj+gLkwBgmPqUw8CJYBCC7nbl/B/23bBCcjCI5Hsh6hazC1A==", "cpu": [ "x64" ], @@ -2259,9 +2259,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.8.tgz", - "integrity": "sha512-+316WswI8ScVgZeUd+1KGaXkHhaYQzCjvH/05TZSpJ8zBizb1a4G7DtO7F12jcBIqMOtsz9ji1t48fmKtzqsGA==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.4.1.tgz", + "integrity": "sha512-aSC9wMNiZRRFVpPkPWBzRwLQxIGm/CiQcXyFF1XNcgodb+uMxdk7aSSGGpDXCbnvZdf4itjHxpOd5h7i7zaiYw==", "cpu": [ "x64" ], @@ -2278,9 +2278,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.8.tgz", - "integrity": "sha512-ji0gd4kMYUxO+1fJBIbiBVRCjzG/lloiyCccnlebvb1ZJ5qXCPZqYg4Jl1DrrixnWNMKylzgpmMWx0yNDYXlzw==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.4.1.tgz", + "integrity": "sha512-t/myvwFDlypMUjRvbBf8qLPyuaB0iXYYdBtwMt6DM6f+60wtGr4bSYholdPqNeV5VUXZhrpMwfza829kmhS3iA==", "cpu": [ "arm64" ], @@ -2294,9 +2294,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.8.tgz", - "integrity": "sha512-WcTlaKt/TWkh5kUjdJcUmB1XgZ+1c6fz4Y9fDHL73YNSdGaUWjceeWrrlwF0nv19iABYWC4iAq1oX1w4Bn0vfg==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.4.1.tgz", + "integrity": "sha512-6ICqPo9awCkrtYNPhwkx1uU7rlFWrZRrIboDHS2LU+jv088UBYQtmJ25h6pCp27/+FZjF6BQ4Te4ErugyMeuBg==", "cpu": [ "x64" ], @@ -4490,14 +4490,14 @@ ] }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.11.tgz", - "integrity": "sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.12.tgz", + "integrity": "sha512-6S7/iVknKglLAxMFxQGNppBI0iqJS2NgM6FBxavitj7KaDq/vywOWGf6dGu7F2L1Qy0G6S3uTp7aGshHrzK8JA==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -4511,8 +4511,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.11", - "vitest": "4.1.11" + "@vitest/browser": "4.1.12", + "vitest": "4.1.12" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -4521,16 +4521,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", - "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.12.tgz", + "integrity": "sha512-vFlTt+YEu9WBpNjdGayNmlMVzrWPi1+sv853/7t2FU7hpO5gAFBKHzQmkCq5Xjpr+evegF2994LQT3hF3mRlqQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -4539,13 +4539,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", - "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.12.tgz", + "integrity": "sha512-fKShNpKZzupWZuY0z7egydvL0sqW4INEZBEExMBMAa6izlENEVHwoXgdaTCaBZ5fwlr4xf2o1wXC83n9RpO+NA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.11", + "@vitest/spy": "4.1.12", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -4566,9 +4566,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", - "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.12.tgz", + "integrity": "sha512-XW9O8+Mxx4VpCtkYdMJLZQQwsVryP9wk7cPKDxtXFfwSak6yRh5+2fep/Y0XF/ElBvJHq8bcYdi6aeuai0opZA==", "dev": true, "license": "MIT", "dependencies": { @@ -4579,13 +4579,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", - "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.12.tgz", + "integrity": "sha512-NKlVKCr5QnCe8jijdMnmkpHHU+WTqiLFR6G0pHji9uRjXK9xJl1k1nTw6F3pOOj2PDcsDt2GKQMI7wIsaT491w==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "pathe": "^2.0.3" }, "funding": { @@ -4593,14 +4593,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", - "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.12.tgz", + "integrity": "sha512-ko1y1Qgb/slzQ9mr+b89Cuqud8WG1TKtsPD8r7ZUBdvVOGt85Af2HwCqHujrhwJq95hs/9RM7fcAPg57o1vlzw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/pretty-format": "4.1.12", + "@vitest/utils": "4.1.12", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -4609,9 +4609,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", - "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.12.tgz", + "integrity": "sha512-RGq9x4JNGNTZHEJxvN0oSiJjeSCExm0n2gZKKJZHY9cX5AVA6M7VJILBT3ZJ/tJ/0J9uWT7fETV9/btoi//vSg==", "dev": true, "license": "MIT", "funding": { @@ -4619,13 +4619,13 @@ } }, "node_modules/@vitest/ui": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-4.1.11.tgz", - "integrity": "sha512-r/rwyKoev21mWdRGSEkZOqkQ2BYy68mwjihg9M90nNRbf4NGrgzZ4cj6JNCEwlOGJkbKeMgsjlykvwKUbRr7gw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-4.1.12.tgz", + "integrity": "sha512-eYUTHtsp52Fu6YGUw9T9173bba7YUr9j/Xj9CdhUaOb8AppLKcknHUaiZQeqJ8MiJ8C2VyKiZuUGSKdvPIKQKA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "fflate": "^0.8.2", "flatted": "^3.4.2", "pathe": "^2.0.3", @@ -4637,17 +4637,17 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": "4.1.11" + "vitest": "4.1.12" } }, "node_modules/@vitest/utils": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", - "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.12.tgz", + "integrity": "sha512-Q3awXrwOfEV+qc3WmO5ONyE0yO7Q1gQ9/oyHxGLZ/pl7N3NO/ZztqQwKLuwMkV+CA1JYSIeO3gkHnRDIo4AVaA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", + "@vitest/pretty-format": "4.1.12", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -9710,12 +9710,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.8.tgz", - "integrity": "sha512-U7QEZaTini6wKrb8A8hqLLqYQyCetegKjCpJOyxk642vWoMoU1x5PyZCJFvgYgiptA8xc5j/9xYlZFO7w9Sjmw==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/next/-/next-16.4.1.tgz", + "integrity": "sha512-uc657V62E34fOI6W5NtG9IzwsPcms0Z60grz2MJGtglgD+pTFnNt4Yw2PzaU622tfX4vhrDgLZzicE2YlhI20Q==", "license": "MIT", "dependencies": { - "@next/env": "16.3.8", + "@next/env": "16.4.1", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -9729,15 +9729,15 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.8", - "@next/swc-darwin-x64": "16.3.8", - "@next/swc-linux-arm64-gnu": "16.3.8", - "@next/swc-linux-arm64-musl": "16.3.8", - "@next/swc-linux-x64-gnu": "16.3.8", - "@next/swc-linux-x64-musl": "16.3.8", - "@next/swc-win32-arm64-msvc": "16.3.8", - "@next/swc-win32-x64-msvc": "16.3.8", - "sharp": "^0.35.4" + "@next/swc-darwin-arm64": "16.4.1", + "@next/swc-darwin-x64": "16.4.1", + "@next/swc-linux-arm64-gnu": "16.4.1", + "@next/swc-linux-arm64-musl": "16.4.1", + "@next/swc-linux-x64-gnu": "16.4.1", + "@next/swc-linux-x64-musl": "16.4.1", + "@next/swc-win32-arm64-msvc": "16.4.1", + "@next/swc-win32-x64-msvc": "16.4.1", + "sharp": "^0.35.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -12512,19 +12512,19 @@ } }, "node_modules/vitest": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", - "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.12.tgz", + "integrity": "sha512-5eWdz2yb3ajxWF3C8C4TjkXl7FGm3n2E/VlazALrQHzK7nYMSGFnCZto0bbPNl77AwkOZVLk0YnBAejUu0NiWA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.11", - "@vitest/mocker": "4.1.11", - "@vitest/pretty-format": "4.1.11", - "@vitest/runner": "4.1.11", - "@vitest/snapshot": "4.1.11", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/expect": "4.1.12", + "@vitest/mocker": "4.1.12", + "@vitest/pretty-format": "4.1.12", + "@vitest/runner": "4.1.12", + "@vitest/snapshot": "4.1.12", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -12552,12 +12552,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.11", - "@vitest/browser-preview": "4.1.11", - "@vitest/browser-webdriverio": "4.1.11", - "@vitest/coverage-istanbul": "4.1.11", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/browser-playwright": "4.1.12", + "@vitest/browser-preview": "4.1.12", + "@vitest/browser-webdriverio": "^4.1.11", + "@vitest/coverage-istanbul": "4.1.12", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -12876,9 +12876,9 @@ } }, "node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index f2a1a5e4e7f..b1b8fde44ee 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -45,7 +45,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.8", + "next": "16.4.1", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "6.49.0", @@ -71,7 +71,7 @@ "tailwind-merge": "3.4.0", "usehooks-ts": "3.1.1", "uuid": "14.0.0", - "zod": "4.6.5" + "zod": "4.6.6" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -86,8 +86,8 @@ "@types/react-copy-to-clipboard": "5.0.7", "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "eslint": "9.39.2", "eslint-config-next": "16.3.3", "eslint-config-prettier": "10.1.8", @@ -104,9 +104,13 @@ "typescript": "5.9.3", "typescript-eslint": "8.60.1", "vite": "7.3.7", - "vitest": "4.1.11" + "vitest": "4.1.12" }, "overrides": { + "@litellm/lens-ui": { + "next": "$next" + }, + "zod": "$zod", "prismjs": "1.30.0", "js-yaml": "4.3.2", "brace-expansion": "5.0.12", diff --git a/ui/litellm-dashboard/public/assets/aria-agent.png b/ui/litellm-dashboard/public/assets/aria-agent.png new file mode 100644 index 00000000000..870773933e3 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/aria-agent.png differ diff --git a/ui/litellm-dashboard/public/assets/logos/claude.svg b/ui/litellm-dashboard/public/assets/logos/claude.svg new file mode 100644 index 00000000000..ede30652b25 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/claude.svg @@ -0,0 +1,27 @@ + +Claude diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx index ae898645de4..b4fb7586e6d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx @@ -134,13 +134,16 @@ describe("BudgetTable", () => { } }); - it("should not make the reset column sortable", async () => { + it("should request duration sorting from the server", async () => { const user = userEvent.setup(); - renderWithProviders(); + const onSortingChange = vi.fn(); + renderWithProviders(); await showColumn(user, "budget_duration"); const headers = screen.getAllByRole("columnheader").map((header) => header.textContent); expect(headers).toContain("Reset"); - expect(screen.queryByTestId("sort-header-budget_duration")).not.toBeInTheDocument(); + await user.click(screen.getByTestId("sort-header-budget_duration")); + expect(onSortingChange).toHaveBeenCalled(); + expect(onSortingChange.mock.calls[0][0]([])).toEqual([{ id: "budget_duration", desc: false }]); }); it("should ask the list for a new sort when a sortable header is clicked", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index fb894322208..3ec291c1158 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -139,8 +139,7 @@ export const getBudgetTableColumns = ({ accessorKey: "budget_duration", filterFn: serverFilter, meta: { title: "Reset" }, - // "7d"/"30d" sort lexicographically, not chronologically, so the route does not offer it. - enableSorting: false, + sortDescFirst: false, header: ({ column }) => , size: 110, cell: ({ row }) => , diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx index b957393be64..1891a580088 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx @@ -49,6 +49,9 @@ const requestUrl = (input: RequestInfo | URL): string => { }; const fakeFetch = (input: RequestInfo | URL) => { + if (requestUrl(input).endsWith("/models")) { + return Promise.resolve(Response.json({ data: [{ id: "claude-sonnet-5-5" }] })); + } if (requestUrl(input).includes("/model_group/info")) { return Promise.resolve(Response.json({ data: [{ model_group: "claude-sonnet-5-5", mode: "chat" }] })); } @@ -141,7 +144,7 @@ describe("HomePage", () => { expect(screen.queryByRole("link", { name: /create new virtual key/i })).not.toBeInTheDocument(); }); - it("offers LiteAdmin on Home inside the dashboard frame", async () => { + it("should offer Aria Agent on Home inside the dashboard frame", async () => { render( @@ -149,7 +152,7 @@ describe("HomePage", () => { , ); - const entry = screen.getByRole("region", { name: "Ask LiteAdmin" }); + const entry = screen.getByRole("region", { name: "Ask Aria Agent" }); expect(await within(entry).findByPlaceholderText("What would you like to do?")).toBeInTheDocument(); expect(within(entry).getByRole("button", { name: "Send message" })).toBeDisabled(); expect(await screen.findByText("$501.74")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx index 7ae7e976651..ad08ff48a5b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx @@ -4,8 +4,6 @@ import PassThroughSettings from "@/components/PassThroughSettings/PassThroughSet import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function PassThroughPanel() { - const { accessToken, userRole, userId: userID, premiumUser } = useAuthorized(); - return ( - - ); + const { accessToken, userRole, userId: userID } = useAuthorized(); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx index 839eb406200..724a72a77f3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx @@ -66,6 +66,85 @@ const baseProps = { }; describe("OrganizationsTable", () => { + it.each(["max_budget", "tpm_limit", "rpm_limit"])( + "sorts %s numerically with Unlimited as infinity", + async (field) => { + const user = userEvent.setup(); + const organizations = [20, 3, null, 100, 0].map((value, index) => + makeOrganization({ + organization_id: `org-${index}`, + organization_alias: `Value ${value}`, + litellm_budget_table: { [field]: value }, + }), + ); + renderWithProviders(, { + searchParams: { sort_by: field, sort_order: "asc", page_size: "25" }, + }); + const names = () => + screen + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getByText(/^Value /).textContent); + expect(names()).toEqual(["Value 0", "Value 3", "Value 20", "Value 100", "Value null"]); + if (field === "max_budget") { + await user.click(screen.getByTestId("sort-header-max_budget")); + } else { + await user.click(screen.getByRole("button", { name: "Sort options for TPM or RPM" })); + await user.click( + await screen.findByRole("menuitem", { name: `${field === "tpm_limit" ? "TPM" : "RPM"} descending` }), + ); + } + await waitFor(() => expect(names()).toEqual(["Value null", "Value 100", "Value 20", "Value 3", "Value 0"])); + }, + ); + + it.each(["asc", "desc"])("groups both All Proxy Models representations when sorting %s", (direction) => { + const organizations = [ + makeOrganization({ organization_id: "org-empty", organization_alias: "Name Empty", models: [] }), + makeOrganization({ organization_id: "org-z", organization_alias: "Name Z", models: ["z"] }), + makeOrganization({ + organization_id: "org-wildcard", + organization_alias: "Name Wildcard", + models: ["all-proxy-models"], + }), + makeOrganization({ organization_id: "org-a", organization_alias: "Name A", models: ["a"] }), + ]; + renderWithProviders(, { + searchParams: { sort_by: "models", sort_order: direction }, + }); + const names = screen + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getByText(/^Name /).textContent); + const models = direction === "asc" ? ["Name A", "Name Z"] : ["Name Z", "Name A"]; + expect(names).toEqual([...models, "Name Empty", "Name Wildcard"]); + expect(screen.getAllByText("All Proxy Models")).toHaveLength(2); + }); + + it.each(["models", "members"])("sorts %s by its displayed data", async (field) => { + const user = userEvent.setup(); + const organizations = [ + { organization_id: "org-zed", organization_alias: "Zed", models: ["z"], members: [] }, + { + organization_id: "org-ace", + organization_alias: "Ace", + models: ["a"], + members: [{ user_id: "1" }, { user_id: "2" }], + }, + { + organization_id: "org-mid", + organization_alias: "Mid", + models: ["m"], + members: [{ user_id: "3" }], + }, + ].map(makeOrganization); + renderWithProviders(); + await user.click(screen.getByTestId(`sort-header-${field}`)); + await waitFor(() => expect(bodyRowAliases()).toEqual(["Ace", "Mid", "Zed"])); + await user.click(screen.getByTestId(`sort-header-${field}`)); + await waitFor(() => expect(bodyRowAliases()).toEqual(["Zed", "Mid", "Ace"])); + }); + it("renders every column header", () => { renderWithProviders(); for (const header of [ @@ -75,7 +154,6 @@ describe("OrganizationsTable", () => { "Spend (USD)", "Budget (USD)", "Models", - "TPM / RPM Limits", "Members", ]) { expect(screen.getByText(header)).toBeInTheDocument(); @@ -87,7 +165,7 @@ describe("OrganizationsTable", () => { for (const header of ["Spend (USD)", "Budget (USD)", "Members"]) { expect(screen.getByRole("columnheader", { name: header })).toHaveClass("text-right"); } - for (const header of ["Organization Name", "TPM / RPM Limits"]) { + for (const header of ["Organization Name"]) { expect(screen.getByRole("columnheader", { name: header })).not.toHaveClass("text-right"); } }); @@ -270,7 +348,7 @@ describe("OrganizationsTable URL state", () => { it("falls back to sorting by creation date for a ?sort_by= column that cannot be sorted", () => { renderWithProviders(, { - searchParams: "?sort_by=members&sort_order=asc", + searchParams: "?sort_by=actions&sort_order=asc", }); expect(bodyRowAliases()).toEqual(["Zed", "Mid", "Ace"]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx index a9ac0b7e699..df1c475ebb1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx @@ -62,6 +62,7 @@ const OrganizationsTable: React.FC = ({ onPaginationChange={onPaginationChange} columns={columns} getRowId={(organization, index) => organization.organization_id || String(index)} + defaultColumnVisibility={{ tpm_limit: false, rpm_limit: false }} sortingMode="client" sorting={sorting} onSortingChange={onSortingChange} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx index 5f170a32941..dcbf141314b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx @@ -3,7 +3,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; -import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable"; import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; import { Organization } from "@/components/networking"; import { buttonVariants } from "@/components/ui/button"; @@ -137,38 +137,67 @@ export const getOrganizationsTableColumns = ({ }, { id: "max_budget", + accessorFn: (organization) => getOrganizationBudget(organization).max_budget ?? Number.POSITIVE_INFINITY, + sortUndefined: "last", + sortDescFirst: true, meta: { title: "Budget (USD)", numeric: true }, - header: "Budget (USD)", + header: ({ column }) => , size: 120, - enableSorting: false, + enableSorting: true, cell: ({ row }) => ( ), }, { id: "models", + accessorFn: (organization) => + organization.models?.length && !organization.models.includes("all-proxy-models") + ? [...organization.models].sort().join(", ") + : undefined, + sortUndefined: "last", + sortDescFirst: false, meta: { title: "Models", skeleton: "chips" }, - header: "Models", + header: ({ column }) => , size: 260, - enableSorting: false, + enableSorting: true, cell: ({ row }) => , }, { id: "limits", meta: { title: "TPM / RPM Limits" }, - header: "TPM / RPM Limits", + header: ({ table }) => ( + + ), size: 150, enableSorting: false, cell: ({ row }) => , }, { id: "members", + accessorFn: (organization) => organization.members?.length ?? 0, + sortUndefined: "last", + sortDescFirst: true, meta: { title: "Members", numeric: true }, - header: "Members", + header: ({ column }) => , size: 100, - enableSorting: false, + enableSorting: true, cell: ({ row }) => {row.original.members?.length ?? 0} Members, }, + ...(["tpm_limit", "rpm_limit"] as const).map( + (field): ColumnDef => ({ + id: field, + accessorFn: (organization) => getOrganizationBudget(organization)[field] ?? Number.POSITIVE_INFINITY, + sortUndefined: "last", + sortDescFirst: true, + enableHiding: false, + }), + ), { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts index 20a54a25ba1..ffe67c0aa14 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts @@ -4,7 +4,17 @@ const FILTER_COLUMNS = ["org_id"] as const; type FilterColumn = (typeof FILTER_COLUMNS)[number]; const TABLE_STATE_OPTIONS: UrlTableStateOptions = { - sortFields: ["organization_id", "organization_alias", "created_at", "spend"], + sortFields: [ + "organization_id", + "organization_alias", + "created_at", + "spend", + "max_budget", + "models", + "members", + "tpm_limit", + "rpm_limit", + ], defaultSort: { id: "created_at", desc: true }, defaultPageSize: 25, filterColumns: FILTER_COLUMNS, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx index da2e4debb3c..4903a431ff7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx @@ -1,4 +1,5 @@ import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import { ChatComposer } from "./ChatComposer"; @@ -111,4 +112,23 @@ describe("ChatComposer", () => { expect(screen.getByTestId("chat-composer-input")).toHaveFocus(); }); + + it.each([ + { label: "Find a model", control: }, + { + label: "Choose a model", + control: ( + + ), + }, + ])("should preserve focus when clicking the $label toolbar control", async ({ label, control }) => { + renderComposer({ tools: control }); + + await userEvent.setup().click(screen.getByLabelText(label)); + + expect(screen.getByLabelText(label)).toHaveFocus(); + expect(screen.getByPlaceholderText("Send a message")).not.toHaveFocus(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx index 5d2edbdfc94..c33b5e992fc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx @@ -51,6 +51,14 @@ export function ChatComposer({ } }; + const handleToolbarClick = (event: React.MouseEvent) => { + const control = (event.target as HTMLElement).closest( + "button, input, select, textarea, a[href], [contenteditable='true'], [tabindex]", + ); + if (event.defaultPrevented || (control && event.currentTarget.contains(control))) return; + event.currentTarget.parentElement?.querySelector("[data-slot=input-group-control]")?.focus(); + }; + return (

{showSuggestions && suggestions.length > 0 && ( @@ -99,12 +107,7 @@ export function ChatComposer({ { - if ((event.target as HTMLElement).closest("button")) { - return; - } - event.currentTarget.parentElement?.querySelector("[data-slot=input-group-control]")?.focus(); - }} + onClick={handleToolbarClick} >
{tools}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx new file mode 100644 index 00000000000..39627329b85 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx @@ -0,0 +1,411 @@ +import { useState, type ComponentProps } from "react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { uiSpendLogsCall, userListCall, dailyActivityExportCall, type UserInfo } from "@/components/networking"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { + chooseSelectOption, + renderWithProviders, + screen, + testQueryClient, + waitFor, + within, +} from "../../../../../../tests/test-utils"; +import ClientUsagePanel from "./ClientUsagePanel"; +import { UsageClient, type AgentRow } from "./overview/agentCatalog"; + +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + uiSpendLogsCall: vi.fn(), + dailyActivityExportCall: vi.fn(), + userListCall: vi.fn(), +})); + +const desktopTag = "User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)"; +const agent: AgentRow = { + id: "claude-desktop", + label: "Claude Desktop", + description: "Desktop assistant", + kind: "agent", + logo: null, + spend: 12.5, + tokens: 9000, + requests: 3, + users: 2, + tags: [desktopTag], +}; +const metrics: SpendMetrics = { + spend: 12.5, + total_tokens: 9000, + prompt_tokens: 8000, + completion_tokens: 1000, + cache_read_input_tokens: 5000, + cache_creation_input_tokens: 1000, + api_requests: 3, + successful_requests: 3, + failed_requests: 0, +}; +const daily: DailyData[] = [ + { + date: "2026-10-22", + metrics, + breakdown: { + models: {}, + model_groups: {}, + providers: {}, + api_keys: {}, + mcp_servers: {}, + entities: { [desktopTag]: { metrics, metadata: {}, api_key_breakdown: {} } }, + }, + }, +]; +const defaultProps: ComponentProps = { + client: UsageClient.ClaudeDesktop, + userId: null, + canChooseUser: true, + onUserChange: vi.fn(), + onClose: vi.fn(), + accessToken: "test-token", + startTime: new Date("2026-10-22T00:00:00Z"), + endTime: new Date("2026-10-22T23:59:59.999Z"), + dateValue: { from: new Date("2026-10-22T00:00:00Z"), to: new Date("2026-10-22T23:59:59.999Z") }, + presetShortLabel: null, + onDateChange: vi.fn(), + agent, + daily: [], + loading: false, +}; +const personLog = { + request_id: "request-1", + user: "person-1", + startTime: "2026-10-22T12:00:00Z", + model: "test-model", + spend: 3, + total_tokens: 1000, + prompt_tokens: 800, + completion_tokens: 200, + status: "success", + request_tags: [desktopTag], + metadata: {}, +}; +const person: UserInfo = { + user_id: "person-1", + user_email: "person@example.com", + user_alias: null, + allowed_cache_controls: [], + key_count: 0, + model_max_budget: {}, + model_spend: {}, + models: [], + policies: [], + spend: 0, + teams: [], +}; + +describe("ClientUsagePanel", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + vi.stubGlobal("requestIdleCallback", (callback: IdleRequestCallback) => { + callback({ didTimeout: false, timeRemaining: () => 50 }); + return 1; + }); + const userPage = { + users: [person], + total: 1, + page: 1, + page_size: 50, + total_pages: 1, + }; + const requestPage = { + data: [personLog], + total: 1, + page: 1, + total_pages: 1, + total_is_capped: false, + }; + const exportedRows = [ + { + date: "2026-10-22", + entity_id: desktopTag, + model: "test-model", + spend: 12.5, + flat_cost: 0, + prompt_tokens: 8000, + completion_tokens: 1000, + api_requests: 3, + successful_requests: 3, + failed_requests: 0, + cache_read_input_tokens: 5000, + cache_creation_input_tokens: 1000, + }, + ]; + vi.mocked(dailyActivityExportCall).mockResolvedValue( + Object.assign(new Blob([JSON.stringify(exportedRows)], { type: "application/json" }), { + text: async () => JSON.stringify(exportedRows), + }), + ); + vi.mocked(userListCall).mockResolvedValue(userPage); + vi.mocked(uiSpendLogsCall).mockResolvedValue(requestPage); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ results: [] })), + ); + }); + + afterEach(() => vi.unstubAllGlobals()); + + it("should immediately show the clicked all-users summary and logo without requiring a person", () => { + renderWithProviders(); + + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + expect(within(panel).getByRole("img", { name: "Claude Desktop logo" })).toBeInTheDocument(); + expect(panel).toHaveTextContent("All users"); + expect(panel).toHaveTextContent(/Spend\s*\$12\.50/); + expect(panel).toHaveTextContent(/Total tokens\s*9K/); + expect(panel).toHaveTextContent(/Requests\s*3/); + expect(screen.queryByRole("status", { name: "Loading client usage" })).not.toBeInTheDocument(); + expect(screen.queryByText(/Select a user to/)).not.toBeInTheDocument(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + }); + + it("should filter to a person inside the open panel and keep absent cache details unavailable", async () => { + const user = userEvent.setup(); + const onUserChange = vi.fn(); + const onClose = vi.fn(); + function ControlledPanel() { + const [userId, setUserId] = useState(null); + return ( + { + setUserId(id); + onUserChange(id); + }} + /> + ); + } + renderWithProviders(); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by person" }), /person@example.com/); + expect(onUserChange).toHaveBeenCalledWith("person-1"); + expect(onClose).not.toHaveBeenCalled(); + await waitFor(() => + expect(uiSpendLogsCall).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ user_id: "person-1" }), + }), + ), + ); + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$3\.00/)); + expect(panel).toHaveTextContent("Selected user"); + expect(panel).toHaveTextContent(/Cache reads\s*Unavailable/); + expect(panel).toHaveTextContent(/Cache writes\s*Unavailable/); + expect(panel).toHaveTextContent(/Uncached input\s*Unavailable/); + expect(panel).toHaveTextContent(/Output\s*200/); + + await user.click(screen.getByRole("button", { name: "All users" })); + expect(onUserChange).toHaveBeenLastCalledWith(null); + expect(onClose).not.toHaveBeenCalled(); + expect(screen.getByRole("dialog", { name: "Claude Desktop" })).toHaveTextContent(/Spend\s*\$12\.50/); + }); + + it.each(["Back to Usage", "Close"])("should return to the overview through %s", async (button) => { + const user = userEvent.setup(); + const onClose = vi.fn(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: button })); + expect(onClose).toHaveBeenCalledOnce(); + }); + + it("should preserve summary totals and mark partial model details unavailable", async () => { + const partialAgent = { ...agent, spend: 25, tokens: 18000, requests: 6 }; + renderWithProviders(); + + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + expect(panel).toHaveTextContent(/Spend\s*\$25\.00/); + expect(within(panel).getAllByText("Unavailable")).toHaveLength(4); + expect(await screen.findByText(/Model usage does not match these client totals yet/)).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Daily usage" })).not.toBeInTheDocument(); + expect(screen.queryByText("5,000")).not.toBeInTheDocument(); + }); + + it("should not expose a cached all-users summary without a permitted user scope", () => { + renderWithProviders(); + + expect(screen.getByRole("alert")).toHaveTextContent("Sign in to view your client usage"); + expect(screen.queryByText("$12.50")).not.toBeInTheDocument(); + expect(screen.queryByText("Total tokens")).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: "Filter by person" })).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Daily usage" })).not.toBeInTheDocument(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + expect(screen.queryByRole("button", { name: "Try again" })).not.toBeInTheDocument(); + }); + + it("should show a failed summary as an error and retry without closing or showing stale totals", async () => { + const user = userEvent.setup(); + const onRetry = vi.fn(); + const onClose = vi.fn(); + renderWithProviders(); + + expect(screen.getByRole("alert")).toHaveTextContent("Could not load client totals"); + expect(screen.queryByText("$12.50")).not.toBeInTheDocument(); + expect(screen.queryByText("No Claude Desktop usage recorded in this date range.")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Try again" })); + expect(onRetry).toHaveBeenCalledOnce(); + expect(onClose).not.toHaveBeenCalled(); + }); + it("should change the panel time range through the existing picker without closing", async () => { + const user = userEvent.setup(); + const onDateChange = vi.fn(); + const onClose = vi.fn(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /22 Oct/ })); + await user.click(screen.getByRole("button", { name: /Last 7 days/ })); + await user.click(screen.getByRole("button", { name: "Apply" })); + expect(onDateChange).toHaveBeenCalledWith( + expect.objectContaining({ from: expect.any(Date), to: expect.any(Date) }), + "7d", + ); + expect(onClose).not.toHaveBeenCalled(); + expect(screen.getByRole("dialog", { name: "Claude Desktop" })).toBeInTheDocument(); + }); + + it("should show the client model breakdown using the overview controls", async () => { + renderWithProviders(); + expect(await screen.findByText("test-model")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Daily usage" })).toBeInTheDocument(); + expect(screen.getByRole("radiogroup", { name: "Metric" })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: "Tokens" })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: "Weekly" })).toBeInTheDocument(); + }); + + it("should show personal client usage without loading the global user list or all-user export", async () => { + renderWithProviders(); + + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$3\.00/)); + expect(panel).toHaveTextContent("My usage"); + expect(screen.queryByRole("combobox", { name: "Filter by person" })).not.toBeInTheDocument(); + expect(userListCall).not.toHaveBeenCalled(); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + expect(uiSpendLogsCall).toHaveBeenCalledWith( + expect.objectContaining({ params: expect.objectContaining({ user_id: "person-1" }) }), + ); + expect(vi.mocked(uiSpendLogsCall).mock.lastCall?.[0].params).not.toHaveProperty("team_id"); + }); + + it("should keep team and member filters intersected inside the open panel without fetching global users", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + const onTeamChange = vi.fn(); + const onUserChange = vi.fn(); + const teamOptions = [{ value: "engineering", label: "Engineering" }]; + const memberOptions = [ + { value: "person-1", label: "John" }, + { value: "person-2", label: "Maya" }, + ]; + const johnLog = { ...personLog, team_id: "engineering" }; + const mayaLog = { ...johnLog, request_id: "request-2", user: "person-2", spend: 7 }; + vi.mocked(uiSpendLogsCall).mockImplementation(async (request) => { + const logs = [johnLog, mayaLog].filter((log) => !request.params?.user_id || log.user === request.params.user_id); + return { data: logs, total: logs.length, page: 1, total_pages: 1, total_is_capped: false }; + }); + function TeamPanel() { + const [teamId, setTeamId] = useState(null); + const [userId, setUserId] = useState("person-1"); + const changeTeam = (id: string | null) => { + setTeamId(id); + setUserId(id ? null : "person-1"); + onTeamChange(id); + }; + return ( + { + setUserId(id); + onUserChange(id); + }} + /> + ); + } + renderWithProviders(); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Usage scope" }), "Engineering"); + expect(onTeamChange).toHaveBeenCalledWith("engineering"); + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$10\.00/)); + expect(panel).toHaveTextContent("Engineering · All members"); + expect(uiSpendLogsCall).toHaveBeenLastCalledWith( + expect.objectContaining({ params: expect.objectContaining({ team_id: "engineering" }) }), + ); + expect(vi.mocked(uiSpendLogsCall).mock.lastCall?.[0].params).not.toHaveProperty("user_id"); + + await user.click(screen.getByRole("combobox", { name: "Team member" })); + expect(await screen.findByRole("option", { name: "Maya" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "John" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Outside user" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("option", { name: "Maya" })); + expect(onUserChange).toHaveBeenCalledWith("person-2"); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$7\.00/)); + expect(panel).toHaveTextContent("Engineering · Maya"); + expect(uiSpendLogsCall).toHaveBeenLastCalledWith( + expect.objectContaining({ params: expect.objectContaining({ team_id: "engineering", user_id: "person-2" }) }), + ); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Team member" }), "All members"); + expect(onUserChange).toHaveBeenLastCalledWith(null); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$10\.00/)); + expect(onClose).not.toHaveBeenCalled(); + expect(userListCall).not.toHaveBeenCalled(); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + }); + + it.each([ + [null, "Choose an administered team before viewing another member's usage"], + ["engineering", "Choose a member of the selected team"], + ])("should block requests and cached summaries for an invalid scope in team %s", (teamId, scopeError) => { + renderWithProviders( + , + ); + + expect(screen.getByRole("alert")).toHaveTextContent(scopeError); + expect(screen.queryByText("$12.50")).not.toBeInTheDocument(); + expect(screen.queryByText("Total tokens")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Daily usage" })).not.toBeInTheDocument(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + expect(userListCall).not.toHaveBeenCalled(); + }); + it("should recover a newer model export by refreshing the cached summary on retry", async () => { + const user = userEvent.setup(); + function StaleSummaryPanel() { + const [summary, setSummary] = useState({ ...agent, spend: 12 }); + return setSummary(agent)} />; + } + renderWithProviders(); + expect(await screen.findByText(/Model usage does not match these client totals/)).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Retry model breakdown" })); + expect(await screen.findByText("test-model")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx new file mode 100644 index 00000000000..3ca8f2e038d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx @@ -0,0 +1,362 @@ +"use client"; + +import { useMemo } from "react"; +import { ArrowLeft, Coins } from "lucide-react"; +import { useTagSummary } from "@/app/(dashboard)/hooks/tags/useTagSummary"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import UserDropdown from "@/components/common_components/UserDropdown"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import type { ClientUsageScopeOption } from "../hooks/clientUsageScope"; +import { stackedUsageColor } from "@/components/shared/charts"; +import { Button } from "@/components/ui/button"; +import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; +import { Skeleton } from "@/components/ui/skeleton"; +import { EMPTY_DAILY_ACTIVITY_METADATA, toDailyData } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData } from "@/components/UsagePage/types"; +import claudeLogo from "../../../../../../public/assets/logos/claude.svg"; +import type { UsageClient } from "../hooks/clientUsageData"; +import { useClientUsage } from "../hooks/useClientUsage"; +import { topAgents, type AgentRow } from "./overview/agentCatalog"; +import { useClientAgentModels } from "../hooks/useClientAgentModels"; +import UsageBreakdownChart from "./overview/UsageBreakdownChart"; +import { clientAgentDetail, type ClientAgentDetail } from "./overview/clientAgentDetail"; +import { modelColorMap, formatCompact, formatUsd, overviewTotals } from "./overview/overviewData"; +import { reconciledTokenBreakdown } from "./overview/tokenBreakdown"; +import { Panel, Stat } from "./overview/Primitives"; +import { AgentMark } from "./overview/TopAgents"; + +interface ClientUsagePanelProps { + client: UsageClient; + userId: string | null; + canChooseUser: boolean; + teamId?: string | null; + scopeError?: string | null; + teams?: readonly ClientUsageScopeOption[]; + members?: readonly ClientUsageScopeOption[]; + defaultScopeLabel?: string; + onTeamChange?: (teamId: string | null) => void; + onUserChange: (userId: string | null) => void; + onClose: () => void; + accessToken: string | null; + startTime: Date | null; + endTime: Date | null; + dateValue: DateRangePickerValue; + presetShortLabel: string | null; + onDateChange: (value: DateRangePickerValue, preset: string | null) => void; + colorSource?: readonly DailyData[]; + agent: AgentRow | null; + daily: readonly DailyData[]; + loading: boolean; + summaryFailed?: boolean; + onRetry?: () => void; +} + +const dateLabel = (date: Date | null) => date?.toLocaleDateString("en-US", { month: "short", day: "numeric" }) ?? "—"; +const countLabel = (value: number | null) => (value === null ? "Unavailable" : value.toLocaleString()); + +export default function ClientUsagePanel({ + client, + userId, + canChooseUser, + teamId = null, + scopeError: accessError = null, + teams = [], + members = [], + defaultScopeLabel = "All users", + onTeamChange, + onUserChange, + onClose, + accessToken, + startTime, + endTime, + dateValue, + presetShortLabel, + onDateChange, + colorSource, + agent, + daily, + loading, + summaryFailed = false, + onRetry, +}: ClientUsagePanelProps) { + const label = client === "claude-desktop" ? "Claude Desktop" : "Claude Code"; + const scopedRequest = Boolean(userId || teamId) && !accessError; + const personRequest = { + accessToken, + startTime, + endTime, + userId, + teamId, + client: scopedRequest ? client : null, + }; + const personUsage = useClientUsage(personRequest); + const allUsersEnabled = canChooseUser && !scopedRequest && !accessError; + const teamLabel = teams.find((team) => team.value === teamId)?.label ?? teamId; + const personLabel = members.find((member) => member.value === userId)?.label ?? "Selected user"; + const selectedScopeLabel = (() => { + if (teamId) return `${teamLabel} · ${userId ? personLabel : "All members"}`; + if (!canChooseUser) return "My usage"; + return userId ? "Selected user" : "All users"; + })(); + const summaryQuery = useTagSummary(startTime, endTime, allUsersEnabled && agent === null); + const activeAgent = useMemo( + () => agent ?? topAgents(summaryQuery.data ?? []).find((row) => row.id === client) ?? null, + [summaryQuery.data, agent, client], + ); + const modelRequest = { accessToken, startTime, endTime, agent: activeAgent, enabled: allUsersEnabled }; + const agentModels = useClientAgentModels(modelRequest); + const agentDaily = agentModels.results.length > 0 ? agentModels.results : daily; + const agentDetail = useMemo( + () => (activeAgent ? clientAgentDetail(activeAgent, agentDaily) : null), + [activeAgent, agentDaily], + ); + const colorByModel = useMemo(() => (colorSource ? modelColorMap(colorSource) : undefined), [colorSource]); + const personTotals = overviewTotals(personUsage.data.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA, null); + const pending = + !accessError && (scopedRequest ? personUsage.loading : (loading || summaryQuery.isLoading) && activeAgent === null); + const summaryError = summaryQuery.isError || summaryFailed ? "Could not load client totals" : null; + const scopeError = + accessError ?? (!canChooseUser && !scopedRequest ? "Sign in to view your client usage" : summaryError); + const error = accessError ?? (scopedRequest ? personUsage.error : scopeError); + const summary = scopedRequest ? personTotals : agentDetail?.summary; + const rawBreakdown = scopedRequest + ? { + inputTokens: personTotals.inputTokens, + outputTokens: personTotals.outputTokens, + cacheReadTokens: personUsage.cacheAvailable ? personTotals.cacheReadTokens : null, + cacheWriteTokens: personUsage.cacheAvailable ? personTotals.cacheWriteTokens : null, + } + : agentDetail?.tokenBreakdown; + const breakdown = rawBreakdown && summary ? reconciledTokenBreakdown(summary.tokens, rawBreakdown) : undefined; + const modelResults = scopedRequest ? toDailyData(personUsage.data) : agentModels.results; + const chartLoading = scopedRequest ? personUsage.loading : agentModels.loading; + const chartError = scopedRequest ? personUsage.error : agentModels.error; + + const canFilterPerson = canChooseUser && !teamId && !accessError; + const canRetry = !accessError && (scopedRequest || canChooseUser); + const ready = !pending && !error; + const retryModels = () => { + void summaryQuery.refetch(); + onRetry?.(); + agentModels.reload(); + }; + + return ( + { + if (!open) onClose(); + }} + > + + + +
+ +
+ {label} + + {dateLabel(startTime)} – {dateLabel(endTime)} · {selectedScopeLabel} + +
+
+ +
+
+ +
+ {teams.length > 0 && onTeamChange && ( +
+ + ({ ...team, value: `team:${team.value}` })), + ]} + allowClear={false} + onValueChange={(value) => onTeamChange(value?.startsWith("team:") ? value.slice(5) : null)} + /> +
+ )} + {teamId && ( +
+ + ({ ...member, value: `user:${member.value}` })), + ]} + allowClear={false} + onValueChange={(value) => onUserChange(value?.startsWith("user:") ? value.slice(5) : null)} + /> +
+ )} + {canFilterPerson && ( +
+
+ + {userId && ( + + )} +
+ +
+ )} + {pending && ( +
+ +
+ )} + {error && ( +
+

Could not load client usage

+

{error}

+ {canRetry && ( + + )} +
+ )} + {ready && !summary && ( +

No {label} usage recorded in this date range.

+ )} + {ready && summary && ( + <> +
+ + + +
+ {summary.requests === 0 ? ( +

+ No {label} requests in the selected range. Try another date range. + {(canChooseUser || teamId) && " You can also choose another person."} +

+ ) : ( + <> + {!chartError && ( + + )} + {chartError && ( +
+

+ {chartError} +

+ +
+ )} + {breakdown && } + + )} +

+ {scopedRequest + ? "Based on retained request logs within the exact selected time range." + : "All-user totals use UTC daily aggregates for the selected dates. Filtering by person uses the exact selected times. Cache values are reported counts."}{" "} + Spend is the cost recorded by LiteLLM. +

+ + )} +
+ + + ); +} + +function SummaryStat({ label, value }: { label: string; value: string }) { + return ( +
+ +
+ ); +} + +function TokenExplanation({ total, breakdown }: { total: number; breakdown: ClientAgentDetail["tokenBreakdown"] }) { + const { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens } = breakdown; + const freshInput = + inputTokens !== null && cacheReadTokens !== null && cacheWriteTokens !== null + ? inputTokens - cacheReadTokens - cacheWriteTokens + : null; + const rows = [ + { label: "Cache reads", value: cacheReadTokens, color: stackedUsageColor(0) }, + { label: "Cache writes", value: cacheWriteTokens, color: stackedUsageColor(1) }, + { label: "Uncached input", value: freshInput, color: stackedUsageColor(2) }, + { label: "Output", value: outputTokens, color: stackedUsageColor(3) }, + ]; + const complete = rows.every((row) => row.value !== null); + return ( + + {complete && total > 0 && ( + + )} + {cacheReadTokens !== null && total > 0 && ( +

+ {((cacheReadTokens / total) * 100).toFixed(1)}% of these tokens are cache reads. +

+ )} +
+ {rows.map((row) => ( +
+
+ + {row.label} +
+
{countLabel(row.value)}
+
+ ))} +
+

+ Total tokens include input, cached input and output. Cached input is priced differently, so token count alone + does not explain spend. Compare the same person and dates when checking the count shown by a client. +

+ + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx new file mode 100644 index 00000000000..d0b0c7750d6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx @@ -0,0 +1,25 @@ +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { parseUsageClient, UsageClient } from "./overview/agentCatalog"; + +const CLIENT_OPTIONS = [ + { value: UsageClient.ClaudeDesktop, label: "Claude Desktop" }, + { value: UsageClient.ClaudeCode, label: "Claude Code" }, +]; + +export default function ClientUsageSelect({ + value, + onChange, +}: { + value: UsageClient | null; + onChange: (client: UsageClient | null) => void; +}) { + return ( + onChange(parseUsageClient(client))} + /> + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx deleted file mode 100644 index d971f5d0506..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { screen } from "@testing-library/react"; -import { beforeAll, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; -import UsageAIChatPanel from "./UsageAIChatPanel"; - -beforeAll(() => { - if (typeof window !== "undefined" && !window.ResizeObserver) { - window.ResizeObserver = class ResizeObserver { - observe() {} - unobserve() {} - disconnect() {} - } as any; - } -}); - -vi.mock("@/components/networking", () => ({ - modelHubCall: vi.fn().mockResolvedValue({ - data: [{ model_group: "gpt-4" }, { model_group: "claude-3-opus" }], - }), - usageAiChatStream: vi.fn(), -})); - -const defaultProps = { - open: true, - onClose: vi.fn(), - accessToken: "test-token", -}; - -describe("UsageAIChatPanel", () => { - it("should render the panel when open", () => { - renderWithProviders(); - - expect(screen.getByText("Ask AI")).toBeInTheDocument(); - expect(screen.getByText("Ask about your spend, models, keys, and trends")).toBeInTheDocument(); - }); - - it("should render model selector", () => { - renderWithProviders(); - - // One library paints the prompt as its own text node and the other leaves it on the input's - // placeholder attribute, so either one means the user is being told what to pick. - const prompt = "Select a model (optional, defaults to gpt-4o-mini)"; - expect(screen.queryAllByText(prompt).length + screen.queryAllByPlaceholderText(prompt).length).toBeGreaterThan(0); - }); - - it("should render empty state message when no conversation", () => { - renderWithProviders(); - - expect(screen.getByText("Ask a question about your usage")).toBeInTheDocument(); - }); - - it("should render the send button", () => { - renderWithProviders(); - - expect(screen.getByText("Send")).toBeInTheDocument(); - }); - - it("should render input placeholder", () => { - renderWithProviders(); - - expect(screen.getByPlaceholderText("Ask about your usage...")).toBeInTheDocument(); - }); - - it("should render clear chat button", () => { - renderWithProviders(); - - expect(screen.getByText("Clear chat")).toBeInTheDocument(); - }); - - it("should have the panel element even when closed (just off-screen)", () => { - renderWithProviders(); - - expect(screen.getByTestId("usage-ai-chat-panel")).toBeInTheDocument(); - expect(screen.getByTestId("usage-ai-chat-panel")).toHaveClass("translate-x-full"); - }); - - it("should not have translate-x-full class when open", () => { - renderWithProviders(); - - expect(screen.getByTestId("usage-ai-chat-panel")).not.toHaveClass("translate-x-full"); - expect(screen.getByTestId("usage-ai-chat-panel")).toHaveClass("translate-x-0"); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx deleted file mode 100644 index dc4291d25e1..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx +++ /dev/null @@ -1,385 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import ReactMarkdown from "react-markdown"; -import { MessageSquare, Sparkles, X } from "lucide-react"; -import { Button } from "@/components/ui/button"; -import { - Combobox, - ComboboxContent, - ComboboxEmpty, - ComboboxInput, - ComboboxItem, - ComboboxList, -} from "@/components/ui/combobox"; -import { Textarea } from "@/components/ui/textarea"; -import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; -import { modelHubCall, usageAiChatStream, UsageAiToolCallEvent } from "@/components/networking"; - -interface ToolCallStep { - tool_name: string; - tool_label: string; - arguments: Record; - status: "running" | "complete" | "error"; - error?: string; -} - -interface ChatMessage { - role: "user" | "assistant"; - content: string; - toolCalls?: ToolCallStep[]; -} - -interface UsageAIChatPanelProps { - open: boolean; - onClose: () => void; - accessToken: string | null; -} - -const TOOL_ICONS: Record = { - get_usage_data: "📊", - get_team_usage_data: "👥", - get_tag_usage_data: "🏷️", -}; - -const ToolCallDisplay: React.FC<{ step: ToolCallStep }> = ({ step }) => { - const icon = TOOL_ICONS[step.tool_name] || "🔧"; - const args = step.arguments; - const dateRange = args.start_date && args.end_date ? `${args.start_date} → ${args.end_date}` : ""; - const filter = args.team_ids || args.tags || args.user_id || ""; - - return ( -
- - {step.status === "running" ? ( - - ) : step.status === "error" ? ( - ✗ - ) : ( - ✓ - )} - -
-
- {icon} {step.tool_label} -
- {dateRange &&
{dateRange}
} - {filter &&
Filter: {filter}
} - {step.status === "error" && step.error &&
{step.error}
} -
-
- ); -}; - -const MarkdownContent: React.FC<{ content: string }> = ({ content }) => ( -

{children}

, - strong: ({ children }) => {children}, - ul: ({ children }) =>
    {children}
, - ol: ({ children }) =>
    {children}
, - li: ({ children }) =>
  • {children}
    • , - h1: ({ children }) =>

    {children}

    , - h2: ({ children }) =>

    {children}

    , - h3: ({ children }) =>

    {children}

    , - code: ({ children, className }) => { - const isBlock = className?.includes("language-"); - return isBlock ? ( - 
    -            {children}
-
    - ) : ( - {children} - ); - }, - table: ({ children }) => ( -
    - {children}
    -
    - ), - th: ({ children }) => ( - {children} - ), - td: ({ children }) => {children}, - }} - > - {content} -     -); - -const UsageAIChatPanel: React.FC = ({ open, onClose, accessToken }) => { - const [messages, setMessages] = useState([]); - const [inputText, setInputText] = useState(""); - const [isLoading, setIsLoading] = useState(false); - const [selectedModel, setSelectedModel] = useState(undefined); - const [availableModels, setAvailableModels] = useState([]); - const [isLoadingModels, setIsLoadingModels] = useState(false); - const [streamingContent, setStreamingContent] = useState(""); - const [statusMessage, setStatusMessage] = useState(null); - const [activeToolCalls, setActiveToolCalls] = useState([]); - const messagesEndRef = useRef(null); - const abortControllerRef = useRef(null); - - useEffect(() => { - if (open && availableModels.length === 0) { - loadModels(); - } - }, [open]); - - useEffect(() => { - if (typeof messagesEndRef.current?.scrollIntoView === "function") { - messagesEndRef.current.scrollIntoView({ behavior: "smooth" }); - } - }, [messages, streamingContent, activeToolCalls, statusMessage]); - - const loadModels = async () => { - if (!accessToken) return; - setIsLoadingModels(true); - try { - const fetchedModels = await modelHubCall(accessToken); - if (fetchedModels?.data?.length > 0) { - const models = fetchedModels.data.map((item: any) => item.model_group as string).sort(); - setAvailableModels(models); - } - } catch (error) { - console.error("Failed to load models:", error); - } finally { - setIsLoadingModels(false); - } - }; - - const handleSend = async () => { - if (!accessToken || !inputText.trim() || isLoading) return; - - const userMessage: ChatMessage = { role: "user", content: inputText.trim() }; - const updatedMessages = [...messages, userMessage]; - setMessages(updatedMessages); - setInputText(""); - setIsLoading(true); - setStreamingContent(""); - setStatusMessage(null); - setActiveToolCalls([]); - - const abortController = new AbortController(); - abortControllerRef.current = abortController; - - let accumulated = ""; - const toolCalls: ToolCallStep[] = []; - - try { - await usageAiChatStream( - accessToken, - updatedMessages.slice(-20).map((m) => ({ role: m.role, content: m.content })), - selectedModel || "", - (content: string) => { - setStatusMessage(null); - accumulated += content; - setStreamingContent(accumulated); - }, - () => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [ - ...prev, - { role: "assistant", content: accumulated, toolCalls: toolCalls.length > 0 ? [...toolCalls] : undefined }, - ]); - setStreamingContent(""); - }, - (errorMsg: string) => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - }, - (status: string) => { - setStatusMessage(status); - }, - (event: UsageAiToolCallEvent) => { - const idx = toolCalls.findIndex((tc) => tc.tool_name === event.tool_name); - if (idx >= 0) { - toolCalls[idx] = { ...event }; - } else { - toolCalls.push({ ...event }); - } - setActiveToolCalls([...toolCalls]); - }, - abortController.signal, - ); - } catch (error: any) { - if (error?.name === "AbortError" || abortController.signal.aborted) { - return; - } - const errorMsg = error?.message || "Failed to get response. Please try again."; - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - } finally { - setIsLoading(false); - abortControllerRef.current = null; - } - }; - - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleSend(); - } - }; - - const handleClose = () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - } - onClose(); - }; - - const handleClear = () => { - setMessages([]); - setStreamingContent(""); - setActiveToolCalls([]); - setStatusMessage(null); - }; - - return ( -
    - {/* Header */} -
    -
    -

    -

    -

    Ask about your spend, models, keys, and trends

    -
    - -
    - - {/* Model selector */} -
    - setSelectedModel(value ?? undefined)} - > - - - {isLoadingModels ? "Loading models…" : "No models found"} - - {(model: string) => ( - - {model} - - )} - - - -
    - - {/* Chat messages */} -
    - {messages.length === 0 && !streamingContent && !isLoading && ( -
    -
    - )} - - {messages.map((msg, idx) => ( -
    - {msg.role === "user" ? ( -
    -
    - {msg.content} -
    -
    - ) : ( -
    - {/* Tool calls for this message */} - {msg.toolCalls && msg.toolCalls.length > 0 && ( -
    - {msg.toolCalls.map((tc, tcIdx) => ( - - ))} -
    - )} - {/* Response */} -
    - -
    -
    - )} -
    - ))} - - {/* Active tool calls (in-progress) */} - {isLoading && activeToolCalls.length > 0 && ( -
    - {activeToolCalls.map((tc, idx) => ( - - ))} -
    - )} - - {/* Status / spinner */} - {isLoading && !streamingContent && ( -
    - - {statusMessage || "Thinking..."} -
    - )} - - {/* Streaming response */} - {streamingContent && ( -
    - -
    - )} - -
    -
    - - {/* Input area */} -
    -
    -