diff --git a/.github/deploy-on-aws.png b/.github/deploy-on-aws.png new file mode 100644 index 00000000000..06d41f2a5e0 Binary files /dev/null and b/.github/deploy-on-aws.png differ diff --git a/.github/deploy-on-gcp.png b/.github/deploy-on-gcp.png new file mode 100644 index 00000000000..e831a8c2e4e Binary files /dev/null and b/.github/deploy-on-gcp.png differ diff --git a/.github/workflows/osv-scan.yml b/.github/workflows/osv-scan.yml index 9dd321f88db..0cd94fdd9e2 100644 --- a/.github/workflows/osv-scan.yml +++ b/.github/workflows/osv-scan.yml @@ -7,11 +7,6 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" - paths: - - uv.lock - - ui/litellm-dashboard/package-lock.json - - osv-scanner.toml - - .github/workflows/osv-scan.yml schedule: - cron: "23 6 * * *" workflow_dispatch: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index de7e1b68346..950d6ca31a6 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -14,7 +14,7 @@ permissions: jobs: lint: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 @@ -87,9 +87,11 @@ jobs: run: | uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - - name: Run basedpyright type checking + - name: Check basedpyright budget (delta vs base) + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py + (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA" - name: Check for circular imports run: | diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml new file mode 100644 index 00000000000..3d0a159cdc7 --- /dev/null +++ b/.github/workflows/test-rust.yml @@ -0,0 +1,65 @@ +name: LiteLLM Rust + +on: + push: + paths: + - "litellm-rust/**" + - ".github/workflows/test-rust.yml" + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_branch + - "litellm_**" + paths: + - "litellm-rust/**" + - ".github/workflows/test-rust.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + rust-checks: + name: rustfmt, clippy, test + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: litellm-rust + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Rust + run: | + rustup toolchain install stable --profile minimal --component clippy,rustfmt + rustup default stable + + - name: Cache Cargo registry and target + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Check Rust formatting + run: cargo fmt --check + + - name: Run Clippy + run: cargo clippy --workspace --all-targets --locked -- -D warnings + + - name: Run Rust tests + run: cargo test --workspace --locked diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index a7363ac3b43..2226d519331 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -32,7 +32,9 @@ jobs: tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions + tests/test_litellm/ocr tests/test_litellm/passthrough + tests/test_litellm/sandbox tests/test_litellm/vector_stores tests/test_litellm/test_*.py workers: 2 diff --git a/CLAUDE.md b/CLAUDE.md index 2070b6fcdd6..b721064aaa7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ If you ever make public-facing PR descriptions, comments, issues, commit message - don't use "—". Instead, reach for ";", ".", etc. - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose -- don't add a trailing "." at the end of paragraphs (just like this file) +- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: unless there's a sentence immediately after, don't add a "." - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs diff --git a/Dockerfile b/Dockerfile index 4d55148ff89..af49dc8d8cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/Makefile b/Makefile index 27150aec938..076eac0f4a7 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,8 @@ lint-ruff-FULL-dev: install-dev else echo "No changed .py files to check."; fi lint-basedpyright: install-dev - ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py + git fetch origin litellm_internal_staging + ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging lint-basedpyright-budget-update: install-dev ($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update diff --git a/README.md b/README.md index b26ad39eada..3d0f7282d7c 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render - - Deploy on Railway - + Deploy to Render + Deploy on Railway + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

@@ -406,6 +406,140 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+). +### Deploy on AWS or GCP with Terraform + +Run the LiteLLM proxy as a production-ready componentized stack (gateway, backend, UI on separate services; managed Postgres + Redis + object store) using the published Terraform modules. Both modules are on the [public Terraform Registry](https://registry.terraform.io/namespaces/BerriAI) — no auth needed. + +#### AWS — ECS Fargate + Aurora + ElastiCache + ALB + +[![Launch in AWS CloudShell](https://img.shields.io/badge/Launch-AWS_CloudShell-FF9900?logo=amazon-aws&logoColor=white)](https://console.aws.amazon.com/cloudshell/home) — opens an in-browser shell, already authenticated to your AWS account. Once inside, run: + +```bash +git clone https://github.com/BerriAI/litellm.git +cd litellm/terraform/litellm/aws/examples/default +cp terraform.tfvars.example terraform.tfvars # edit region/tenant/env +terraform init && terraform apply +``` + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest) + +Or call the module from your own root config: + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + aws = { source = "hashicorp/aws", version = "~> 5.60" } + } +} + +provider "aws" { + region = "us-west-2" +} + +module "litellm" { + source = "BerriAI/litellm/aws" + version = "~> 1.89" + + region = "us-west-2" + azs = ["us-west-2a", "us-west-2b"] + tenant = "acme" + env = "prod" + + # Production: provide an ACM cert. Without one, set allow_plaintext_alb = true + # (dev/trial only). + # acm_certificate_arn = "arn:aws:acm:us-west-2:111122223333:certificate/..." + allow_plaintext_alb = true +} + +output "litellm_url" { + value = module.litellm.alb_dns_name +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest?tab=inputs). + +#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB + +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) + +Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply. + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest) + +To call the module from your own config instead, Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: + +```bash +gcloud artifacts repositories create litellm \ + --location=us-central1 \ + --repository-format=docker \ + --mode=remote-repository \ + --remote-docker-repo=https://ghcr.io \ + --project=my-gcp-project +``` + +Then: + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + google = { source = "hashicorp/google", version = "~> 6.10" } + google-beta = { source = "hashicorp/google-beta", version = "~> 6.10" } + } +} + +provider "google" { project = "my-gcp-project"; region = "us-central1" } +provider "google-beta" { project = "my-gcp-project"; region = "us-central1" } + +module "litellm" { + source = "BerriAI/litellm/google" + version = "~> 1.89" + + project_id = "my-gcp-project" + region = "us-central1" + tenant = "acme" + env = "prod" + + # Replace my-gcp-project with your GCP project ID (same value as project_id above). + image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" + + # Production: provide DNS already pointing at the LB IP for Google-managed certs. + # Without one, set allow_plaintext_lb = true (dev/trial only). + # lb_domains = ["proxy.example.com"] + allow_plaintext_lb = true +} + +output "litellm_url" { + value = module.litellm.load_balancer_url +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in Secret Manager; reference resource IDs (e.g. `projects/my-gcp-project/secrets/openai-api-key`) via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/google/latest?tab=inputs). + +#### Both stacks include + +- The full componentized split (gateway / backend / UI as independent services) +- Managed Postgres (writer + reader) and Redis +- Versioned object store for proxy state + file uploads +- An auto-generated `LITELLM_MASTER_KEY` in your cloud's secret manager +- A one-off migration job that runs `prisma migrate deploy` before the proxy starts +- The same `proxy_config` surface as the [Helm chart](./helm/litellm/) — pass YAML as a typed map + +The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/) and [`terraform/litellm/gcp/`](./terraform/litellm/gcp/) in this repo; the registry entries are read-only mirrors updated on each release. + ### Run in Developer Mode #### Services 1. Setup .env file in root diff --git a/backend/Dockerfile b/backend/Dockerfile index 2cfdde8a517..667bdb073eb 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 7ba7656e407..f5b0a9aaf81 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -121,7 +121,7 @@ }, "reportReturnType": { "baseline": 126, - "slack": 13 + "slack": 100 }, "reportTypedDictNotRequiredAccess": { "baseline": 20, @@ -157,7 +157,7 @@ }, "reportUnnecessaryComparison": { "baseline": 683, - "slack": 10 + "slack": 100 }, "reportUnnecessaryContains": { "baseline": 4, diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index e591a4a2adb..50ef55e3261 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index eafbd23fd90..ab02b43d0f9 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,6 +1,6 @@ # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 19c8a10fdfe..716b2fa09d1 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/litellm-rust/.gitignore b/litellm-rust/.gitignore new file mode 100644 index 00000000000..b83d22266ac --- /dev/null +++ b/litellm-rust/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md new file mode 100644 index 00000000000..2fa81798605 --- /dev/null +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -0,0 +1,9 @@ +# Adding a provider / route to litellm-rust + +Three layers, same for every route (see `ocr` and `realtime` as references): + +1. **Transform contract (pure)** — `crates/core/src//transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth. +2. **Provider config (pure)** — `crates/providers/src///transformation.rs`: implement that trait as a `const __CONFIG`, mirroring the Python provider tree. Add parity unit tests. +3. **HTTP / transport (the host)** — `crates/providers/src/.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O. + +**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`. diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md new file mode 100644 index 00000000000..1d2987e0a1a --- /dev/null +++ b/litellm-rust/CLAUDE.md @@ -0,0 +1,88 @@ +# CLAUDE.md + +This file defines the rules for Rust work in LiteLLM. + +## Core Boundary + +The `core` and `providers` crates describe work; hosts execute work. + +Route-level Rust structure mirrors LiteLLM's Python responsibilities: +- `core/src//` owns the route contract, shared types, and provider + template traits. For OCR, this means `core/src/ocr`. +- `providers/src///transformation.rs` owns the + provider-specific transform. For Mistral OCR, this means + `providers/src/mistral/ocr/transformation.rs`. +- Future network execution belongs in a host/transport layer such as + `llm_http_handler`, not inside `core` or `providers`. + +Allowed in `core` and `providers`: +- Pure request transforms +- Pure response transforms +- Pure stream chunk normalization +- Shared data types and validation errors +- Deterministic token/cost helper logic + +Not allowed in `core` or `providers`: +- Network calls +- Environment variable or secret reads +- Filesystem access +- Database or cache access +- Provider SDK signing or auth flows +- Logging callbacks, spend writes, or custom callbacks +- Global mutable runtime state + +Python owns rollout state and fallback while Rust is being introduced. Rust +paths must be off by default until parity tests prove equivalence with Python. + +## Production Bar + +Rust code in this workspace is held to a strict parity and robustness bar from +the first PR: + +- Correctness parity is proven with tests. Do not rely on README claims or + manual inspection for a port that mirrors Python behavior. +- Every provider transform must have unit tests for supported-parameter + filtering, request body shape, response normalization, missing/null fields, + and bad-input errors. +- When Rust is exposed through Python, add Python tests that prove disabled, + enabled, and unavailable-bridge fallback behavior. +- Avoid panics on user/provider input. Return typed errors and let the host map + them to Python exceptions or HTTP responses. +- OCR handles documents that often contain personal data. Do not log document + contents, base64 payloads, provider response bodies, or secrets. +- Error messages must be useful but data-minimized. Truncate or sanitize any + upstream body before it crosses a host boundary. +- Treat empty or whitespace-only credentials, URLs, and config values as absent + at the host/config resolution layer. +- Preserve Python output shape intentionally. If a field is always serialized as + `null` for Python parity, leave a short comment explaining that parity choice. + +## Host I/O Rules + +These rules apply when adding future crates or modules that execute network I/O, +such as `ai-gateway`, router hosts, or standalone servers: + +- Set connect and full-request timeouts. No unbounded waits. +- Reuse HTTP clients; do not construct clients per request. +- Prefer rustls TLS for portable Python wheels and Linux images unless there is + a documented reason not to. +- Add request IDs and structured tracing at the host layer, without logging OCR + document contents or secrets. +- Do not echo raw upstream response bodies to callers. Sanitize and bound them. +- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is + impossible by construction and documented. + +## Checks + +Run these before pushing Rust changes. The same checks run in GitHub Actions +for changes under `litellm-rust/`. + +```bash +cd litellm-rust +cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` + +When a Rust path is exposed through Python, add Python parity tests that compare +the existing Python output with the Rust-backed output. diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock new file mode 100644 index 00000000000..a269a224d97 --- /dev/null +++ b/litellm-rust/Cargo.lock @@ -0,0 +1,1872 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "base64", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "litellm-ai-gateway" +version = "0.1.0" +dependencies = [ + "axum", + "futures-util", + "litellm-core", + "litellm-providers", + "pyo3", + "serde", + "serde_json", + "subtle", + "tokio", +] + +[[package]] +name = "litellm-core" +version = "0.1.0" +dependencies = [ + "rand 0.8.6", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "litellm-providers" +version = "0.1.0" +dependencies = [ + "futures-channel", + "futures-util", + "litellm-core", + "reqwest", + "serde_json", + "tokio", + "tokio-tungstenite", +] + +[[package]] +name = "litellm-python-bridge" +version = "0.1.0" +dependencies = [ + "litellm-core", + "litellm-providers", + "pyo3", + "serde_json", +] + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml new file mode 100644 index 00000000000..06289e5a46f --- /dev/null +++ b/litellm-rust/Cargo.toml @@ -0,0 +1,28 @@ +[workspace] +members = [ + "crates/core", + "crates/providers", + "crates/python-bridge", + "crates/ai-gateway", +] +resolver = "2" + +[workspace.package] +edition = "2021" +license = "MIT" +repository = "https://github.com/BerriAI/litellm" + +[workspace.dependencies] +litellm-core = { path = "crates/core" } +litellm-providers = { path = "crates/providers" } +axum = "0.7" +pyo3 = "0.23.5" +rand = "0.8" +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +subtle = "2" +thiserror = "2.0" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } +tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } +futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } diff --git a/litellm-rust/README.md b/litellm-rust/README.md new file mode 100644 index 00000000000..15ad1855420 --- /dev/null +++ b/litellm-rust/README.md @@ -0,0 +1,34 @@ +# LiteLLM Rust + +This workspace contains the staged Rust implementation for LiteLLM. + +Rust starts as a pure transform core used by the existing Python host. Python +continues to own auth, configuration, network I/O, retries, routing, logging, +callbacks, spend tracking, and customer plugins until each Rust path has parity +coverage and production evidence. + +## Layout + +```text +crates/ + core/ Route contracts, shared pure types, errors, and templates. + src/ocr/ + providers/ Provider-specific pure transforms. + src/mistral/ocr/transformation.rs + python-bridge/ PyO3 bridge for Python LiteLLM. +``` + +The folder shape should follow the Python provider tree: +`providers/src///transformation.rs`. The bridge should expose +one function per top-level route, starting with `ocr(payload)`. + +## Checks + +Run these before pushing Rust changes. GitHub Actions runs the same checks for +changes under `litellm-rust/`. + +```bash +cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` diff --git a/litellm-rust/crates/ai-gateway/AGENTS.md b/litellm-rust/crates/ai-gateway/AGENTS.md new file mode 100644 index 00000000000..d9e6e1adde5 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/AGENTS.md @@ -0,0 +1,50 @@ +# ai-gateway — folder architecture + +The Axum server that fronts the Rust gateway. It owns transport + config + auth +only; deployment selection lives in `core::router`, transforms in `core`/`providers`. + +``` +src/ + main.rs # entrypoint: build AppState (router + master key), bind, serve + state.rs # AppState — shared Arc + master_key + gil.rs # GIL-activity tracker (records Python acquisitions) + auth/ # authentication as an axum extractor — added to handler args + mod.rs # RequireMasterKey: FromRequestParts, single master key (LITELLM_MASTER_KEY) + routes/ # one module per route, all matching the same template + AGENTS.md # ← the route template (read this before adding a route) + mod.rs # app(): merges every module's router() + health.rs # simple route (one file): router() + liveness/readiness + gil.rs # simple route (one file): router() + GET /health/gil + realtime/ # route with logic → axum surface + a no-axum service: + mod.rs # router() + handler + WS<->events adapter (the axum surface) + service.rs # business logic (select deployment, call provider) — no axum, testable + python/ # Python interop (feature: python-config) — load-time only + mod.rs, config.rs, AGENTS.md +``` + +## Rules + +- **Routes follow one template.** Each route module exposes + `pub fn router() -> Router`; `routes/mod.rs` only merges them. Simple + routes are one file; non-trivial routes are a folder (`handler`/`service`/ + `transport`). See `routes/AGENTS.md`. +- **Auth is an extractor.** Add `crate::auth::RequireMasterKey` to a handler's + args; it runs during extraction. Never re-implement the check per route. +- **Handlers are thin.** A handler validates and delegates to its `service`. No + business logic, no provider calls, no transforms in handlers. +- **State is shared and cheap to clone.** Long-lived handles live behind `Arc` in + `state.rs`; read env/config only in `main.rs` when building state. + +## Auth (interim) + +A single **master key** (`LITELLM_MASTER_KEY`), enforced by the +`auth::RequireMasterKey` extractor: any caller presenting it as +`Authorization: Bearer ` may invoke the gateway. Fails closed (500) when +unset; constant-time compare. The server binds `127.0.0.1` by default (`HOST` to +override). Full per-key auth + budgets/rate-limits are delegated to the Python +proxy in a later phase. Health routes don't add the extractor (unauthenticated). + +## Python interop + +Anything that calls into Python lives in `python/` and is **load-time only** — see +`python/AGENTS.md`. The realtime data path never takes the GIL. diff --git a/litellm-rust/crates/ai-gateway/Cargo.toml b/litellm-rust/crates/ai-gateway/Cargo.toml new file mode 100644 index 00000000000..79bdc4bdb26 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "litellm-ai-gateway" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "litellm-ai-gateway" +path = "src/main.rs" + +[dependencies] +litellm-core.workspace = true +litellm-providers.workspace = true +axum = { workspace = true, features = ["ws"] } +futures-util.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "time"] } +serde.workspace = true +serde_json.workspace = true +subtle.workspace = true +pyo3 = { workspace = true, features = ["auto-initialize"], optional = true } + +[features] +# Build the gateway's config from the proxy YAML via an embedded Python +# interpreter (links libpython; requires `litellm` importable at runtime). +python-config = ["dep:pyo3"] diff --git a/litellm-rust/crates/ai-gateway/Dockerfile b/litellm-rust/crates/ai-gateway/Dockerfile new file mode 100644 index 00000000000..adf6fca0741 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/Dockerfile @@ -0,0 +1,86 @@ +# Multi-stage build for the LiteLLM Rust AI Gateway (realtime WebSocket proxy). +# +# Build context is the **repo root** so we can install `litellm` from this repo's +# source (the gateway loads its model_list via litellm.proxy.read_model_list, +# which is not in any PyPI release yet) AND build the rust workspace under +# litellm-rust/. +# +# docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway . +# +# No secrets live in this file. Runtime config (LITELLM_MASTER_KEY, +# OPENAI_API_KEY referenced by config.yaml, etc.) is injected as environment +# variables at deploy time. + +# ---- Chef ------------------------------------------------------------------- +# cargo-chef caches the dependency build so only the gateway crate recompiles on +# a source-only change. python3-dev is present in every rust stage because the +# `python-config` feature links libpython via pyo3 (even in the cook step). +FROM rust:1.90-slim-bookworm AS chef +ENV PYO3_PYTHON=python3.11 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 python3-dev pkg-config libssl-dev clang \ + && rm -rf /var/lib/apt/lists/* \ + && cargo install cargo-chef --locked --version 0.1.77 +WORKDIR /build/litellm-rust + +# ---- Planner ---------------------------------------------------------------- +# Produce the dependency recipe from the rust workspace manifests + Cargo.lock. +FROM chef AS planner +COPY litellm-rust/ . +RUN cargo chef prepare --recipe-path recipe.json + +# ---- Builder ---------------------------------------------------------------- +FROM chef AS builder +# Cook (compile) just the dependencies first — this layer is cached and reused +# whenever only gateway source changes. +COPY --from=planner /build/litellm-rust/recipe.json recipe.json +RUN cargo chef cook --locked --release \ + -p litellm-ai-gateway --features python-config \ + --recipe-path recipe.json +# Now copy the real sources and build the gateway binary. Deps are already cooked +# above, so this step only recompiles the gateway crate. +COPY litellm-rust/ . +RUN cargo build --locked --release -p litellm-ai-gateway --features python-config + +# ---- Runtime ---------------------------------------------------------------- +# python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3 +# 3.11 ABI so the embedded interpreter links and imports cleanly. +FROM python:3.11-slim-bookworm AS runtime + +# CA certificates for outbound TLS to the OpenAI realtime endpoint. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install litellm (with proxy extras) FROM THIS REPO'S SOURCE so +# `import litellm.proxy.read_model_list` works — it is not on PyPI yet. Copy the +# package + packaging metadata, then pip install the proxy extra. +COPY pyproject.toml README.md LICENSE ./ +COPY litellm/ ./litellm/ +RUN pip install --no-cache-dir ".[proxy]" + +# The compiled gateway binary (pure-Rust realtime hot path; Python is load-time +# only). +COPY --from=builder /build/litellm-rust/target/release/litellm-ai-gateway /usr/local/bin/litellm-ai-gateway + +# Default config.yaml. A real deploy can override this (e.g. mount a Render +# secret file at the same path) — never bake secrets into the image. +COPY litellm-rust/crates/ai-gateway/config.yaml /app/config.yaml + +# Bind to all interfaces (Render routes to 0.0.0.0:$PORT) and load the model_list +# from config.yaml via the embedded python config reader. +ENV HOST=0.0.0.0 \ + LITELLM_CONFIG_PATH=/app/config.yaml + +# Drop to a non-root user. The realtime hot path needs no root privileges, so +# running unprivileged limits blast radius if the process is ever compromised. +# The binary in /usr/local/bin is world-executable (COPY default mode 755); we +# only need /app (and the config.yaml it reads) owned by the unprivileged user. +RUN useradd --system --no-create-home --uid 10001 appuser \ + && chown -R appuser:appuser /app +USER appuser + +ENTRYPOINT ["/usr/local/bin/litellm-ai-gateway"] diff --git a/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore b/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore new file mode 100644 index 00000000000..030ee6a37c5 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore @@ -0,0 +1,45 @@ +# Dockerfile-specific ignore-file for the Rust AI Gateway build. +# +# The build context is the repo root (so the image can pip install litellm from +# source AND build the rust workspace). BuildKit honors `.dockerignore` +# next to the Dockerfile and it takes precedence over the repo-root `.dockerignore`, +# so this file shrinks the (large) repo-root context for THIS build only without +# touching the root `.dockerignore` used by the main litellm images. +# +# Strategy: ignore everything, then re-include only what the build needs: +# - litellm/ (pip install . needs the full package + proxy reader) +# - litellm-rust/ (the rust workspace; Cargo.lock + crate sources) +# - pyproject.toml / README.md / LICENSE (packaging metadata for pip install) +* + +# --- re-include the build inputs --- +!litellm/ +!litellm-rust/ +!pyproject.toml +!README.md +!LICENSE + +# --- prune heavy / irrelevant subpaths back out of the re-included trees --- +# Rust build artifacts (huge; regenerated in the builder). +**/target/ +# Python caches and compiled bytecode. +**/__pycache__/ +**/*.pyc +**/*.pyo +**/.pytest_cache/ +**/.ruff_cache/ +**/.mypy_cache/ +# Node / UI build output bundled under the python package (not needed to import +# litellm.proxy.read_model_list). +**/node_modules/ +litellm/proxy/_experimental/out/ +# Tests, logs, and local scratch. +**/tests/ +**/test/ +*.log +log.txt +*.tgz +# VCS / editor / CI metadata that may live under re-included trees. +**/.git/ +.git/ +**/.DS_Store diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md new file mode 100644 index 00000000000..3662ce2584a --- /dev/null +++ b/litellm-rust/crates/ai-gateway/README.md @@ -0,0 +1,172 @@ +# LiteLLM Rust AI Gateway + +A minimal Axum service that fronts OpenAI's realtime API. Clients open a +WebSocket to `GET /v1/realtime`; the gateway authenticates, selects a deployment, +dials OpenAI upstream, and splices the two sockets frame-by-frame. + +- **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) +- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) +- **Health:** `GET /health/readiness`, `GET /health/liveness`, `GET /health/gil` + +> **Realtime serving is pure Rust.** Python is used at **load time only** — to +> read the config once at boot. The realtime hot path never touches Python. + +## Configuration (config.yaml) + +The gateway loads its `model_list` from a **config.yaml**, the same as the +LiteLLM proxy. Point `LITELLM_CONFIG_PATH` at the file: + +```yaml +# config.yaml +model_list: + - model_name: gpt-realtime + litellm_params: + model: openai/gpt-realtime + api_key: os.environ/OPENAI_API_KEY +``` + +```bash +LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway +``` + +At boot the gateway calls into `litellm.proxy.read_model_list`, which reuses the +**real proxy config reader** (`ProxyConfig.get_config`). That means everything +the proxy supports in config.yaml works here too: + +- `include:` to merge in other config files, +- `os.environ/VAR` secret references (resolved via the secret manager, never + inlined), +- DB-stored models (when a database is configured). + +Secrets stay out of the config — reference them with `os.environ/...` and set +the env var at deploy time. The shipped Docker image is built with the +`python-config` feature and **bundles litellm**, so config loading works out of +the box; the default baked config lives at `/app/config.yaml` and can be +overridden at deploy time (e.g. a Render secret file mounted at the same path). + +### Environment variables + +| Var | Required | Default | Purpose | +|---|---|---|---| +| `LITELLM_CONFIG_PATH` | yes (config mode) | — | Path to the config.yaml the gateway loads its `model_list` from. The Docker image defaults this to `/app/config.yaml`. | +| `LITELLM_MASTER_KEY` | yes | — | Bearer token clients must send. Unset ⇒ all `/v1/realtime` requests are rejected (fail closed). | +| `OPENAI_API_KEY` | yes | — | Upstream OpenAI key. Referenced by config.yaml as `os.environ/OPENAI_API_KEY` for the gateway→OpenAI dial. | +| `HOST` | no | `127.0.0.1` | **Set to `0.0.0.0` in any container/deploy** or external traffic is refused. | +| `PORT` | no | `4001` | Listen port. Render and most PaaS inject this automatically. | + +> Secrets (`LITELLM_MASTER_KEY`, `OPENAI_API_KEY`) are never baked into the image +> or `render.yaml` — inject them at deploy time only. + +### Lean env stand-in (fallback) + +If the binary is built **without** `python-config` (default features), or +`LITELLM_CONFIG_PATH` is unset, the gateway falls back to a single-deployment +stand-in built from the environment: + +| Var | Default | Purpose | +|---|---|---| +| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). | + +This mode links no libpython and needs no config file, but it only supports one +hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the +stand-in only for the leanest possible build. + +## Build & run with Docker + +The image is built `--features python-config` and installs litellm **from this +repo's source** (the config reader is newer than any PyPI release), so the build +**context is the repo root**: + +```bash +# from the repo root +docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway . + +docker run --rm -p 4001:4001 \ + -e HOST=0.0.0.0 -e PORT=4001 \ + -e LITELLM_MASTER_KEY=sk-local \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + litellm-ai-gateway # LITELLM_CONFIG_PATH defaults to /app/config.yaml + +# smoke test +curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/health/readiness # -> 200 +curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/v1/realtime # -> 401 (auth fails closed) +``` + +On boot you should see `loaded model_list from /app/config.yaml via python +config reader` — that confirms the config path (not the env stand-in fallback). +To use your own config, mount it over the default: + +```bash +docker run --rm -p 4001:4001 \ + -e HOST=0.0.0.0 -e LITELLM_MASTER_KEY=sk-local -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -v $(pwd)/my-config.yaml:/app/config.yaml:ro \ + litellm-ai-gateway +``` + +### Cargo-only (no Docker) + +```bash +# config.yaml mode — needs litellm importable in the active python env +LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \ + cargo run --release -p litellm-ai-gateway --features python-config + +# env stand-in mode — no python, no config +cargo run --release -p litellm-ai-gateway +``` + +## Deploy on Render + +The service is a Docker **web service**; Render terminates TLS and supports +WebSockets, so the public endpoint is `wss://.onrender.com/v1/realtime`. + +### Option A — Blueprint (`render.yaml`) + +`crates/ai-gateway/render.yaml` describes the service (Docker runtime, +`healthCheckPath: /health/readiness`, repo-root `dockerContext: .`, +`dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile`, +`LITELLM_CONFIG_PATH: /app/config.yaml`). `LITELLM_MASTER_KEY` and +`OPENAI_API_KEY` are `sync: false` — set them in the dashboard after the first +deploy. To use a non-default model_list, mount a **Render Secret File** at +`/app/config.yaml`. Point a Render Blueprint at this repo/branch and apply. + +### Option B — Render API + +```bash +# create a Docker web service from this repo+branch, then set env vars: +curl -X POST https://api.render.com/v1/services \ + -H "Authorization: Bearer $RENDER_API_KEY" -H "Content-Type: application/json" \ + -d '{ + "type": "web_service", "name": "litellm-rust-ai-gateway", + "ownerId": "", "repo": "https://github.com/BerriAI/litellm", + "branch": "", + "serviceDetails": { + "env": "docker", + "envSpecificDetails": { + "dockerfilePath": "./litellm-rust/crates/ai-gateway/Dockerfile", + "dockerContext": "." + }, + "healthCheckPath": "/health/readiness" + } + }' +# then set env vars LITELLM_MASTER_KEY, OPENAI_API_KEY, HOST=0.0.0.0, +# LITELLM_CONFIG_PATH=/app/config.yaml +``` + +Health check path **must** be `/health/readiness`. `autoDeploy` is off by default +in the blueprint — trigger deploys manually (or flip it on) to pick up new commits. + +## Scaling + +Concurrency is what matters, not total connections: each in-flight session holds +one client socket + one upstream socket. To scale, raise the instance count / +enable autoscaling on the Render service (e.g. baseline 10, max 100). Each +instance needs file descriptors for `2 × peak_concurrent_sessions` — raise +`ulimit -n` if you push very high concurrency. + +## Latency note + +The gateway adds the cost of one extra hop: client→gateway, then a fresh +gateway→OpenAI realtime handshake (TLS + WS upgrade + `session.created`). In +benchmarks this is ~100–150 ms of added session-establishment time; first-audio +and steady-state streaming add no measurable overhead. To minimize it, deploy the +gateway in the Render region with the lowest RTT to OpenAI's realtime endpoint. diff --git a/litellm-rust/crates/ai-gateway/benchmarks/realtime/README.md b/litellm-rust/crates/ai-gateway/benchmarks/realtime/README.md new file mode 100644 index 00000000000..84e926af243 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/benchmarks/realtime/README.md @@ -0,0 +1,55 @@ +# Realtime gateway benchmark — pool on/off + +Measures what the gateway adds over talking to OpenAI's realtime WebSocket +directly, and what the pre-warmed connection pool removes. See +`../../src/routes/realtime/README.md` for how the pool works. + +## Results + +5000 calls / 500 concurrency, gateway at 10 instances, pool ON +(`REALTIME_POOL_SIZE=64`), upstream OpenAI `gpt-realtime`. Each leg run twice. +Times in **ms**. Phases per connection: **dial** = TCP+TLS+WS upgrade, +**session** = upgrade → `session.created` (the phase the pool removes), +**1st-audio** = `response.create` → first audio delta (OpenAI inference), +**total** = full wall-clock. + +| metric | Direct OpenAI | Gateway (pool ON) | Overhead (ms) | vs OpenAI | +| ------------------ | ------------- | ----------------- | ------------- | ---------- | +| success rate (%) | 99.8 | 99.8 | — | — | +| dial p50 (ms) | 276 | 158 | −118 | **faster** | +| session p50 (ms) | 7 | 0 | −7 | **faster** | +| 1st-audio p50 (ms) | 440 | 664 | +224 | slower¹ | +| total p50 (ms) | 816 | 1010 | +194 | slower¹ | +| total p95 (ms) | 2152 | 1970 | −182 | **faster** | +| total p99 (ms) | 2692 | 2610 | −82 | **faster** | + +The gateway is **faster than direct on 4 of 6 metrics**. The warm pool makes the +**session phase sub-millisecond** at the median — ~76% of connects hit the pool, +~70% had session < 1 ms. ¹ The two "slower" rows are not gateway overhead: +`1st-audio` is OpenAI's own inference time (the gateway only relays it), which ran +slower during the gateway legs and drags `total p50` with it. + +**Pool OFF** (control, `REALTIME_POOL_SIZE=0`): session p50 was **367 ms** — the +fresh-dial overhead the pool removes. + +## Reproduce + +The load generator lives in a separate repo: +**https://github.com/ishaan-berri/litellm-realtime-bench** + +```bash +git clone https://github.com/ishaan-berri/litellm-realtime-bench +cd litellm-realtime-bench && go build -o wsbench . + +# Direct to OpenAI (baseline) +./wsbench -host api.openai.com -key "$OPENAI_API_KEY" -m gpt-realtime -n 5000 -c 500 -t 60 + +# Through the gateway — run once with pool ON, once with REALTIME_POOL_SIZE=0 +./wsbench -host -key "$LITELLM_MASTER_KEY" -m gpt-realtime -n 5000 -c 500 -t 60 +``` + +Run the gateway with the env stand-in (`OPENAI_REALTIME_MODEL=gpt-realtime`, +`OPENAI_API_KEY`, `LITELLM_MASTER_KEY`, `REALTIME_POOL_SIZE`, `HOST=0.0.0.0`). At +500 concurrency over N instances, size the pool to `≈ 500 / N` per instance (64 was +used here for 10 instances). The bench repo's README covers running 500-concurrency +legs from a hosted multi-vCPU runner. **Never commit keys — pass them via `-key`.** diff --git a/litellm-rust/crates/ai-gateway/config.yaml b/litellm-rust/crates/ai-gateway/config.yaml new file mode 100644 index 00000000000..ac598c220dd --- /dev/null +++ b/litellm-rust/crates/ai-gateway/config.yaml @@ -0,0 +1,13 @@ +# Sample realtime config for the LiteLLM Rust AI Gateway. +# +# The gateway loads this model_list at boot via the embedded python config +# reader (litellm.proxy.read_model_list), which reuses the proxy's own reader — +# so include:, os.environ/ secrets, and DB-stored models all work here too. +# +# Secrets are referenced (never inlined) via os.environ/. A real deploy can +# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH). +model_list: + - model_name: gpt-realtime + litellm_params: + model: openai/gpt-realtime + api_key: os.environ/OPENAI_API_KEY diff --git a/litellm-rust/crates/ai-gateway/render.yaml b/litellm-rust/crates/ai-gateway/render.yaml new file mode 100644 index 00000000000..4170849f65d --- /dev/null +++ b/litellm-rust/crates/ai-gateway/render.yaml @@ -0,0 +1,35 @@ +# Render blueprint for the LiteLLM Rust AI Gateway (realtime WebSocket proxy). +# +# Single instance for now (no autoscaling). The public endpoint is a +# WebSocket served over TLS: wss://.onrender.com/v1/realtime +# +# Paths are relative to the **repo root** (Render's convention). The build +# context is the repo root so the image can install litellm from source — the +# gateway loads its model_list via litellm.proxy.read_model_list at boot. +# +# Secrets (LITELLM_MASTER_KEY, OPENAI_API_KEY) are marked sync: false — set +# them in the Render dashboard or via the API, never inline here. +services: + - type: web + name: litellm-rust-ai-gateway + runtime: docker + plan: standard + dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile + dockerContext: . + healthCheckPath: /health/readiness + numInstances: 1 + envVars: + # The gateway loads its model_list from this config.yaml via the embedded + # python config reader. The image bakes a default config at /app/config.yaml; + # a real deploy can override it by mounting a Render secret file at this + # same path (Dashboard → Environment → Secret Files) — never inline secrets. + - key: LITELLM_CONFIG_PATH + value: /app/config.yaml + - key: HOST + value: 0.0.0.0 + # Bearer token clients must send on /v1/realtime (fail closed if unset). + - key: LITELLM_MASTER_KEY + sync: false + # Referenced by config.yaml as os.environ/OPENAI_API_KEY for the upstream dial. + - key: OPENAI_API_KEY + sync: false diff --git a/litellm-rust/crates/ai-gateway/src/auth/mod.rs b/litellm-rust/crates/ai-gateway/src/auth/mod.rs new file mode 100644 index 00000000000..e2dd51f656d --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/auth/mod.rs @@ -0,0 +1,54 @@ +//! Gateway authentication, as an axum **extractor** (the idiomatic pattern — +//! keeps handlers clean and auth testable). +//! +//! For now this is a single **master key**: any caller presenting it as +//! `Authorization: Bearer ` may invoke the gateway. Per-key auth, budgets, +//! and rate limits are delegated to the Python proxy in a later phase. +//! +//! A handler opts in by adding [`RequireMasterKey`] to its arguments; auth then +//! runs during extraction, before the handler body. Routes never re-implement it. + +use axum::extract::FromRequestParts; +use axum::http::header::AUTHORIZATION; +use axum::http::request::Parts; +use axum::http::StatusCode; +use subtle::ConstantTimeEq; + +use crate::state::AppState; + +/// Extractor that requires the configured master key as a bearer token. +/// +/// Rejections: `500` when no master key is configured (permanent +/// misconfiguration, not a transient outage); `401` on a missing/incorrect +/// token. The comparison is constant-time. +pub struct RequireMasterKey; + +#[axum::async_trait] +impl FromRequestParts for RequireMasterKey { + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &AppState, + ) -> Result { + let Some(expected) = state.master_key.as_deref() else { + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + "gateway auth not configured (set LITELLM_MASTER_KEY)".to_string(), + )); + }; + let provided = parts + .headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .map(str::trim); + match provided { + Some(token) if bool::from(token.as_bytes().ct_eq(expected.as_bytes())) => Ok(Self), + _ => Err(( + StatusCode::UNAUTHORIZED, + "missing or invalid bearer token".to_string(), + )), + } + } +} diff --git a/litellm-rust/crates/ai-gateway/src/gil.rs b/litellm-rust/crates/ai-gateway/src/gil.rs new file mode 100644 index 00000000000..c749f722c73 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/gil.rs @@ -0,0 +1,58 @@ +//! GIL-activity tracking. +//! +//! Every acquisition of the Python GIL is recorded here so the `/health/gil` +//! endpoint can report whether Python was touched recently. The design goal is +//! that the GIL is acquired **only at load time** (config read) and never on the +//! realtime hot path — polling this endpoint during traffic should show the +//! count holding steady and `acquired_last_30s` falling to `false`. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Window (seconds) for the "recently acquired" signal. +pub const RECENT_WINDOW_SECS: u64 = 30; + +static GIL_ACQUISITIONS: AtomicU64 = AtomicU64::new(0); +/// Unix seconds of the last acquisition; `0` means "never". +static LAST_GIL_UNIX_SECS: AtomicU64 = AtomicU64::new(0); + +fn now_unix_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// Record that the GIL was just acquired. Call immediately before taking the GIL. +/// +/// Only invoked under the `python-config` feature; without it the gateway never +/// touches Python, so the recorder is unused (and the endpoint reports zero). +#[cfg_attr(not(feature = "python-config"), allow(dead_code))] +pub fn record_acquisition() { + GIL_ACQUISITIONS.fetch_add(1, Ordering::Relaxed); + LAST_GIL_UNIX_SECS.store(now_unix_secs(), Ordering::Relaxed); +} + +/// Point-in-time view of GIL activity. +pub struct GilSnapshot { + pub total_acquisitions: u64, + pub seconds_since_last: Option, + pub acquired_last_30s: bool, +} + +/// Read the current GIL-activity snapshot. +pub fn snapshot() -> GilSnapshot { + let total = GIL_ACQUISITIONS.load(Ordering::Relaxed); + let last = LAST_GIL_UNIX_SECS.load(Ordering::Relaxed); + let seconds_since_last = if last == 0 { + None + } else { + Some(now_unix_secs().saturating_sub(last)) + }; + let acquired_last_30s = seconds_since_last.is_some_and(|secs| secs <= RECENT_WINDOW_SECS); + GilSnapshot { + total_acquisitions: total, + seconds_since_last, + acquired_last_30s, + } +} diff --git a/litellm-rust/crates/ai-gateway/src/main.rs b/litellm-rust/crates/ai-gateway/src/main.rs new file mode 100644 index 00000000000..71e4a6836ad --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/main.rs @@ -0,0 +1,152 @@ +//! LiteLLM AI Gateway — a minimal Axum server fronting the Rust router. +//! +//! Flow: client → `POST /v1/realtime` → `router.realtime()` selects a deployment +//! (simple-shuffle) → `providers::realtime::realtime()` invokes OpenAI. The +//! server owns transport + config; routing lives in the `router` crate. + +mod auth; +mod gil; +#[cfg(feature = "python-config")] +mod python; +mod routes; +mod state; + +use std::sync::Arc; + +use litellm_core::router::{Deployment, LiteLLMParams, Router}; +use litellm_providers::realtime_pool::{upstream_key, PoolConfig, RealtimePool}; + +use crate::state::AppState; + +/// Bind to localhost by default so the gateway is not a public, unauthenticated +/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`). +const DEFAULT_HOST: &str = "127.0.0.1"; +const DEFAULT_PORT: u16 = 4001; + +#[tokio::main] +async fn main() { + // Trim before storing so it matches the trimmed bearer token in `auth` + // (avoids a silent auth failure when the env var has surrounding whitespace). + let master_key: Option> = std::env::var("LITELLM_MASTER_KEY") + .ok() + .map(|key| key.trim().to_string()) + .filter(|key| !key.is_empty()) + .map(Arc::from); + if master_key.is_none() { + eprintln!( + "warning: LITELLM_MASTER_KEY is not set; /v1/realtime will reject all requests (fail closed)" + ); + } + + let router = Arc::new(build_router()); + + // Build the pre-warmed realtime pool and register each deployment's upstream + // so the background replenisher starts warming it. `REALTIME_POOL_SIZE=0` + // yields a disabled pool → every connect fresh-dials (original behavior). + let pool_config = PoolConfig::from_env(); + let realtime_pool = RealtimePool::spawn(pool_config); + if pool_config.enabled() { + register_deployments(&router, &realtime_pool); + eprintln!( + "realtime connection pool enabled: target {} warm sockets/key, max idle {}s", + pool_config.target_size, + pool_config.max_idle.as_secs() + ); + } else { + eprintln!( + "realtime connection pool disabled (REALTIME_POOL_SIZE=0); fresh-dialing each connect" + ); + } + + let state = AppState { + router, + master_key, + realtime_pool, + }; + + let host = std::env::var("HOST").unwrap_or_else(|_| DEFAULT_HOST.to_string()); + let port = resolve_port(); + + let listener = tokio::net::TcpListener::bind((host.as_str(), port)) + .await + .expect("failed to bind listener"); + eprintln!("litellm-ai-gateway listening on {host}:{port}"); + axum::serve(listener, routes::app(state)) + .await + .expect("server error"); +} + +/// Register every deployment's upstream key with the pool so the replenisher +/// pre-warms it. Mirrors `service::run`'s key derivation (strip `openai/`, resolve +/// api_key); deployments whose key can't be resolved are skipped (they fresh-dial +/// and surface the auth error on the request path, as before). +fn register_deployments(router: &Router, pool: &RealtimePool) { + for deployment in router.deployments() { + let params = &deployment.litellm_params; + let provider_model = params + .model + .strip_prefix("openai/") + .unwrap_or(¶ms.model); + if let Some(key) = upstream_key( + provider_model, + params.api_key.as_deref(), + params.api_base.as_deref(), + ) { + pool.register(key); + } + } +} + +/// Resolve `PORT`, warning (rather than silently defaulting) on an invalid value. +fn resolve_port() -> u16 { + match std::env::var("PORT") { + Ok(raw) => raw.parse().unwrap_or_else(|_| { + eprintln!("warning: PORT={raw:?} is not a valid port; using {DEFAULT_PORT}"); + DEFAULT_PORT + }), + Err(_) => DEFAULT_PORT, + } +} + +/// Build the router. With the `python-config` feature and `LITELLM_CONFIG_PATH` +/// set, load the resolved `model_list` from the proxy config via the embedded +/// Python reader (load time only). Otherwise fall back to the env stand-in. +fn build_router() -> Router { + #[cfg(feature = "python-config")] + if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") { + match python::config::load_router_from_config(&config_path) { + Ok(router) => { + eprintln!("loaded model_list from {config_path} via python config reader"); + return router; + } + Err(err) => { + eprintln!("config load failed ({err}); falling back to env deployment"); + } + } + } + build_router_from_env() +} + +/// Build a minimal single-deployment `model_list` from the environment. +/// +/// A real deployment loads `model_list` from config; this is the minimal stand-in +/// so the gateway has one OpenAI deployment to route to. +fn build_router_from_env() -> Router { + let model = + std::env::var("OPENAI_REALTIME_MODEL").unwrap_or_else(|_| "gpt-realtime".to_string()); + let api_key = std::env::var("OPENAI_API_KEY").ok(); + if api_key.is_none() { + eprintln!( + "warning: OPENAI_API_KEY is not set; realtime requests will fail with auth errors" + ); + } + let deployment = Deployment { + model_name: model.clone(), + litellm_params: LiteLLMParams { + model, + api_key, + api_base: None, + }, + }; + Router::new(vec![deployment]) +} diff --git a/litellm-rust/crates/ai-gateway/src/python/AGENTS.md b/litellm-rust/crates/ai-gateway/src/python/AGENTS.md new file mode 100644 index 00000000000..47aa117e0b9 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/python/AGENTS.md @@ -0,0 +1,27 @@ +# ai-gateway/src/python — Python interop (load-time only) + +Functions here embed the Python interpreter (pyo3) and take the GIL to call into +`litellm` (e.g. read the proxy `model_list`). Compiled only under the +`python-config` feature. + +## Hard rule: non-hot-path functions only + +Everything in this folder MUST run **at most once per process lifetime — at +startup / load time** (config read, warm-up). NEVER call into Python on the +request path: + +- No GIL acquisition per request, per connection, or per realtime event. +- No Python call inside a route handler, the router's hot path, or any loop that + scales with traffic. + +**Why:** the GIL serializes execution and would cap throughput; the realtime data +path must stay pure Rust. Every acquisition is recorded by `crate::gil` — poll +`GET /health/gil`, and `total_acquisitions` MUST stay flat under load. + +## How to add one + +Resolve whatever Python-derived data you need **once at boot** and hand the rest +of the gateway an owned, plain-Rust value (e.g. build a `Router` from the +resolved `model_list`). Record the acquisition via `crate::gil::record_acquisition()` +immediately before taking the GIL. If a function would need to run per request, +it does not belong here — move the work to Rust, or pre-resolve it at startup. diff --git a/litellm-rust/crates/ai-gateway/src/python/config.rs b/litellm-rust/crates/ai-gateway/src/python/config.rs new file mode 100644 index 00000000000..6ec9595469d --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/python/config.rs @@ -0,0 +1,39 @@ +//! Build the router by calling the Python proxy config reader (load time only). +//! +//! Embeds the interpreter via pyo3 and calls +//! `litellm.proxy.read_model_list.read_model_list`, which reuses the proxy's +//! `os.environ/` + secret-manager resolution. The GIL is taken **once at boot** +//! (and recorded in [`crate::gil`]); the realtime hot path never touches Python. +//! +//! Compiled only under the `python-config` feature. + +use litellm_core::error::CoreError; +use litellm_core::router::{Deployment, Router}; +use litellm_core::CoreResult; +use pyo3::prelude::*; + +use crate::gil; + +/// Load the router's `model_list` from `config_path` via the Python reader. +pub fn load_router_from_config(config_path: &str) -> CoreResult { + gil::record_acquisition(); + Python::with_gil(|py| { + let model_list = py + .import("litellm.proxy.read_model_list") + .and_then(|module| module.getattr("read_model_list")) + .and_then(|reader| reader.call1((config_path,))) + .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?; + + let model_list_json: String = py + .import("json") + .and_then(|json| json.getattr("dumps")) + .and_then(|dumps| dumps.call1((model_list,))) + .and_then(|encoded| encoded.extract()) + .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?; + + let deployments: Vec = serde_json::from_str(&model_list_json) + .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?; + + Ok(Router::new(deployments)) + }) +} diff --git a/litellm-rust/crates/ai-gateway/src/python/mod.rs b/litellm-rust/crates/ai-gateway/src/python/mod.rs new file mode 100644 index 00000000000..a677bade676 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/python/mod.rs @@ -0,0 +1,4 @@ +//! Python interop for the gateway. See `AGENTS.md`: **load-time / non-hot-path +//! only.** Compiled only under the `python-config` feature. + +pub mod config; diff --git a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md new file mode 100644 index 00000000000..02c5f18c4f3 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md @@ -0,0 +1,38 @@ +# routes/ — the route template + +Every route follows the **same shape** so the layout is predictable. The rule: + +> **Each route module exposes `pub fn router() -> Router`.** +> `routes/mod.rs::app` merges them all and applies state once. Adding a route is: +> create the module, then add one `.merge(::router())` line. + +## Default: one file +A route is a single file containing `router()` + its handler(s) (handlers stay +private). This is the norm — don't split until it hurts. +``` +pub fn router() -> Router { Router::new().route(PATH, get(handle)) } +async fn handle(...) -> impl IntoResponse { ... } +``` +`health.rs` and `gil.rs` are examples. + +## Split out `service` when there's real logic +When a route has business logic worth testing without axum, put it in a sibling +`service` (a file, or a folder if the route grows). The route file stays the +**axum surface** (router + handler + any socket/SSE adapter); `service` is plain +Rust with **no axum types**. `realtime/` is the example: +``` +realtime/ + mod.rs # axum surface: router() + handler + the WS<->events adapter + service.rs # pure logic: select deployment + call provider (no axum) — testable +``` +Split `service` further (or add `transport`, `repo`, …) only once a single file +genuinely gets hard to read. + +## Invariants +- **Auth is an extractor, not a manual call.** A handler requires auth by adding + `crate::auth::RequireMasterKey` to its arguments; it runs during extraction. + Never re-implement the check per route. +- **Handlers contain no business logic; `service` contains no axum types.** +- A route owns its paths in its own `router()`; `mod.rs` only merges. +- Cross-cutting concerns (logging, CORS, timeouts) → Tower layers in `mod.rs`, + not duplicated in handlers. diff --git a/litellm-rust/crates/ai-gateway/src/routes/gil.rs b/litellm-rust/crates/ai-gateway/src/routes/gil.rs new file mode 100644 index 00000000000..0db0c6f0b14 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/gil.rs @@ -0,0 +1,30 @@ +//! `GET /health/gil` — poll to confirm Python is only touched at load time. +//! Simple-route template: a `router()` plus its handler, in one file. + +use axum::routing::get; +use axum::{Json, Router}; +use serde::Serialize; + +use crate::gil; +use crate::state::AppState; + +/// This route's contribution to the app router. +pub fn router() -> Router { + Router::new().route("/health/gil", get(status)) +} + +#[derive(Debug, Serialize)] +struct GilStatusResponse { + gil_acquired_last_30s: bool, + total_acquisitions: u64, + seconds_since_last: Option, +} + +async fn status() -> Json { + let snapshot = gil::snapshot(); + Json(GilStatusResponse { + gil_acquired_last_30s: snapshot.acquired_last_30s, + total_acquisitions: snapshot.total_acquisitions, + seconds_since_last: snapshot.seconds_since_last, + }) +} diff --git a/litellm-rust/crates/ai-gateway/src/routes/health.rs b/litellm-rust/crates/ai-gateway/src/routes/health.rs new file mode 100644 index 00000000000..15c67fea325 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/health.rs @@ -0,0 +1,24 @@ +//! Health probes. Simple-route template: a `router()` plus its handlers, in one file. + +use axum::http::StatusCode; +use axum::routing::get; +use axum::Router; + +use crate::state::AppState; + +/// This route's contribution to the app router. +pub fn router() -> Router { + Router::new() + .route("/health/liveness", get(liveness)) + .route("/health/readiness", get(readiness)) +} + +/// The process is up. +async fn liveness() -> StatusCode { + StatusCode::OK +} + +/// The server is ready to accept traffic. +async fn readiness() -> StatusCode { + StatusCode::OK +} diff --git a/litellm-rust/crates/ai-gateway/src/routes/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/mod.rs new file mode 100644 index 00000000000..c6b9573781a --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/mod.rs @@ -0,0 +1,23 @@ +//! HTTP routes. +//! +//! **Template:** every route module exposes `pub fn router() -> Router` +//! that mounts its own paths; [`app`] merges them. A trivial route is a single +//! file (`health.rs`, `gil.rs`); a non-trivial one is a folder (`realtime/`) with +//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md. + +pub mod gil; +pub mod health; +pub mod realtime; + +use axum::Router; + +use crate::state::AppState; + +/// Assemble the application router by merging every route module's `router()`. +pub fn app(state: AppState) -> Router { + Router::new() + .merge(health::router()) + .merge(gil::router()) + .merge(realtime::router()) + .with_state(state) +} diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md b/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md new file mode 100644 index 00000000000..3301576bb85 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md @@ -0,0 +1,87 @@ +# Realtime route (`GET /v1/realtime`) + +Proxies OpenAI's realtime WebSocket. `mod.rs` is the axum surface (handler + +socket↔events adapter); `service.rs` is the pure logic (select a deployment, then +splice client ↔ upstream). The pool itself lives in +`crates/providers/src/realtime_pool.rs`. + +## Connection pooling + +### The problem + +The gateway's realtime overhead lives **entirely in session establishment**. On each +client connect it dials a *fresh* upstream WS to OpenAI and waits for +`session.created` before it can serve. Measured at 5000 calls / 500 concurrency, the +fresh-dial session phase is **~360 ms** vs **~7 ms** direct; dial, first-audio, and +streaming add ~0. So the one lever is removing that per-connect handshake from the +critical path. + +### The idea + +Keep a few upstream OpenAI sockets **already connected and already past +`session.created`** (buffered). On a client connect, hand off a warm socket — relay +its buffered `session.created` instantly (a local `Vec::pop`, sub-millisecond) and +splice exactly as a fresh dial would. A background task keeps the pool topped up. On +a miss or dead socket we fall back to fresh-dial: the pool is a latency optimization, +never a correctness dependency. + +``` + ┌───────────────────────────────────────┐ + client connect ──────► │ routes/realtime → service::run │ + │ pool.take(key) │ + │ hit → relay buffered │ + │ session.created, then splice │ + │ miss → fresh dial (original path) │ + └───────────────┬───────────────────────┘ + │ replenish (async, concurrent) + ┌───────────────▼───────────────────────┐ + background task ─────► │ RealtimePool: per-key warm sockets │ + │ each = { ws, buffered session.created}│ + │ liveness-checked before handoff │ + └─────────────────────────────────────────┘ +``` + +A warm session is indistinguishable from a fresh one: OpenAI sends `session.created` +unprompted on connect, we pre-read exactly that one frame and relay it on handoff, +and we send nothing else on the socket before a client exists — so the client's first +`session.update` behaves identically either way. + +### Sizing + +Each warm socket serves **exactly one** session (realtime isn't multiplexed), so the +pool is sized to the **peak concurrent connects per instance**, not total live +connections: + +``` +REALTIME_POOL_SIZE ≈ peak_concurrency / instance_count +``` + +e.g. 500 concurrency over 10 instances → ~50–64 per instance. The replenisher dials +the missing sockets **concurrently**, so a drained pool refills in ~one handshake +window and keeps supply close to the connect rate. Over-provisioning just burns idle +upstream sockets, which is why warm sockets are short-lived +(`REALTIME_POOL_MAX_IDLE_SECS`). + +### Config + +| env | default | meaning | +| ----------------------------- | ------- | --------------------------------------------------------------- | +| `REALTIME_POOL_SIZE` | `4` | target warm sockets per key. `0` disables pooling (fresh-dial). | +| `REALTIME_POOL_MAX_IDLE_SECS` | `30` | max time a warm socket sits before it's closed and replaced. | + +### Notes + +- **Miss / dead socket → fresh dial.** Burst beyond warm supply, or a socket that + died, never blocks or fails — it falls back to the original path. The pool can only + make a connect faster, never slower or more fragile. +- **Auth scope.** The pool key includes `api_key`, so a warm socket is only handed to + a request resolving to the same key — no cross-tenant reuse. +- **Idle billing.** Warm sockets are liveness-checked at handoff and capped at + `REALTIME_POOL_MAX_IDLE_SECS` to bound idle billing and dodge OpenAI's idle timeout. +- **Replenish backoff.** If a key's warm-up dials all fail (invalid credentials, an + unreachable upstream), the replenisher puts that key into exponential backoff + (500 ms → 30 s cap) instead of re-dialing it every tick. This bounds connection + attempts against a broken key so it can't exhaust upstream rate limits and degrade + valid cold-path traffic; the backoff resets the moment a dial succeeds. + +Benchmarks and repro: `../../benchmarks/realtime/README.md`. diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs new file mode 100644 index 00000000000..695e0c6bb39 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs @@ -0,0 +1,88 @@ +//! `GET /v1/realtime` (WebSocket). +//! +//! This file is the **axum surface**: `router()`, the handler, and the small +//! socket↔events adapter. The pure logic (no axum) lives in [`service`]. Auth is +//! the `RequireMasterKey` extractor, so the handler stays thin. + +mod service; + +use std::sync::Arc; + +use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::Response; +use axum::routing::get; +use axum::Router; +use futures_util::{SinkExt, StreamExt}; +use litellm_core::realtime::types::RealtimeEvent; +use litellm_core::router::Router as ModelRouter; +use litellm_providers::realtime_pool::RealtimePool; +use serde::Deserialize; + +use crate::auth::RequireMasterKey; +use crate::state::AppState; + +/// This route's contribution to the app router. +pub fn router() -> Router { + Router::new().route("/v1/realtime", get(handle)) +} + +#[derive(Debug, Deserialize)] +struct RealtimeQuery { + model: String, +} + +/// Auth runs via the `RequireMasterKey` extractor. We validate the model BEFORE +/// the upgrade so failures are clean HTTP (400/404), not a socket that opens then +/// closes, then hand the socket to `bridge`. +async fn handle( + _auth: RequireMasterKey, + ws: WebSocketUpgrade, + State(state): State, + Query(query): Query, +) -> Result { + if query.model.trim().is_empty() { + return Err(( + StatusCode::BAD_REQUEST, + "missing 'model' query param".to_string(), + )); + } + if !state.router.has_deployment(&query.model) { + return Err(( + StatusCode::NOT_FOUND, + format!("no deployment for model '{}'", query.model), + )); + } + + let router = state.router.clone(); + let pool = state.realtime_pool.clone(); + let model = query.model; + Ok(ws.on_upgrade(move |socket| bridge(socket, router, pool, model))) +} + +/// Adapt the axum socket (text frames) to the typed-event `Stream`/`Sink` the +/// service wants, keeping axum types out of `service`. +async fn bridge( + socket: WebSocket, + router: Arc, + pool: Arc, + model: String, +) { + let (ws_sink, ws_stream) = socket.split(); + + let client_in = ws_stream.filter_map(|message| async move { + match message { + Ok(Message::Text(text)) => serde_json::from_str::(&text).ok(), + _ => None, + } + }); + let client_out = ws_sink.with(|event: RealtimeEvent| async move { + Ok::(Message::Text( + serde_json::to_string(&event).unwrap_or_default(), + )) + }); + + futures_util::pin_mut!(client_in, client_out); + let _ = service::run(&router, &pool, &model, None, client_in, client_out).await; +} diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs new file mode 100644 index 00000000000..0cbd00d664f --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -0,0 +1,76 @@ +//! Business logic: select a deployment with the (pure) core router, then call the +//! provider splice. The seam between `core::router` (selection only) and +//! `providers` (the actual WebSocket I/O). +//! +//! On connect we try a pre-warmed upstream from the pool (handshake already paid, +//! `session.created` buffered) and relay it instantly. On a pool miss or dead warm +//! socket we fresh-dial exactly as before — the pool is never on the critical path +//! for correctness, only latency. + +use std::time::Duration; + +use futures_util::{Sink, Stream}; +use litellm_core::error::CoreError; +use litellm_core::realtime::types::RealtimeEvent; +use litellm_core::router::Router; +use litellm_core::CoreResult; +use litellm_providers::realtime_pool::{upstream_key, RealtimePool}; + +/// Select a deployment for `model` and splice the client stream to the provider. +/// +/// `pool` supplies a pre-warmed upstream when one is available; otherwise we +/// fresh-dial. A disabled pool always misses, so this collapses to the original +/// fresh-dial behavior. +pub async fn run( + router: &Router, + pool: &RealtimePool, + model: &str, + idle_timeout: Option, + client_in: In, + client_out: Out, +) -> CoreResult<()> +where + In: Stream + Unpin + Send, + Out: Sink + Unpin + Send, + >::Error: std::fmt::Display, +{ + let deployment = router.get_available_deployment(model).ok_or_else(|| { + CoreError::Routing(format!("no deployment available for model '{model}'")) + })?; + let params = &deployment.litellm_params; + // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. + let provider_model = params + .model + .strip_prefix("openai/") + .unwrap_or(¶ms.model); + + // Warm path: take a pooled upstream (handshake already paid) and relay its + // buffered session.created immediately. On miss/dead socket fall through. + if let Some(key) = upstream_key( + provider_model, + params.api_key.as_deref(), + params.api_base.as_deref(), + ) { + if let Some(handoff) = pool.take(&key) { + return litellm_providers::realtime::realtime_warm( + provider_model, + handoff, + idle_timeout, + client_in, + client_out, + ) + .await; + } + } + + // Cold path: fresh dial (the original behavior). + litellm_providers::realtime::realtime( + provider_model, + params.api_key.as_deref(), + params.api_base.as_deref(), + idle_timeout, + client_in, + client_out, + ) + .await +} diff --git a/litellm-rust/crates/ai-gateway/src/state.rs b/litellm-rust/crates/ai-gateway/src/state.rs new file mode 100644 index 00000000000..ef96037d477 --- /dev/null +++ b/litellm-rust/crates/ai-gateway/src/state.rs @@ -0,0 +1,17 @@ +use std::sync::Arc; + +use litellm_core::router::Router; +use litellm_providers::realtime_pool::RealtimePool; + +/// Shared application state handed to every route handler. +#[derive(Clone)] +pub struct AppState { + pub router: Arc, + /// The gateway master key. Any caller presenting it as a bearer token may + /// invoke the gateway. `None` → auth not configured (routes fail closed). + pub master_key: Option>, + /// Pre-warmed upstream realtime connection pool. Disabled + /// (`RealtimePool::disabled()`) when `REALTIME_POOL_SIZE=0`, in which case + /// every realtime connect fresh-dials exactly as before. + pub realtime_pool: Arc, +} diff --git a/litellm-rust/crates/core/CLAUDE.md b/litellm-rust/crates/core/CLAUDE.md new file mode 100644 index 00000000000..20873878967 --- /dev/null +++ b/litellm-rust/crates/core/CLAUDE.md @@ -0,0 +1,47 @@ +# CLAUDE.md + +Rules for `litellm-rust/crates/core`. + +## Responsibility + +`core` owns shared data types, typed errors, and deterministic helper contracts. +It must stay pure and host-independent. + +Allowed: +- Shared request/response structs. +- Typed errors with stable, non-sensitive messages. +- Deterministic validation helpers. +- Serialization helpers that intentionally mirror Python output shape. +- Route templates that match Python base config responsibilities, such as + `ocr::transformation::OcrProviderConfig`. + +Not allowed: +- Network, filesystem, database, cache, or environment access. +- Secret reads or auth/header construction. +- Logging callbacks, tracing spans, spend writes, or customer callbacks. +- Provider-specific branching that belongs in `providers`. +- Panics for user/provider-controlled input. + +## Typed Contracts (core rule) + +Trait and function boundaries MUST be strongly typed. No stringly-typed JSON +(`&str` / `String` / `Vec` / bare `serde_json::Value`) as a transform +input or output. Parse wire bytes into typed structs/enums at the host edge; +`core` and `providers` operate only on those types (e.g. `RealtimeEvent`, +`RealtimeTransformResult`, `OcrRequestData`). A `type`-style discriminator is a +typed field on a struct, not a raw string threaded through the API. + +## Structure + +Use route names directly under `src/`: `ocr`, future `messages`, +`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not +invent broad names like `engine` for route contracts. + +## Parity Rules + +- Every shared type used by a provider transform needs unit tests for + serialization shape. +- If Python parity requires always emitting a `null` field instead of omitting + it, document that in code and pin it with a test. +- Error enums should preserve enough detail for Python/HTTP hosts to map errors + consistently without exposing document contents or upstream bodies. diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml new file mode 100644 index 00000000000..1881bcfa602 --- /dev/null +++ b/litellm-rust/crates/core/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "litellm-core" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +rand.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs new file mode 100644 index 00000000000..9b29260cca4 --- /dev/null +++ b/litellm-rust/crates/core/src/error.rs @@ -0,0 +1,35 @@ +use thiserror::Error; + +pub type CoreResult = Result; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CoreError { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("{0}")] + Auth(String), + #[error("OCR request failed with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("OCR network error: {0}")] + Network(String), + #[error("routing error: {0}")] + Routing(String), +} + +pub fn json_type_name(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs new file mode 100644 index 00000000000..9d686626edc --- /dev/null +++ b/litellm-rust/crates/core/src/lib.rs @@ -0,0 +1,6 @@ +pub mod error; +pub mod ocr; +pub mod realtime; +pub mod router; + +pub use error::{CoreError, CoreResult}; diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs new file mode 100644 index 00000000000..ec2fbb969a6 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -0,0 +1,2 @@ +pub mod transformation; +pub mod types; diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs new file mode 100644 index 00000000000..7353d9d22c4 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -0,0 +1,32 @@ +use serde_json::{Map, Value}; + +use crate::CoreResult; + +use super::types::{OcrRequestData, OcrResponseData}; + +pub trait OcrProviderConfig { + fn supported_ocr_params(&self) -> &'static [&'static str]; + + fn map_ocr_params(&self, non_default_params: &Map) -> Map { + let mut mapped_params = Map::new(); + for (param, value) in non_default_params { + if self.supported_ocr_params().contains(¶m.as_str()) { + mapped_params.insert(param.clone(), value.clone()); + } + } + mapped_params + } + + fn transform_ocr_request( + &self, + model: &str, + document: Value, + optional_params: Map, + ) -> CoreResult; + + fn transform_ocr_response( + &self, + model: &str, + response_json: Value, + ) -> CoreResult; +} diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs new file mode 100644 index 00000000000..1a72b8f1d66 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct OcrRequestData { + pub data: Value, + pub files: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct OcrResponseData { + pub pages: Vec, + pub model: String, + pub document_annotation: Option, + pub usage_info: Option, + pub object: String, +} + +impl OcrResponseData { + pub fn into_json(self) -> Value { + serde_json::json!({ + "pages": self.pages, + "model": self.model, + "document_annotation": self.document_annotation, + "usage_info": self.usage_info, + "object": self.object, + }) + } +} diff --git a/litellm-rust/crates/core/src/realtime/mod.rs b/litellm-rust/crates/core/src/realtime/mod.rs new file mode 100644 index 00000000000..ec2fbb969a6 --- /dev/null +++ b/litellm-rust/crates/core/src/realtime/mod.rs @@ -0,0 +1,2 @@ +pub mod transformation; +pub mod types; diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs new file mode 100644 index 00000000000..a4baa27a6c2 --- /dev/null +++ b/litellm-rust/crates/core/src/realtime/transformation.rs @@ -0,0 +1,22 @@ +use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; +use crate::CoreResult; + +pub trait RealtimeProviderConfig { + /// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`). + /// Pure string construction only — no network, no env. + fn complete_url(&self, api_base: Option<&str>, model: &str) -> String; + + /// Transform a client → backend event before it is forwarded upstream. + fn transform_realtime_request( + &self, + event: &RealtimeEvent, + model: &str, + ) -> CoreResult; + + /// Transform a backend → client event before it is forwarded downstream. + fn transform_realtime_response( + &self, + event: &RealtimeEvent, + model: &str, + ) -> CoreResult; +} diff --git a/litellm-rust/crates/core/src/realtime/types.rs b/litellm-rust/crates/core/src/realtime/types.rs new file mode 100644 index 00000000000..3b59224b6e9 --- /dev/null +++ b/litellm-rust/crates/core/src/realtime/types.rs @@ -0,0 +1,60 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A single realtime event exchanged over the WebSocket. +/// +/// The `type` discriminator is a typed field; the remaining fields are +/// preserved losslessly in `data` so a transform can pass an event through, or +/// inspect/modify specific fields, without enumerating every event variant. +/// Wire (de)serialization happens at the host edge — `core`/`providers` operate +/// only on this typed form. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RealtimeEvent { + #[serde(rename = "type")] + pub event_type: String, + #[serde(flatten)] + pub data: Map, +} + +/// One or more typed events produced by a realtime transform. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RealtimeTransformResult { + pub events: Vec, +} + +impl RealtimeTransformResult { + /// Forward a single event unchanged (the OpenAI baseline). + pub fn passthrough(event: RealtimeEvent) -> Self { + Self { + events: vec![event], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(raw: &str) -> RealtimeEvent { + serde_json::from_str(raw).expect("valid event json") + } + + #[test] + fn realtime_event_round_trips_type_and_extra_fields() { + let raw = r#"{"type":"response.output_text.delta","delta":"hi","response_id":"r1"}"#; + let parsed = event(raw); + assert_eq!(parsed.event_type, "response.output_text.delta"); + assert_eq!(parsed.data.get("delta"), Some(&Value::String("hi".into()))); + // Re-serializing yields a semantically-equal event (key order may differ). + let reparsed: RealtimeEvent = + serde_json::from_str(&serde_json::to_string(&parsed).unwrap()).unwrap(); + assert_eq!(parsed, reparsed); + } + + #[test] + fn passthrough_produces_single_element_vec() { + let parsed = event(r#"{"type":"session.update"}"#); + let result = RealtimeTransformResult::passthrough(parsed.clone()); + assert_eq!(result.events, vec![parsed]); + } +} diff --git a/litellm-rust/crates/core/src/router/deployment.rs b/litellm-rust/crates/core/src/router/deployment.rs new file mode 100644 index 00000000000..1ee88e682a3 --- /dev/null +++ b/litellm-rust/crates/core/src/router/deployment.rs @@ -0,0 +1,44 @@ +//! `model_list` data types, mirroring Python's deployment dict. Deserialize-ready +//! so a deployment can be loaded straight from the proxy config's `model_list`. + +use serde::Deserialize; + +/// Per-deployment call parameters, mirroring Python's `litellm_params`. +#[derive(Clone, Debug, Deserialize)] +pub struct LiteLLMParams { + /// Provider model, e.g. `gpt-realtime` or `openai/gpt-realtime`. + pub model: String, + #[serde(default)] + pub api_key: Option, + #[serde(default)] + pub api_base: Option, +} + +/// One entry of the `model_list`, mirroring Python's deployment dict. +#[derive(Clone, Debug, Deserialize)] +pub struct Deployment { + /// Public alias clients request, e.g. `gpt-realtime`. + pub model_name: String, + pub litellm_params: LiteLLMParams, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deserializes_from_model_list_entry() { + let entry = r#"{ + "model_name": "gpt-realtime", + "litellm_params": {"model": "openai/gpt-realtime", "api_base": "https://x"} + }"#; + let deployment: Deployment = serde_json::from_str(entry).expect("valid entry"); + assert_eq!(deployment.model_name, "gpt-realtime"); + assert_eq!(deployment.litellm_params.model, "openai/gpt-realtime"); + assert_eq!(deployment.litellm_params.api_key, None); + assert_eq!( + deployment.litellm_params.api_base.as_deref(), + Some("https://x") + ); + } +} diff --git a/litellm-rust/crates/core/src/router/mod.rs b/litellm-rust/crates/core/src/router/mod.rs new file mode 100644 index 00000000000..96bc91bc6b5 --- /dev/null +++ b/litellm-rust/crates/core/src/router/mod.rs @@ -0,0 +1,93 @@ +//! Minimal Rust port of LiteLLM's `router.py` deployment selection. +//! +//! A [`Router`] is built from a `model_list` of [`Deployment`]s +//! (`{ model_name, litellm_params: { model, api_key, api_base } }`) and selects +//! one per request via a [`RoutingStrategy`]. For now the only strategy is +//! `simple-shuffle` — a uniform random pick within a `model_name` group. +//! +//! This stays pure (no I/O): it only *chooses* a deployment. The host (the +//! gateway) takes the chosen deployment and performs the actual provider call. +//! +//! - [`deployment`] — the `model_list` data types. +//! - [`strategy`] — how a deployment is chosen. + +mod deployment; +mod strategy; + +pub use deployment::{Deployment, LiteLLMParams}; +pub use strategy::RoutingStrategy; + +/// Load-balancing router over a `model_list`. +#[derive(Clone, Debug, Default)] +pub struct Router { + model_list: Vec, + routing_strategy: RoutingStrategy, +} + +impl Router { + /// Build a router from a `model_list` using the default `simple-shuffle` strategy. + pub fn new(model_list: Vec) -> Self { + Self { + model_list, + routing_strategy: RoutingStrategy::SimpleShuffle, + } + } + + /// All deployments in the `model_list`. Read-only; used by the host to + /// enumerate upstreams (e.g. to pre-warm a connection pool per deployment). + pub fn deployments(&self) -> &[Deployment] { + &self.model_list + } + + /// Whether any deployment is registered under `model`. + pub fn has_deployment(&self, model: &str) -> bool { + self.model_list + .iter() + .any(|deployment| deployment.model_name == model) + } + + /// Pick a deployment for `model` per the routing strategy. Returns `None` + /// when no deployment is registered under that `model_name`. + pub fn get_available_deployment(&self, model: &str) -> Option<&Deployment> { + let candidates: Vec<&Deployment> = self + .model_list + .iter() + .filter(|deployment| deployment.model_name == model) + .collect(); + self.routing_strategy.select(&candidates) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn deployment(name: &str, model: &str) -> Deployment { + Deployment { + model_name: name.to_string(), + litellm_params: LiteLLMParams { + model: model.to_string(), + api_key: None, + api_base: None, + }, + } + } + + #[test] + fn selects_a_matching_deployment() { + let router = Router::new(vec![ + deployment("gpt-realtime", "gpt-realtime"), + deployment("other", "other-model"), + ]); + let chosen = router + .get_available_deployment("gpt-realtime") + .expect("a deployment should match"); + assert_eq!(chosen.model_name, "gpt-realtime"); + } + + #[test] + fn unknown_model_returns_none() { + let router = Router::new(vec![deployment("gpt-realtime", "gpt-realtime")]); + assert!(router.get_available_deployment("missing").is_none()); + } +} diff --git a/litellm-rust/crates/core/src/router/strategy/mod.rs b/litellm-rust/crates/core/src/router/strategy/mod.rs new file mode 100644 index 00000000000..7e8ac217db3 --- /dev/null +++ b/litellm-rust/crates/core/src/router/strategy/mod.rs @@ -0,0 +1,26 @@ +//! Routing policy: how the router picks one deployment from a model group. +//! +//! One module per strategy; [`RoutingStrategy::select`] dispatches to it. New +//! strategies (least-busy, latency-based, …) get their own file here. + +mod simple_shuffle; + +use super::Deployment; + +/// How the router chooses among the deployments sharing a `model_name`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum RoutingStrategy { + /// Uniform random pick among the matching deployments. + #[default] + SimpleShuffle, +} + +impl RoutingStrategy { + /// Choose one deployment from `candidates` (all sharing the requested + /// `model_name`). Returns `None` when there are no candidates. + pub fn select<'a>(&self, candidates: &[&'a Deployment]) -> Option<&'a Deployment> { + match self { + RoutingStrategy::SimpleShuffle => simple_shuffle::select(candidates), + } + } +} diff --git a/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs b/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs new file mode 100644 index 00000000000..74ce0c21e80 --- /dev/null +++ b/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs @@ -0,0 +1,47 @@ +//! `simple-shuffle`: a uniform random pick among the candidate deployments. + +use rand::seq::SliceRandom; + +use crate::router::Deployment; + +/// Uniform random choice among `candidates` (all sharing the requested +/// `model_name`). Returns `None` when there are no candidates. +pub fn select<'a>(candidates: &[&'a Deployment]) -> Option<&'a Deployment> { + candidates.choose(&mut rand::thread_rng()).copied() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::router::{Deployment, LiteLLMParams}; + + fn deployment(model: &str) -> Deployment { + Deployment { + model_name: "gpt-realtime".to_string(), + litellm_params: LiteLLMParams { + model: model.to_string(), + api_key: None, + api_base: None, + }, + } + } + + #[test] + fn picks_from_candidates() { + let a = deployment("key-a"); + let b = deployment("key-b"); + let candidates = vec![&a, &b]; + for _ in 0..20 { + let chosen = select(&candidates).expect("non-empty"); + assert!(matches!( + chosen.litellm_params.model.as_str(), + "key-a" | "key-b" + )); + } + } + + #[test] + fn empty_candidates_select_none() { + assert!(select(&[]).is_none()); + } +} diff --git a/litellm-rust/crates/providers/CLAUDE.md b/litellm-rust/crates/providers/CLAUDE.md new file mode 100644 index 00000000000..0f7fdcda2aa --- /dev/null +++ b/litellm-rust/crates/providers/CLAUDE.md @@ -0,0 +1,53 @@ +# CLAUDE.md + +Rules for `litellm-rust/crates/providers`. + +## Responsibility + +`providers` owns provider-specific pure transforms. It mirrors the existing +Python provider modules closely enough that parity review is mechanical. + +Provider files should map to the Python provider tree: + +```text +providers/src///transformation.rs +``` + +For example, Mistral OCR lives at +`providers/src/mistral/ocr/transformation.rs`, matching +`litellm/llms/mistral/ocr/transformation.py`. + +Allowed: +- Provider request transforms. +- Provider response normalization. +- Supported-parameter filtering. +- Provider-specific validation that does not require I/O or secrets. + +Not allowed: +- HTTP clients or provider SDK calls. +- Environment variable reads. +- API key resolution or auth header construction. +- Logging, callbacks, spend tracking, retries, routing, cooldowns, or fallbacks. +- Panics on bad user/provider input. + +## Required Tests + +Every provider transform must include focused unit tests for: +- Supported params matching the Python provider config. +- Unknown params being dropped or transformed the same way as Python. +- Request body shape matching Python output. +- Response normalization with complete, missing, null, and extra fields. +- Bad input returning typed errors. + +For OCR specifically, assume documents can contain personal data. Tests should +prove transforms do not copy document contents into error messages. + +## Implementation Rules + +- Prefer static supported-parameter lists over allocating strings on every call. +- Keep transforms deterministic and allocation-conscious, but choose clarity over + premature micro-optimization for tiny parameter lists. +- Use typed errors from `core`; avoid stringly-typed error plumbing. +- Add comments only when they explain Python-parity decisions or provider quirks. +- Put route-level provider dispatch in a route file such as `providers/src/ocr.rs`. + Do not move provider-specific transform logic into the Python bridge. diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/providers/Cargo.toml new file mode 100644 index 00000000000..c5b41424d66 --- /dev/null +++ b/litellm-rust/crates/providers/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "litellm-providers" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-core.workspace = true +reqwest.workspace = true +serde_json.workspace = true +tokio.workspace = true +tokio-tungstenite.workspace = true +futures-util.workspace = true + +[dev-dependencies] +serde_json.workspace = true +futures-channel = "0.3" diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/providers/src/lib.rs new file mode 100644 index 00000000000..40e18961f43 --- /dev/null +++ b/litellm-rust/crates/providers/src/lib.rs @@ -0,0 +1,5 @@ +pub mod mistral; +pub mod ocr; +pub mod openai; +pub mod realtime; +pub mod realtime_pool; diff --git a/litellm-rust/crates/providers/src/mistral/mod.rs b/litellm-rust/crates/providers/src/mistral/mod.rs new file mode 100644 index 00000000000..3621ff6a2fd --- /dev/null +++ b/litellm-rust/crates/providers/src/mistral/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/providers/src/mistral/ocr/mod.rs b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/providers/src/mistral/ocr/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs new file mode 100644 index 00000000000..fd691177783 --- /dev/null +++ b/litellm-rust/crates/providers/src/mistral/ocr/transformation.rs @@ -0,0 +1,292 @@ +use litellm_core::error::{json_type_name, CoreError, CoreResult}; +use litellm_core::ocr::transformation::OcrProviderConfig; +use litellm_core::ocr::types::{OcrRequestData, OcrResponseData}; +use serde_json::{Map, Value}; + +const SUPPORTED_OCR_PARAMS: &[&str] = &[ + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "id", +]; + +/// Default Mistral API base, used when the caller does not override `api_base`. +pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1"; + +/// Environment variable holding the Mistral API key. +pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; + +/// Error message raised when no Mistral API key can be resolved. +pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"; + +/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`. +/// +/// Blank/whitespace `api_base` is treated as absent (guard at resolution time). +pub fn complete_url(api_base: Option<&str>) -> String { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(MISTRAL_DEFAULT_API_BASE) + .trim_end_matches('/'); + + if base.ends_with("/v1") { + format!("{base}/ocr") + } else { + format!("{base}/v1/ocr") + } +} + +/// Resolve the Mistral API key from the explicit param or the environment. +/// +/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` +/// when no usable key is available. +/// +/// Note: the env fallback only reads the process environment. Secret-manager +/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in +/// via `api_key`; this fallback is a last resort for direct/standalone use. +pub fn resolve_api_key( + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> CoreResult { + api_key + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) + .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) +} + +pub struct MistralOcrConfig; + +pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig; + +impl OcrProviderConfig for MistralOcrConfig { + fn supported_ocr_params(&self) -> &'static [&'static str] { + SUPPORTED_OCR_PARAMS + } + + fn transform_ocr_request( + &self, + model: &str, + document: Value, + optional_params: Map, + ) -> CoreResult { + if !document.is_object() { + return Err(CoreError::InvalidType { + expected: "object", + actual: json_type_name(&document), + }); + } + + let mut data = Map::new(); + data.insert("model".to_string(), Value::String(model.to_string())); + data.insert("document".to_string(), document); + for (param, value) in optional_params { + data.insert(param, value); + } + + Ok(OcrRequestData { + data: Value::Object(data), + files: None, + }) + } + + fn transform_ocr_response( + &self, + model: &str, + response_json: Value, + ) -> CoreResult { + let response_object = response_json + .as_object() + .ok_or_else(|| CoreError::InvalidType { + expected: "object", + actual: json_type_name(&response_json), + })?; + + let pages = response_object + .get("pages") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let model = response_object + .get("model") + .and_then(Value::as_str) + .unwrap_or(model) + .to_string(); + let document_annotation = response_object.get("document_annotation").cloned(); + let usage_info = response_object.get("usage_info").cloned(); + + Ok(OcrResponseData { + pages, + model, + document_annotation, + usage_info, + object: "ocr".to_string(), + }) + } +} + +pub fn supported_ocr_params() -> &'static [&'static str] { + MISTRAL_OCR_CONFIG.supported_ocr_params() +} + +pub fn map_ocr_params(non_default_params: &Map) -> Map { + MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params) +} + +pub fn transform_ocr_request( + model: &str, + document: Value, + optional_params: Map, +) -> CoreResult { + MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) +} + +pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult { + MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn supported_params_match_python_mistral_ocr_config() { + assert_eq!( + supported_ocr_params(), + &[ + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "id", + ] + ); + } + + #[test] + fn map_ocr_params_drops_unknown_params() { + let params = json!({ + "extract_header": true, + "unsupported_param": "value", + "pages": [0, 1] + }); + let mapped = map_ocr_params(params.as_object().unwrap()); + + assert_eq!(mapped.get("extract_header"), Some(&json!(true))); + assert_eq!(mapped.get("pages"), Some(&json!([0, 1]))); + assert!(!mapped.contains_key("unsupported_param")); + } + + #[test] + fn transform_ocr_request_builds_mistral_body() { + let document = json!({ + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }); + let optional_params = json!({ + "include_image_base64": true, + "table_format": "html" + }) + .as_object() + .unwrap() + .clone(); + + let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params) + .expect("request should transform"); + + assert_eq!( + result.data, + json!({ + "model": "mistral-ocr-latest", + "document": document, + "include_image_base64": true, + "table_format": "html" + }) + ); + assert_eq!(result.files, None); + } + + #[test] + fn transform_ocr_request_rejects_non_object_document() { + let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new()) + .expect_err("string document should be rejected"); + + assert_eq!( + err, + CoreError::InvalidType { + expected: "object", + actual: "string", + } + ); + } + + #[test] + fn transform_ocr_response_normalizes_mistral_json() { + let response = json!({ + "pages": [{"index": 0, "markdown": "hello"}], + "model": "mistral-ocr-2505-completion", + "document_annotation": null, + "usage_info": {"pages_processed": 1} + }); + + let result = transform_ocr_response("mistral-ocr-latest", response) + .expect("response should transform"); + + assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]); + assert_eq!(result.model, "mistral-ocr-2505-completion"); + assert_eq!(result.document_annotation, Some(Value::Null)); + assert_eq!(result.usage_info, Some(json!({"pages_processed": 1}))); + assert_eq!(result.object, "ocr"); + } + + #[test] + fn complete_url_defaults_and_dedupes_v1() { + assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr"); + assert_eq!(complete_url(Some(" ")), "https://api.mistral.ai/v1/ocr"); + assert_eq!( + complete_url(Some("https://proxy.internal")), + "https://proxy.internal/v1/ocr" + ); + assert_eq!( + complete_url(Some("https://proxy.internal/v1/")), + "https://proxy.internal/v1/ocr" + ); + } + + #[test] + fn resolve_api_key_prefers_param_then_env() { + let no_env = |_: &str| None; + assert_eq!( + resolve_api_key(Some("sk-param"), &no_env).unwrap(), + "sk-param" + ); + + let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string()); + assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env"); + // Blank param falls through to the environment. + assert_eq!(resolve_api_key(Some(" "), &with_env).unwrap(), "sk-env"); + } + + #[test] + fn resolve_api_key_errors_when_absent() { + let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); + assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string())); + } +} diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs new file mode 100644 index 00000000000..dcd56a5f0b4 --- /dev/null +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -0,0 +1,127 @@ +//! End-to-end OCR orchestration. +//! +//! Owns the whole Mistral OCR call so the Python side stays a thin bridge: +//! resolve the API key, build the URL + body via the pure transforms, POST it, +//! and normalize the response. The HTTP client is built once and reused. + +use std::sync::OnceLock; +use std::time::Duration; + +use litellm_core::error::CoreError; +use litellm_core::ocr::transformation::OcrProviderConfig; +use litellm_core::CoreResult; +use serde_json::{Map, Value}; + +use crate::mistral::ocr::transformation as mistral; +use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; + +/// OCR over large documents can take a while; bound it generously rather than +/// hanging forever on an unresponsive upstream. The client-level limit is the +/// outer ceiling; callers can tighten it per request via ``run_ocr``'s ``timeout``. +const OCR_TIMEOUT_SECS: u64 = 600; + +/// Maximum upstream body characters retained in error messages. OCR responses +/// can echo document contents and prompts; keep enough for debugging without +/// forwarding sensitive payloads across the host boundary. +const ERROR_BODY_MAX_CHARS: usize = 256; + +/// Process-wide blocking HTTP client (connection pool + TLS reused across calls). +fn http_client() -> &'static reqwest::blocking::Client { + static CLIENT: OnceLock = OnceLock::new(); + CLIENT.get_or_init(|| { + reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(OCR_TIMEOUT_SECS)) + .build() + .expect("failed to build reqwest client") + }) +} + +fn truncate_error_body(body: &str) -> String { + if body.chars().count() <= ERROR_BODY_MAX_CHARS { + return body.to_string(); + } + let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect(); + format!("{truncated}... (truncated)") +} + +/// Perform a Mistral OCR call end to end and return the normalized response as +/// JSON (the shape the Python `OCRResponse` model expects). +/// +/// Blocking: intended to be called with the GIL released from the Python bridge. +pub fn run_ocr( + model: &str, + document: Value, + api_key: Option<&str>, + api_base: Option<&str>, + optional_params: Map, + timeout: Option, +) -> CoreResult { + let config = &MISTRAL_OCR_CONFIG; + + let api_key = mistral::resolve_api_key(api_key, &|key| std::env::var(key).ok())?; + let url = mistral::complete_url(api_base); + let filtered_params = config.map_ocr_params(&optional_params); + let body = config + .transform_ocr_request(model, document, filtered_params)? + .data; + + let mut request = http_client().post(&url).bearer_auth(&api_key).json(&body); + if let Some(duration) = timeout { + request = request.timeout(duration); + } + + let response = request + .send() + .map_err(|err| CoreError::Network(err.to_string()))?; + + let status = response.status(); + let text = response + .text() + .map_err(|err| CoreError::Network(err.to_string()))?; + + if !status.is_success() { + return Err(CoreError::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }); + } + + let response_json: Value = serde_json::from_str(&text) + .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; + + Ok(config + .transform_ocr_response(model, response_json)? + .into_json()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn truncate_error_body_passes_short_strings_through() { + let body = "Unauthorized"; + assert_eq!(truncate_error_body(body), "Unauthorized"); + } + + #[test] + fn truncate_error_body_caps_long_payloads() { + let body = "x".repeat(ERROR_BODY_MAX_CHARS + 50); + let truncated = truncate_error_body(&body); + + assert!(truncated.ends_with("... (truncated)")); + let prefix_chars = truncated + .strip_suffix("... (truncated)") + .expect("truncated marker present") + .chars() + .count(); + assert_eq!(prefix_chars, ERROR_BODY_MAX_CHARS); + } + + #[test] + fn truncate_error_body_does_not_split_multibyte_chars() { + let body = "é".repeat(ERROR_BODY_MAX_CHARS + 10); + let truncated = truncate_error_body(&body); + assert!(truncated.is_char_boundary(truncated.len())); + } +} diff --git a/litellm-rust/crates/providers/src/openai/mod.rs b/litellm-rust/crates/providers/src/openai/mod.rs new file mode 100644 index 00000000000..403e32975cf --- /dev/null +++ b/litellm-rust/crates/providers/src/openai/mod.rs @@ -0,0 +1 @@ +pub mod realtime; diff --git a/litellm-rust/crates/providers/src/openai/realtime/mod.rs b/litellm-rust/crates/providers/src/openai/realtime/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/providers/src/openai/realtime/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/providers/src/openai/realtime/transformation.rs b/litellm-rust/crates/providers/src/openai/realtime/transformation.rs new file mode 100644 index 00000000000..2e127c699e0 --- /dev/null +++ b/litellm-rust/crates/providers/src/openai/realtime/transformation.rs @@ -0,0 +1,189 @@ +use litellm_core::realtime::transformation::RealtimeProviderConfig; +use litellm_core::realtime::types::{RealtimeEvent, RealtimeTransformResult}; +use litellm_core::CoreResult; + +/// Default OpenAI API base, used when the caller does not override `api_base`. +pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com"; + +/// Path appended to the resolved host base to reach the realtime endpoint. +pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime"; + +/// Percent-encode a query value, escaping any char outside the RFC 3986 +/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime +/// model slugs have no special chars, but this stays correct for the rest. +fn percent_encode(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~'); + if unreserved { + encoded.push(byte as char); + } else { + encoded.push('%'); + encoded.push_str(&format!("{byte:02X}")); + } + } + encoded +} + +/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`. +/// +/// Blank/whitespace `api_base` is treated as absent (guard at resolution time), +/// falling back to the default. The scheme is swapped to its WebSocket +/// equivalent (`https://`→`wss://`, `http://`→`ws://`); bases already using +/// `ws`/`wss` are left untouched. A bare host or unrecognized scheme defaults to +/// secure `wss://` so we never hand a scheme-less URL to the connector (this is +/// a deliberate hardening over Python's `_construct_url`, which would emit a +/// scheme-less URL here). A trailing `/` is trimmed before the path and +/// `?model=` are appended. +pub fn complete_url(api_base: Option<&str>, model: &str) -> String { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(OPENAI_REALTIME_DEFAULT_API_BASE); + + let base = if let Some(rest) = base.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = base.strip_prefix("http://") { + format!("ws://{rest}") + } else if base.starts_with("wss://") || base.starts_with("ws://") { + base.to_string() + } else { + format!("wss://{base}") + }; + + let base = base.trim_end_matches('/'); + + format!( + "{base}{OPENAI_REALTIME_PATH}?model={}", + percent_encode(model) + ) +} + +pub struct OpenAiRealtimeConfig; + +pub const OPENAI_REALTIME_CONFIG: OpenAiRealtimeConfig = OpenAiRealtimeConfig; + +impl RealtimeProviderConfig for OpenAiRealtimeConfig { + fn complete_url(&self, api_base: Option<&str>, model: &str) -> String { + complete_url(api_base, model) + } + + fn transform_realtime_request( + &self, + event: &RealtimeEvent, + _model: &str, + ) -> CoreResult { + Ok(RealtimeTransformResult::passthrough(event.clone())) + } + + fn transform_realtime_response( + &self, + event: &RealtimeEvent, + _model: &str, + ) -> CoreResult { + Ok(RealtimeTransformResult::passthrough(event.clone())) + } +} + +pub fn transform_realtime_request( + event: &RealtimeEvent, + model: &str, +) -> CoreResult { + OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) +} + +pub fn transform_realtime_response( + event: &RealtimeEvent, + model: &str, +) -> CoreResult { + OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn complete_url_defaults_to_openai_wss() { + assert_eq!( + complete_url(None, "gpt-4o-realtime-preview"), + "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" + ); + } + + #[test] + fn complete_url_blank_base_uses_default() { + assert_eq!( + complete_url(Some(" "), "gpt-4o-realtime-preview"), + "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" + ); + } + + #[test] + fn complete_url_swaps_http_to_ws() { + assert_eq!( + complete_url(Some("http://localhost:8080"), "gpt-4o-realtime-preview"), + "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview" + ); + } + + #[test] + fn complete_url_dedupes_trailing_slash() { + assert_eq!( + complete_url(Some("https://api.openai.com/"), "gpt-4o-realtime-preview"), + "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" + ); + } + + #[test] + fn complete_url_custom_base() { + assert_eq!( + complete_url(Some("https://oai.azure.example"), "gpt-4o-realtime-preview"), + "wss://oai.azure.example/v1/realtime?model=gpt-4o-realtime-preview" + ); + } + + #[test] + fn complete_url_preserves_existing_wss_scheme() { + assert_eq!( + complete_url(Some("wss://api.openai.com"), "gpt-realtime"), + "wss://api.openai.com/v1/realtime?model=gpt-realtime" + ); + } + + #[test] + fn complete_url_bare_host_defaults_to_wss() { + assert_eq!( + complete_url(Some("api.openai.com"), "gpt-realtime"), + "wss://api.openai.com/v1/realtime?model=gpt-realtime" + ); + } + + #[test] + fn complete_url_percent_encodes_model_space() { + assert_eq!( + complete_url(None, "gpt 4o"), + "wss://api.openai.com/v1/realtime?model=gpt%204o" + ); + } + + #[test] + fn transform_realtime_request_passthrough_preserves_event() { + let event: RealtimeEvent = + serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#) + .expect("valid event"); + let result = + transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible"); + assert_eq!(result.events, vec![event]); + } + + #[test] + fn transform_realtime_response_passthrough_preserves_event() { + let event: RealtimeEvent = + serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#) + .expect("valid event"); + let result = + transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible"); + assert_eq!(result.events, vec![event]); + } +} diff --git a/litellm-rust/crates/providers/src/realtime.rs b/litellm-rust/crates/providers/src/realtime.rs new file mode 100644 index 00000000000..398158f6dba --- /dev/null +++ b/litellm-rust/crates/providers/src/realtime.rs @@ -0,0 +1,374 @@ +//! End-to-end OpenAI realtime invocation. +//! +//! The host-facing entry point, mirroring `providers::ocr::run_ocr`: open the +//! WebSocket to OpenAI, then splice a client realtime stream to the upstream, +//! driving typed events through the pure `OPENAI_REALTIME_CONFIG` transforms. +//! Network, auth header, key resolution, and wire (de)serialization live here so +//! the `transformation` module stays pure and typed. +//! +//! The dial and splice steps are factored out ([`dial_upstream`], [`splice`]) so +//! the connection pool ([`crate::realtime_pool`]) can pre-establish an upstream, +//! buffer its `session.created`, and later hand the live socket to the same +//! splice loop a fresh dial uses. + +use std::time::Duration; + +use futures_util::stream::{SplitSink, SplitStream}; +use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::error::CoreError; +use litellm_core::realtime::transformation::RealtimeProviderConfig; +use litellm_core::realtime::types::RealtimeEvent; +use litellm_core::CoreResult; +use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; +use tokio_tungstenite::tungstenite::http::HeaderValue; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; + +use crate::openai::realtime::transformation::OPENAI_REALTIME_CONFIG; + +/// Environment variable holding the OpenAI API key (last-resort fallback). +const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; + +const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable"; + +/// Default **idle** timeout: if neither side sends a frame for this long, the +/// session is reaped. It resets on any activity, so it does not cap a healthy +/// (continuously streaming) session — it only frees a stalled one (e.g. a +/// half-open upstream that keeps the socket open but stops sending). +const DEFAULT_IDLE_TIMEOUT_SECS: u64 = 300; + +/// The concrete upstream WebSocket type (TLS or plain). Shared by the dial path +/// and the pool so warm sockets and fresh sockets are the exact same type. +pub type UpstreamWs = WebSocketStream>; +pub(crate) type UpstreamTx = SplitSink; +pub(crate) type UpstreamRx = SplitStream; + +/// Resolve the OpenAI API key from the explicit param or the environment. +/// +/// Blank/whitespace values are treated as absent (guard at resolution time). +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { + api_key + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| { + std::env::var(OPENAI_API_KEY_ENV) + .ok() + .filter(|key| !key.trim().is_empty()) + }) + .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) +} + +/// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. +/// +/// This is the dial half of [`realtime`], factored out so the pool can +/// pre-establish sockets ahead of any client. `api_key` here is already resolved +/// (non-blank) — the pool resolves it once when it is created. +pub(crate) async fn dial_upstream( + model: &str, + api_key: &str, + api_base: Option<&str>, +) -> CoreResult { + let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); + + let mut request = url + .as_str() + .into_client_request() + .map_err(|err| CoreError::Network(err.to_string()))?; + // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers + // beta_api_shape_disabled, so we do not send it. + request.headers_mut().insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {api_key}")) + .map_err(|err| CoreError::Auth(err.to_string()))?, + ); + + let (upstream, _response) = connect_async(request) + .await + .map_err(|err| CoreError::Network(err.to_string()))?; + Ok(upstream) +} + +/// Read the next text frame from the upstream and decode it as a typed event. +/// +/// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an +/// error on a non-text frame, a closed socket, or undecodable JSON so the pool can +/// discard a misbehaving socket rather than warm it. +pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult { + loop { + let message = upstream_rx + .next() + .await + .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))? + .map_err(|err| CoreError::Network(err.to_string()))?; + match message { + Message::Text(text) => { + return serde_json::from_str(&text) + .map_err(|err| CoreError::InvalidResponse(err.to_string())); + } + // Ignore protocol frames (ping/pong) while waiting for the first event. + Message::Ping(_) | Message::Pong(_) => continue, + Message::Close(_) => { + return Err(CoreError::Network( + "upstream closed before first event".to_string(), + )) + } + _ => continue, + } + } +} + +/// Splice an already-connected upstream to the client streams. +/// +/// `prelude` is relayed to the client first (the pool passes the buffered +/// `session.created` here; the fresh-dial path passes `None` and lets the upstream +/// deliver it). Then a single select loop forwards both directions through the +/// transforms until either side closes or the idle timeout fires. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn splice( + model: &str, + mut upstream_tx: UpstreamTx, + mut upstream_rx: UpstreamRx, + prelude: Option, + idle_timeout: Option, + mut client_in: In, + mut client_out: Out, +) -> CoreResult<()> +where + In: Stream + Unpin + Send, + Out: Sink + Unpin + Send, + >::Error: std::fmt::Display, +{ + let config = &OPENAI_REALTIME_CONFIG; + + // Relay a buffered backend event (warm handoff's session.created) first, so a + // warm session looks identical to a fresh one from the client's view. + if let Some(event) = prelude { + for outbound in config.transform_realtime_response(&event, model)?.events { + client_out + .send(outbound) + .await + .map_err(|err| CoreError::Network(err.to_string()))?; + } + } + + let idle = idle_timeout.unwrap_or(Duration::from_secs(DEFAULT_IDLE_TIMEOUT_SECS)); + + // One loop forwarding both directions. The `sleep(idle)` arm is rebuilt every + // iteration, so any frame (either way) resets it — it fires only when the + // session has been fully idle for `idle`, reaping a stalled connection + // (task + upstream TCP socket) instead of leaking it. + loop { + tokio::select! { + // client -> upstream + client_event = client_in.next() => { + let Some(event) = client_event else { break }; // client disconnected + for outbound in config.transform_realtime_request(&event, model)?.events { + let payload = serde_json::to_string(&outbound) + .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + upstream_tx + .send(Message::Text(payload)) + .await + .map_err(|err| CoreError::Network(err.to_string()))?; + } + } + // upstream -> client + upstream_message = upstream_rx.next() => { + let Some(message) = upstream_message else { break }; // upstream closed + match message.map_err(|err| CoreError::Network(err.to_string()))? { + Message::Text(text) => { + let event: RealtimeEvent = serde_json::from_str(&text) + .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + for outbound in config.transform_realtime_response(&event, model)?.events { + client_out + .send(outbound) + .await + .map_err(|err| CoreError::Network(err.to_string()))?; + } + } + Message::Close(_) => break, + _ => {} + } + } + // idle timeout: no activity from either side within `idle` + _ = tokio::time::sleep(idle) => break, + } + } + Ok(()) +} + +/// Splice a client realtime stream to OpenAI: forward client events upstream +/// (via `transform_realtime_request`) and backend events downstream (via +/// `transform_realtime_response`). Returns when either side closes. +/// +/// Generic over the client transport (typed events) so this crate stays +/// framework-agnostic; the gateway adapts its axum socket to these. This is the +/// fresh-dial path: dial, then splice. The pool's warm-handoff path skips the dial +/// and calls [`splice`] directly with a buffered `session.created`. +pub async fn realtime( + model: &str, + api_key: Option<&str>, + api_base: Option<&str>, + idle_timeout: Option, + client_in: In, + client_out: Out, +) -> CoreResult<()> +where + In: Stream + Unpin + Send, + Out: Sink + Unpin + Send, + >::Error: std::fmt::Display, +{ + let api_key = resolve_api_key(api_key)?; + let upstream = dial_upstream(model, &api_key, api_base).await?; + let (upstream_tx, upstream_rx) = upstream.split(); + splice( + model, + upstream_tx, + upstream_rx, + None, + idle_timeout, + client_in, + client_out, + ) + .await +} + +/// Splice a pre-warmed upstream (taken from [`crate::realtime_pool`]) to the +/// client. Relays the buffered `session.created` first, then splices exactly like +/// the fresh-dial path — so a warm session is indistinguishable from a fresh one. +pub async fn realtime_warm( + model: &str, + handoff: crate::realtime_pool::WarmHandoff, + idle_timeout: Option, + client_in: In, + client_out: Out, +) -> CoreResult<()> +where + In: Stream + Unpin + Send, + Out: Sink + Unpin + Send, + >::Error: std::fmt::Display, +{ + splice( + model, + handoff.tx, + handoff.rx, + Some(handoff.session_created), + idle_timeout, + client_in, + client_out, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(raw: &str) -> RealtimeEvent { + serde_json::from_str(raw).expect("valid event json") + } + + #[test] + fn resolve_api_key_prefers_param_then_blank_falls_through() { + assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test"); + // A blank param with no env set should error. + if std::env::var(OPENAI_API_KEY_ENV).is_err() { + assert!(resolve_api_key(Some(" ")).is_err()); + } + } + + /// Live end-to-end check against OpenAI. Ignored by default (CI never runs + /// it); run explicitly with `OPENAI_API_KEY` set: + /// `cargo test -p litellm-providers realtime_invokes_openai -- --ignored --nocapture` + #[tokio::test] + #[ignore = "hits the live OpenAI realtime API; needs OPENAI_API_KEY"] + async fn realtime_invokes_openai_and_responds() { + use futures_channel::mpsc; + + let key = + std::env::var(OPENAI_API_KEY_ENV).expect("set OPENAI_API_KEY to run this ignored test"); + + // client -> provider (we hold `client_tx` to push events upstream) + let (mut client_tx, client_in) = mpsc::unbounded::(); + // provider -> client (we hold `backend_rx` to read backend events) + let (client_out, mut backend_rx) = mpsc::unbounded::(); + + // Clone the key so the spawned task owns its `String` (no borrow across await). + let key_owned = key.clone(); + let call = tokio::spawn(async move { + realtime( + "gpt-realtime", + Some(&key_owned), + None, + None, + client_in, + client_out, + ) + .await + }); + + // 1. First backend event should be session.created. + let first = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()) + .await + .expect("timed out waiting for session.created") + .expect("backend stream closed before session.created"); + assert_eq!( + first.event_type, "session.created", + "expected session.created, got: {}", + first.event_type + ); + + // 2. Ask for a short audio response. + client_tx + .send(event( + r#"{"type":"conversation.item.create","item":{"type":"message","role":"user","content":[{"type":"input_text","text":"Say hi."}]}}"#, + )) + .await + .expect("send conversation.item.create"); + client_tx + .send(event(r#"{"type":"response.create"}"#)) + .await + .expect("send response.create"); + + // 3. Read backend events; require a non-empty audio delta, then response.done. + let mut saw_audio_delta = false; + let mut saw_done = false; + for _ in 0..500 { + let next = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()).await; + let event = match next { + Ok(Some(event)) => event, + Ok(None) => break, + Err(_) => panic!("timed out waiting for backend events"), + }; + match event.event_type.as_str() { + "response.output_audio.delta" => { + let delta = event + .data + .get("delta") + .and_then(|value| value.as_str()) + .unwrap_or(""); + if !delta.is_empty() { + saw_audio_delta = true; + } + } + "response.done" => { + saw_done = true; + break; + } + _ => {} + } + } + + assert!( + saw_audio_delta, + "expected a response.output_audio.delta with non-empty delta" + ); + assert!(saw_done, "expected a response.done event"); + + // Drop the client sender so the provider's to_upstream side finishes. + drop(client_tx); + let _ = call.await; + } +} diff --git a/litellm-rust/crates/providers/src/realtime_pool.rs b/litellm-rust/crates/providers/src/realtime_pool.rs new file mode 100644 index 00000000000..1b1fc8112c5 --- /dev/null +++ b/litellm-rust/crates/providers/src/realtime_pool.rs @@ -0,0 +1,712 @@ +//! Pre-warmed upstream realtime connection pool. +//! +//! The gateway's realtime overhead lives entirely in session establishment: on +//! every client connect it dials a fresh upstream WS to OpenAI and waits for +//! `session.created` before it can serve. This pool keeps a small set of upstream +//! sockets **already connected and already past `session.created`** so a connect +//! can be served from a warm socket and the handshake is off the critical path. +//! +//! Layering: this stays in `providers` (axum-free) next to the dial/splice it +//! reuses. The gateway holds an `Arc` in its state and asks for a +//! warm socket per connect; on a miss it fresh-dials exactly as before. The pool +//! is a latency optimization, never a correctness dependency — see the gateway's +//! `src/routes/realtime/README.md`. +//! +//! ## Caveats (enforced here) +//! - One warm socket serves exactly one session (realtime isn't multiplexed), so +//! the pool is sized to the connect *rate*, not concurrent connections. +//! - `session.created` is pre-read once and buffered; nothing else is read from a +//! warm socket before handoff, so a warm session starts at OpenAI defaults just +//! like a fresh one (`session.update` semantics unchanged). +//! - Warm sockets are short-lived (`max_idle`) and liveness-checked at handoff to +//! bound idle billing / dodge OpenAI's idle timeout. +//! - On miss or dead socket the caller fresh-dials; the pool never blocks or fails +//! a connect because it is empty. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use futures_util::StreamExt; +use litellm_core::realtime::types::RealtimeEvent; +use litellm_core::CoreResult; + +use crate::realtime::{ + dial_upstream, read_event, resolve_api_key, UpstreamRx, UpstreamTx, UpstreamWs, +}; + +/// Default target warm sockets per key when pooling is enabled. +pub const DEFAULT_POOL_SIZE: usize = 4; + +/// Default max time a warm socket may sit before it is closed and replaced. +pub const DEFAULT_MAX_IDLE: Duration = Duration::from_secs(30); + +/// Env var: target warm sockets per key. `0` disables pooling (fresh-dial only). +pub const POOL_SIZE_ENV: &str = "REALTIME_POOL_SIZE"; + +/// Env var: max warm-socket idle lifetime, in seconds. +pub const MAX_IDLE_ENV: &str = "REALTIME_POOL_MAX_IDLE_SECS"; + +/// How often the background replenisher wakes to top up and reap stale sockets. +const REPLENISH_TICK: Duration = Duration::from_millis(250); + +/// Backoff floor after a key's warm-up dials all fail. The first failed pass +/// waits this long before retrying that key. +const BACKOFF_BASE: Duration = Duration::from_millis(500); + +/// Backoff ceiling. A key that keeps failing (invalid credentials, an +/// unreachable upstream) is retried at most once per this interval — instead of +/// firing `needed` concurrent TLS dials every 250 ms tick, which would hammer +/// the upstream and risk rate-limit exhaustion that degrades valid cold-path +/// traffic. Backoff resets the moment a dial for the key succeeds. +const BACKOFF_MAX: Duration = Duration::from_secs(30); + +/// Identifies an upstream connection: the tuple that fully determines the dial. +/// `api_key` is included so a warm socket is only ever reused for the same key +/// (no cross-tenant reuse). +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct UpstreamKey { + pub model: String, + pub api_key: String, + pub api_base: Option, +} + +impl std::fmt::Debug for UpstreamKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpstreamKey") + .field("model", &self.model) + .field("api_key", &"[REDACTED]") + .field("api_base", &self.api_base) + .finish() + } +} + +/// A warm upstream: split halves + the buffered `session.created` + when it was +/// warmed (for `max_idle` expiry). +struct WarmConnection { + tx: UpstreamTx, + rx: UpstreamRx, + session_created: RealtimeEvent, + warmed_at: Instant, +} + +/// A live upstream taken from the pool, ready to splice. The caller relays +/// `session_created` to the client first, then splices `(tx, rx)` as usual. +pub struct WarmHandoff { + pub tx: UpstreamTx, + pub rx: UpstreamRx, + pub session_created: RealtimeEvent, +} + +/// Pool configuration, resolved once at startup from the environment. +#[derive(Clone, Copy, Debug)] +pub struct PoolConfig { + /// Target warm sockets per key. `0` disables pooling. + pub target_size: usize, + /// Max time a warm socket may sit before it is closed and replaced. + pub max_idle: Duration, +} + +impl Default for PoolConfig { + fn default() -> Self { + Self { + target_size: DEFAULT_POOL_SIZE, + max_idle: DEFAULT_MAX_IDLE, + } + } +} + +impl PoolConfig { + /// Read config from the environment, falling back to defaults. An invalid + /// value warns and uses the default rather than failing startup. + pub fn from_env() -> Self { + let target_size = match std::env::var(POOL_SIZE_ENV) { + Ok(raw) => raw.trim().parse().unwrap_or_else(|_| { + eprintln!("warning: {POOL_SIZE_ENV}={raw:?} is not a valid size; using {DEFAULT_POOL_SIZE}"); + DEFAULT_POOL_SIZE + }), + Err(_) => DEFAULT_POOL_SIZE, + }; + let max_idle = match std::env::var(MAX_IDLE_ENV) { + Ok(raw) => raw + .trim() + .parse() + .map(Duration::from_secs) + .unwrap_or_else(|_| { + eprintln!( + "warning: {MAX_IDLE_ENV}={raw:?} is not a valid number of seconds; using {}s", + DEFAULT_MAX_IDLE.as_secs() + ); + DEFAULT_MAX_IDLE + }), + Err(_) => DEFAULT_MAX_IDLE, + }; + Self { + target_size, + max_idle, + } + } + + /// Whether pooling is on (`target_size > 0`). + pub fn enabled(&self) -> bool { + self.target_size > 0 + } +} + +/// Per-key warm sockets, behind a single `Mutex`. Realtime warm sockets are few +/// (the pool is small), so a plain mutex over a `VecDeque`-ish `Vec` is simpler +/// and faster than sharding; contention is negligible at this scale. +type Warm = HashMap>; + +/// Per-key replenish backoff. Absent (or `consecutive_failures == 0`) means the +/// key is healthy and replenished every tick. After a pass whose dials all fail, +/// `retry_after` is pushed out with exponential backoff so a broken key (invalid +/// credentials, unreachable upstream) is not re-dialed on every 250 ms tick. +#[derive(Default)] +struct Backoff { + /// Don't attempt warm-up dials for this key until this instant. `None` = + /// eligible now. + retry_after: Option, + consecutive_failures: u32, +} + +type Backoffs = HashMap; + +/// Pre-warmed upstream realtime connection pool. +/// +/// Cheap to clone-via-`Arc`. The background replenisher is spawned by +/// [`RealtimePool::spawn`]; a pool built with [`RealtimePool::disabled`] never +/// warms anything and every `take` misses (callers fresh-dial). +pub struct RealtimePool { + config: PoolConfig, + warm: Mutex, + /// Per-key replenish backoff so a broken key doesn't trigger unbounded + /// concurrent dials every tick. Separate lock from `warm` so the request + /// hot path (`take`) never contends on it. + backoff: Mutex, +} + +impl RealtimePool { + /// A disabled pool: no background task, every `take` returns `None`. + pub fn disabled() -> Arc { + Arc::new(Self { + config: PoolConfig { + target_size: 0, + ..PoolConfig::default() + }, + warm: Mutex::new(HashMap::new()), + backoff: Mutex::new(HashMap::new()), + }) + } + + /// Build a pool from config **without** the background replenisher. The pool + /// only warms when [`RealtimePool::warm_now`] is called. Used by deterministic + /// unit tests; production uses [`RealtimePool::spawn`]. + #[cfg(test)] + fn new_unspawned(config: PoolConfig) -> Arc { + Arc::new(Self { + config, + warm: Mutex::new(HashMap::new()), + backoff: Mutex::new(HashMap::new()), + }) + } + + /// Build a pool from config and, if enabled, spawn the background replenisher. + /// Returns the shared handle the gateway stores in its state. + pub fn spawn(config: PoolConfig) -> Arc { + let pool = Arc::new(Self { + config, + warm: Mutex::new(HashMap::new()), + backoff: Mutex::new(HashMap::new()), + }); + if config.enabled() { + let weak = Arc::downgrade(&pool); + tokio::spawn(async move { + let mut tick = tokio::time::interval(REPLENISH_TICK); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tick.tick().await; + // Stop once the gateway has dropped its handle. + let Some(pool) = weak.upgrade() else { break }; + pool.replenish_all().await; + } + }); + } + pool + } + + /// Resolved config (test/inspection). + pub fn config(&self) -> PoolConfig { + self.config + } + + /// Register a key so the replenisher starts warming it. Idempotent. The + /// gateway calls this once per known deployment at startup; the pool only + /// warms keys it has seen, so it never dials a model nobody asked for. + pub fn register(&self, key: UpstreamKey) { + if !self.config.enabled() { + return; + } + self.warm.lock().unwrap().entry(key).or_default(); + } + + /// Take a warm, live socket for `key`, or `None` on miss / dead socket. + /// + /// Pops the freshest non-expired socket and liveness-checks it; a socket that + /// is too old or already dead is dropped (closing it) and the next candidate + /// tried. Never blocks: if nothing warm is live, returns `None` so the caller + /// fresh-dials. + pub fn take(&self, key: &UpstreamKey) -> Option { + if !self.config.enabled() { + return None; + } + loop { + let mut candidate = { + let mut warm = self.warm.lock().unwrap(); + let bucket = warm.get_mut(key)?; + bucket.pop()? + }; + // Discard sockets past their warm lifetime (idle-billing guard). + if candidate.warmed_at.elapsed() > self.config.max_idle { + continue; // drops `candidate`, closing the socket + } + // Liveness: a non-blocking check that the socket hasn't already + // delivered a Close/Err. A warm socket should be silent after + // session.created, so anything pending means it is unhealthy. + if is_dead(&mut candidate.rx) { + continue; + } + return Some(WarmHandoff { + tx: candidate.tx, + rx: candidate.rx, + session_created: candidate.session_created, + }); + } + } + + /// One replenish pass over every registered key: reap stale sockets, then + /// dial up to `target_size`. Dials run concurrently; failures are swallowed + /// (a key that can't be warmed just keeps fresh-dialing on the request path) + /// and put the key into exponential backoff so a broken key isn't re-dialed + /// on every tick. + async fn replenish_all(&self) { + let keys: Vec = { self.warm.lock().unwrap().keys().cloned().collect() }; + for key in keys { + self.reap_stale(&key); + // Skip keys still in backoff from a prior all-failed pass — this is + // what bounds dials against an invalid/unreachable key to once per + // `BACKOFF_MAX` instead of `needed` dials every 250 ms tick. + if self.in_backoff(&key) { + continue; + } + let needed = { + let warm = self.warm.lock().unwrap(); + let have = warm.get(&key).map(Vec::len).unwrap_or(0); + self.config.target_size.saturating_sub(have) + }; + if needed == 0 { + continue; + } + // Dial the missing sockets CONCURRENTLY. A sequential loop here makes + // a full refill cost `needed × handshake` (~needed × 350 ms), which + // can't keep up with a high connect rate — the pool drains faster + // than it refills and most connects miss. Firing the dials together + // refills in ~one handshake window, keeping warm supply ≈ peak + // concurrent connects so the sub-ms warm handoff becomes the median, + // not the lucky-hit tail. + let dials = (0..needed).map(|_| warm_one(&key)); + let results = futures_util::future::join_all(dials).await; + let mut any_ok = false; + // `.flatten()` keeps only the successful dials; a key that can't be + // warmed just keeps fresh-dialing on the request path. + for conn in results.into_iter().flatten() { + any_ok = true; + self.warm + .lock() + .unwrap() + .entry(key.clone()) + .or_default() + .push(conn); + } + // Reset backoff on any success; otherwise grow it. We only ever enter + // backoff when a pass that *attempted* dials produced none — a `needed + // == 0` pass is handled by the `continue` above and never touches it. + self.record_replenish_outcome(&key, any_ok); + } + } + + /// Whether `key` is currently in a backoff window (a prior pass failed and + /// the retry time hasn't arrived). Eligible keys are pruned from the backoff + /// map so it doesn't grow unbounded for healthy keys. + fn in_backoff(&self, key: &UpstreamKey) -> bool { + let mut backoff = self.backoff.lock().unwrap(); + match backoff.get(key).and_then(|b| b.retry_after) { + Some(retry_after) if Instant::now() < retry_after => true, + Some(_) => { + // Window elapsed — allow the attempt. Keep the failure count so a + // still-broken key backs off further, but clear the gate so this + // tick proceeds. + if let Some(b) = backoff.get_mut(key) { + b.retry_after = None; + } + false + } + None => false, + } + } + + /// Update a key's backoff after a replenish attempt. Success clears it; + /// failure grows the retry delay exponentially up to `BACKOFF_MAX`. + fn record_replenish_outcome(&self, key: &UpstreamKey, any_ok: bool) { + let mut backoff = self.backoff.lock().unwrap(); + if any_ok { + backoff.remove(key); + return; + } + let entry = backoff.entry(key.clone()).or_default(); + entry.consecutive_failures = entry.consecutive_failures.saturating_add(1); + // Exponential: BASE * 2^(failures-1), saturating at MAX. `min` of the + // shift exponent keeps the doubling from overflowing. + let shift = (entry.consecutive_failures - 1).min(16); + let delay = BACKOFF_BASE.saturating_mul(1u32 << shift).min(BACKOFF_MAX); + entry.retry_after = Some(Instant::now() + delay); + } + + /// Drop sockets past `max_idle` or already dead for a key. + fn reap_stale(&self, key: &UpstreamKey) { + let mut warm = self.warm.lock().unwrap(); + if let Some(bucket) = warm.get_mut(key) { + bucket.retain_mut(|conn| { + conn.warmed_at.elapsed() <= self.config.max_idle && !is_dead(&mut conn.rx) + }); + } + } + + /// Test/inspection: number of warm sockets currently held for `key`. + #[cfg(test)] + pub fn warm_len(&self, key: &UpstreamKey) -> usize { + self.warm + .lock() + .unwrap() + .get(key) + .map(Vec::len) + .unwrap_or(0) + } + + /// Test/inspection: consecutive replenish failures recorded for `key` (0 if + /// the key is healthy / has no backoff entry). + #[cfg(test)] + pub fn backoff_failures(&self, key: &UpstreamKey) -> u32 { + self.backoff + .lock() + .unwrap() + .get(key) + .map(|b| b.consecutive_failures) + .unwrap_or(0) + } + + /// Test helper: synchronously warm `target_size` sockets for `key` (no + /// background task). Lets tests assert handoff behavior deterministically. + #[cfg(test)] + pub async fn warm_now(&self, key: &UpstreamKey) { + let needed = { + let warm = self.warm.lock().unwrap(); + let have = warm.get(key).map(Vec::len).unwrap_or(0); + self.config.target_size.saturating_sub(have) + }; + for _ in 0..needed { + if let Ok(conn) = warm_one(key).await { + self.warm + .lock() + .unwrap() + .entry(key.clone()) + .or_default() + .push(conn); + } + } + } + + /// Test helper: insert an already-built warm connection (used to inject a + /// dead socket and assert it is discarded at handoff). + #[cfg(test)] + fn insert_warm(&self, key: UpstreamKey, conn: WarmConnection) { + self.warm.lock().unwrap().entry(key).or_default().push(conn); + } +} + +/// Dial one upstream and pre-read its `session.created` into a [`WarmConnection`]. +/// +/// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends +/// unprompted is `session.created`; we buffer exactly that and read nothing more. +async fn warm_one(key: &UpstreamKey) -> CoreResult { + let upstream: UpstreamWs = + dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; + let (tx, mut rx) = upstream.split(); + let session_created = read_event(&mut rx).await?; + Ok(WarmConnection { + tx, + rx, + session_created, + warmed_at: Instant::now(), + }) +} + +/// Resolve a deployment's API key into the pool key, returning `None` when no key +/// can be resolved (those deployments simply aren't pooled — the request path +/// still fresh-dials and surfaces the auth error there). +pub fn upstream_key( + model: &str, + api_key: Option<&str>, + api_base: Option<&str>, +) -> Option { + let api_key = resolve_api_key(api_key).ok()?; + Some(UpstreamKey { + model: model.to_string(), + api_key, + api_base: api_base.map(str::to_string), + }) +} + +/// Non-blocking liveness check: poll the upstream once. A warm socket is silent +/// after `session.created`, so a pending `Close`/`Err`/`None` means it is dead. +/// A pending data frame (shouldn't happen pre-handoff) is also treated as +/// unhealthy — we'd rather discard and fresh-dial than hand over a socket in an +/// unexpected state. `Pending` (the healthy case) returns `false`. +fn is_dead(rx: &mut UpstreamRx) -> bool { + use futures_util::task::noop_waker_ref; + use futures_util::Stream; + use std::pin::Pin; + use std::task::{Context, Poll}; + + let mut cx = Context::from_waker(noop_waker_ref()); + match Pin::new(rx).poll_next(&mut cx) { + Poll::Pending => false, + Poll::Ready(None) => true, + Poll::Ready(Some(Err(_))) => true, + // Any frame arriving before handoff is unexpected for a silent warm + // socket; treat it as unhealthy. + Poll::Ready(Some(Ok(_))) => true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::SinkExt; + use std::net::SocketAddr; + use tokio::net::TcpListener; + use tokio_tungstenite::tungstenite::Message; + + /// An in-process fake OpenAI realtime WS server. On connect it sends + /// `session.created`; on `response.create` it sends `response.created` + + /// `response.output_audio.delta` + `response.done`. Returns its `ws://` base. + async fn spawn_fake_openai() -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(handle_fake_conn(stream)); + } + }); + format!("ws://{addr}") + } + + async fn handle_fake_conn(stream: tokio::net::TcpStream) { + let mut ws = match tokio_tungstenite::accept_async(stream).await { + Ok(ws) => ws, + Err(_) => return, + }; + // Unprompted session.created, exactly like OpenAI. + let _ = ws + .send(Message::Text( + r#"{"type":"session.created","session":{"id":"sess_fake"}}"#.to_string(), + )) + .await; + while let Some(Ok(msg)) = ws.next().await { + if let Message::Text(text) = msg { + if text.contains("response.create") { + for frame in [ + r#"{"type":"response.created"}"#, + r#"{"type":"response.output_audio.delta","delta":"AAAA"}"#, + r#"{"type":"response.done"}"#, + ] { + let _ = ws.send(Message::Text(frame.to_string())).await; + } + } + } + } + } + + fn test_config() -> PoolConfig { + PoolConfig { + target_size: 2, + max_idle: Duration::from_secs(30), + } + } + + fn key_for(base: &str) -> UpstreamKey { + UpstreamKey { + model: "gpt-realtime".to_string(), + api_key: "sk-test".to_string(), + api_base: Some(base.to_string()), + } + } + + #[tokio::test] + async fn warm_handoff_relays_buffered_session_created() { + let base = spawn_fake_openai().await; + let pool = RealtimePool::new_unspawned(test_config()); + let key = key_for(&base); + pool.register(key.clone()); + pool.warm_now(&key).await; + assert_eq!(pool.warm_len(&key), 2); + + let handoff = pool.take(&key).expect("a warm socket should be available"); + assert_eq!(handoff.session_created.event_type, "session.created"); + assert_eq!( + handoff + .session_created + .data + .get("session") + .and_then(|s| s.get("id")) + .and_then(|v| v.as_str()), + Some("sess_fake") + ); + // Taking one leaves one. + assert_eq!(pool.warm_len(&key), 1); + } + + #[tokio::test] + async fn pool_miss_returns_none_for_fresh_dial_fallback() { + let base = spawn_fake_openai().await; + let pool = RealtimePool::new_unspawned(test_config()); + let key = key_for(&base); + // Registered but never warmed → empty bucket → miss. + pool.register(key.clone()); + assert!(pool.take(&key).is_none()); + + // Unknown key → miss. + let other = key_for("ws://127.0.0.1:1"); + assert!(pool.take(&other).is_none()); + } + + #[tokio::test] + async fn disabled_pool_never_hands_off() { + let pool = RealtimePool::disabled(); + let key = key_for("ws://127.0.0.1:1"); + pool.register(key.clone()); + assert_eq!(pool.warm_len(&key), 0); + assert!(pool.take(&key).is_none()); + } + + #[tokio::test] + async fn dead_warm_socket_is_discarded() { + let base = spawn_fake_openai().await; + let pool = RealtimePool::new_unspawned(test_config()); + let key = key_for(&base); + pool.register(key.clone()); + + // Build one real warm connection, then kill the upstream by dropping the + // server side: easiest is to dial, read session.created, then close our + // own rx's peer. Instead we forge "dead" via an already-closed socket: + // dial a connection and immediately send a Close from the client side so + // the server closes back, then warm it. Simpler: warm normally, then + // mark it stale by backdating warmed_at past max_idle and confirm it's + // dropped — that exercises the same discard path. + let mut conn = warm_one(&key).await.expect("warm one"); + conn.warmed_at = Instant::now() - Duration::from_secs(3600); // past max_idle + pool.insert_warm(key.clone(), conn); + assert_eq!(pool.warm_len(&key), 1); + + // take() must discard the stale socket and report a miss. + assert!(pool.take(&key).is_none()); + assert_eq!(pool.warm_len(&key), 0); + } + + #[tokio::test] + async fn background_replenisher_tops_up_registered_key() { + let base = spawn_fake_openai().await; + let pool = RealtimePool::spawn(test_config()); + let key = key_for(&base); + pool.register(key.clone()); + + // Wait (bounded) for the background task to reach the target size. + let mut warmed = 0; + for _ in 0..40 { + tokio::time::sleep(Duration::from_millis(50)).await; + warmed = pool.warm_len(&key); + if warmed >= test_config().target_size { + break; + } + } + assert_eq!( + warmed, + test_config().target_size, + "background replenisher should warm up to target_size" + ); + let handoff = pool.take(&key).expect("a warm socket should be available"); + assert_eq!(handoff.session_created.event_type, "session.created"); + } + + #[tokio::test] + async fn closed_upstream_socket_is_detected_dead() { + // A genuinely dead socket: dial the fake, read session.created, then drop + // the server by closing from our side and waiting for the close to land. + let base = spawn_fake_openai().await; + let pool = RealtimePool::new_unspawned(test_config()); + let key = key_for(&base); + pool.register(key.clone()); + + let mut conn = warm_one(&key).await.expect("warm one"); + // Close the upstream from the client side; the server echoes a close. + let _ = conn.tx.send(Message::Close(None)).await; + // Give the close a moment to arrive on rx. + tokio::time::sleep(Duration::from_millis(50)).await; + pool.insert_warm(key.clone(), conn); + + // Liveness check at take() should detect the close and discard it. + assert!(pool.take(&key).is_none()); + assert_eq!(pool.warm_len(&key), 0); + } + + #[tokio::test] + async fn broken_key_backs_off_instead_of_dialing_every_tick() { + // A key whose upstream is unreachable: every warm-up dial fails. + let pool = RealtimePool::new_unspawned(test_config()); + let key = key_for("ws://127.0.0.1:1"); // nothing listens here + pool.register(key.clone()); + + // First pass attempts dials, they all fail → key enters backoff, no warm + // sockets, one recorded failure. + pool.replenish_all().await; + assert_eq!(pool.warm_len(&key), 0); + assert_eq!(pool.backoff_failures(&key), 1); + assert!( + pool.in_backoff(&key), + "a key whose dials all failed must be in backoff" + ); + + // An immediate next pass must be SKIPPED (still in the backoff window), so + // it does NOT fire another round of dials — the failure count is unchanged. + pool.replenish_all().await; + assert_eq!( + pool.backoff_failures(&key), + 1, + "replenish during the backoff window must not re-dial the broken key" + ); + } + + #[tokio::test] + async fn healthy_key_never_enters_backoff_and_clears_after_recovery() { + let base = spawn_fake_openai().await; + let pool = RealtimePool::new_unspawned(test_config()); + let key = key_for(&base); + pool.register(key.clone()); + + // A reachable upstream: the pass succeeds, so the key is never backed off. + pool.replenish_all().await; + assert_eq!(pool.warm_len(&key), test_config().target_size); + assert_eq!(pool.backoff_failures(&key), 0); + assert!(!pool.in_backoff(&key)); + } +} diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md new file mode 100644 index 00000000000..efa1a554c9c --- /dev/null +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -0,0 +1,36 @@ +# CLAUDE.md + +Rules for `litellm-rust/crates/python-bridge`. + +## Responsibility + +`python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms. +Keep this crate thin. It adapts Python objects to Rust payloads and returns +Python-compatible dictionaries. + +## Bridge Shape + +- Prefer one stable method per top-level LiteLLM route, for example + `ocr(payload)`. +- Do not add one exported PyO3 function per provider helper unless there is a + measured reason. +- Provider dispatch belongs in Rust route modules such as + `litellm_providers::ocr`, not in this PyO3 crate. +- Python owns rollout state and fallback. Rust should return errors; Python + decides whether to raise or fall back. + +## Data Handling + +- OCR payloads can contain personal data and large base64 images. Do not log + payloads or provider responses. +- Avoid copying large payloads more than needed. The current JSON round-trip is + acceptable for the first scaffold, but future performance work should evaluate + direct PyO3 conversion before expanding Rust coverage to image-heavy paths. +- Do not expose raw Rust errors that include document contents or upstream + bodies. + +## Tests + +- `cargo test --workspace` must compile this crate. +- Python tests must cover bridge disabled, bridge enabled, and module-missing + fallback behavior for every exposed route. diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml new file mode 100644 index 00000000000..80b6478daac --- /dev/null +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "litellm-python-bridge" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "litellm_python_bridge" +crate-type = ["cdylib"] + +[dependencies] +litellm-core.workspace = true +litellm-providers.workspace = true +pyo3 = { workspace = true, features = ["extension-module"] } +serde_json.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs new file mode 100644 index 00000000000..dc1b591735c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/gil.rs @@ -0,0 +1,32 @@ +//! GIL accounting. +//! +//! A single chokepoint for releasing the GIL around blocking work. Every +//! blocking call in the bridge goes through [`release_gil`] instead of calling +//! `Python::allow_threads` directly, so the release count stays accurate and we +//! have one place to extend later (timing histograms, per-call labels, etc.). + +use std::sync::atomic::{AtomicU64, Ordering}; + +use pyo3::prelude::*; + +/// Number of times the bridge has released the GIL since process start. +static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); + +/// Release the GIL around `f`, recording the release. +/// +/// `f` must not touch any Python state — that is what makes releasing the GIL +/// safe. Returning the value back to Python re-acquires the GIL at the call +/// site, after `f` has finished. +pub fn release_gil(py: Python<'_>, f: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, +{ + GIL_RELEASES.fetch_add(1, Ordering::Relaxed); + py.allow_threads(f) +} + +/// Total GIL releases performed by the bridge so far. +pub fn release_count() -> u64 { + GIL_RELEASES.load(Ordering::Relaxed) +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs new file mode 100644 index 00000000000..15e93f7b00c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -0,0 +1,100 @@ +use std::time::Duration; + +use litellm_core::error::CoreError; +use litellm_providers::ocr::run_ocr; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyDict}; +use serde_json::{Map, Value}; + +mod gil; + +fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { + let json = py.import("json")?; + let encoded: String = json.call_method1("dumps", (value,))?.extract()?; + serde_json::from_str(&encoded).map_err(|err| PyValueError::new_err(err.to_string())) +} + +fn json_to_py(py: Python<'_>, value: Value) -> PyResult> { + let json = py.import("json")?; + let encoded = + serde_json::to_string(&value).map_err(|err| PyValueError::new_err(err.to_string()))?; + Ok(json.call_method1("loads", (encoded,))?.unbind()) +} + +/// Map a core error to the closest Python exception. Caller-input problems +/// (auth, bad types, missing fields) -> `ValueError`; everything else +/// (network, upstream status, parse failures) -> `RuntimeError`. +fn core_error_to_pyerr(err: CoreError) -> PyErr { + match err { + CoreError::Auth(message) => PyValueError::new_err(message), + CoreError::InvalidType { .. } | CoreError::MissingField(_) => { + PyValueError::new_err(err.to_string()) + } + other => PyRuntimeError::new_err(other.to_string()), + } +} + +/// Perform a Mistral OCR call end to end and return the response as a dict. +#[pyfunction] +#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None, timeout_seconds=None))] +fn ocr( + py: Python<'_>, + model: String, + document: Py, + api_key: Option, + api_base: Option, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult> { + let document = py_to_json(py, document.bind(py))?; + + let optional_params = match optional_params { + Some(params) => match py_to_json(py, params.bind(py))? { + Value::Object(map) => map, + _ => return Err(PyValueError::new_err("optional_params must be a dict")), + }, + None => Map::new(), + }; + + let timeout = timeout_seconds.and_then(|secs| { + if secs.is_finite() && secs > 0.0 { + Some(Duration::from_secs_f64(secs)) + } else { + None + } + }); + + // Release the GIL during the blocking HTTP call (counted for observability). + let result = gil::release_gil(py, || { + run_ocr( + &model, + document, + api_key.as_deref(), + api_base.as_deref(), + optional_params, + timeout, + ) + }); + + match result { + Ok(value) => json_to_py(py, value), + Err(err) => Err(core_error_to_pyerr(err)), + } +} + +/// Bridge GIL accounting, e.g. `{"releases": 12}`. Lets the Python side observe +/// how often the bridge has dropped the GIL for blocking work. +#[pyfunction] +fn gil_stats(py: Python<'_>) -> PyResult> { + let stats = PyDict::new(py); + stats.set_item("releases", gil::release_count())?; + Ok(stats.into_any().unbind()) +} + +#[pymodule] +fn litellm_python_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(ocr, module)?)?; + module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + Ok(()) +} diff --git a/litellm/__init__.py b/litellm/__init__.py index b1ad63d72b0..d0513f77b35 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -80,6 +80,7 @@ from litellm.constants import ( WANDB_MODELS, REPEATED_STREAMING_CHUNK_LIMIT, request_timeout, + request_timeout_explicitly_set as request_timeout_explicitly_set, open_ai_embedding_models, cohere_embedding_models, bedrock_embedding_models, @@ -1405,6 +1406,7 @@ from .skills.main import ( ) from .containers.main import * from .ocr.main import * +from .ocr.rust_bridge import use_litellm_rust from .rag.main import * from .sandbox.main import * from .search.main import * diff --git a/litellm/constants.py b/litellm/constants.py index 083e9a1241b..212d34357f8 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -201,6 +201,18 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int( # Provider-specific API base URLs XAI_API_BASE = "https://api.x.ai/v1" +OPEN_SANDBOX_API_BASE_ENV_VAR = "OPEN_SANDBOX_API_BASE" +OPEN_SANDBOX_API_KEY_ENV_VAR = "OPEN_SANDBOX_API_KEY" +OPEN_SANDBOX_DEFAULT_TEMPLATE = "opensandbox/code-interpreter:v1.1.0" +_OPEN_SANDBOX_FALLBACK_ENTRYPOINT = "/opt/code-interpreter/code-interpreter.sh" +OPEN_SANDBOX_DEFAULT_ENTRYPOINT = (_OPEN_SANDBOX_FALLBACK_ENTRYPOINT,) +OPEN_SANDBOX_DEFAULT_LANGUAGE = "python" +OPEN_SANDBOX_DEFAULT_CPU_LIMIT = "1" +OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT = "2Gi" +OPEN_SANDBOX_EXECD_PORT = 44772 +OPEN_SANDBOX_DEFAULT_TIMEOUT = 300 +OPEN_SANDBOX_READY_TIMEOUT = 30.0 +OPEN_SANDBOX_POLL_INTERVAL = 0.2 DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int( os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024) @@ -456,6 +468,7 @@ HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: float = 5.0 request_timeout: float = float( os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS))) ) +request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ DEFAULT_A2A_AGENT_TIMEOUT: float = float( os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000) ) # 10 minutes diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index da8149eab9b..362581937d7 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -9,9 +9,11 @@ captured stdout back through the typed agentic loop plan. import json import time import uuid -from typing import Any, cast +from typing import Any, Literal, TypedDict, cast import litellm +from pydantic import ValidationError + from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.types.integrations.code_interpreter_interception import ( @@ -20,15 +22,93 @@ from litellm.types.integrations.code_interpreter_interception import ( from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, + CHAT_COMPLETION_AGENTIC_SURFACE, + NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + is_interception_internal_key, +) +from litellm.types.llms.openai import ( + ChatCompletionAssistantMessage, + ChatCompletionAssistantToolCall, + ChatCompletionToolMessage, +) +from litellm.types.utils import ( + CallTypes, + ChatCompletionMessageToolCall, + ModelResponse, ) -from litellm.types.utils import CallTypes LITELLM_CODE_EXECUTION_TOOL_NAME = "litellm_code_execution" _INTERCEPTION_ACTIVE_KEY = "_code_interpreter_interception_active" _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key" +_CONVERTED_STREAM_KEY = "_code_interpreter_interception_converted_stream" +_LITELLM_METADATA_KEY = "litellm_metadata" _CACHE_TTL_SECONDS = 15 * 60 +class CodeExecutionToolCall(TypedDict, total=False): + id: str | None + call_id: str | None + type: Literal["function"] + name: str + arguments: str + + +class CodeInterpreterLogOutput(TypedDict): + type: Literal["logs"] + logs: str + + +class CodeInterpreterCall(TypedDict): + id: str + type: Literal["code_interpreter_call"] + status: Literal["completed"] + code: str + container_id: str | None + outputs: list[CodeInterpreterLogOutput] + + +class CodeExecutionFunctionParameters(TypedDict): + type: Literal["object"] + properties: dict[str, dict[str, str]] + required: list[str] + + +class ResponsesFunctionTool(TypedDict): + type: Literal["function"] + name: str + description: str + parameters: CodeExecutionFunctionParameters + + +class ChatCompletionFunctionDefinition(TypedDict): + name: str + description: str + parameters: CodeExecutionFunctionParameters + + +class ChatCompletionFunctionTool(TypedDict): + type: Literal["function"] + function: ChatCompletionFunctionDefinition + + +CodeExecutionFunctionTool = ResponsesFunctionTool | ChatCompletionFunctionTool + + +class ResponsesFunctionToolChoice(TypedDict): + type: Literal["function"] + name: str + + +class ChatCompletionFunctionToolChoice(TypedDict): + type: Literal["function"] + function: dict[str, str] + + +CodeExecutionFunctionToolChoice = ( + ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice +) + + def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None: try: from litellm.sandbox.sandbox_tools import resolve_sandbox_tool @@ -97,9 +177,15 @@ class CodeInterpreterInterceptionLogger(CustomLogger): if not kwargs.get("_agentic_loop_depth"): kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None) kwargs.pop(_SANDBOX_KEY, None) + self._strip_interception_metadata(kwargs) if not self.enabled: return None - if call_type not in (CallTypes.responses, CallTypes.aresponses): + if call_type not in ( + CallTypes.responses, + CallTypes.aresponses, + CallTypes.completion, + CallTypes.acompletion, + ): return None if ( self.enabled_providers is not None @@ -120,18 +206,10 @@ class CodeInterpreterInterceptionLogger(CustomLogger): kwargs[_SANDBOX_KEY] = uuid.uuid4().hex if kwargs.get("stream"): kwargs["stream"] = False - kwargs["_code_interpreter_interception_converted_stream"] = True + kwargs[_CONVERTED_STREAM_KEY] = True + self._write_interception_metadata(kwargs) - function_tool = { - "type": "function", - "name": LITELLM_CODE_EXECUTION_TOOL_NAME, - "description": "Execute python code in a sandbox and return stdout.", - "parameters": { - "type": "object", - "properties": {"code": {"type": "string"}}, - "required": ["code"], - }, - } + function_tool = self._get_function_tool(call_type=call_type) kwargs["tools"] = [ ( function_tool @@ -141,19 +219,90 @@ class CodeInterpreterInterceptionLogger(CustomLogger): for tool in tools ] if self._tool_choice_targets_code_interpreter(kwargs.get("tool_choice")): - kwargs["tool_choice"] = { - "type": "function", - "name": LITELLM_CODE_EXECUTION_TOOL_NAME, - } + kwargs["tool_choice"] = self._get_function_tool_choice(call_type=call_type) return kwargs + @staticmethod + def _strip_interception_metadata(kwargs: dict[str, Any]) -> None: + metadata = kwargs.get(_LITELLM_METADATA_KEY) + if not isinstance(metadata, dict): + return + filtered_metadata = { + key: value + for key, value in metadata.items() + if not is_interception_internal_key(key) + and not key.startswith("_agentic_loop") + and key != "max_agentic_loops" + } + if filtered_metadata: + kwargs[_LITELLM_METADATA_KEY] = filtered_metadata + else: + kwargs.pop(_LITELLM_METADATA_KEY, None) + + @staticmethod + def _write_interception_metadata(kwargs: dict[str, Any]) -> None: + metadata = kwargs.get(_LITELLM_METADATA_KEY) + metadata = dict(metadata) if isinstance(metadata, dict) else {} + for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _CONVERTED_STREAM_KEY): + if key in kwargs: + metadata[key] = kwargs[key] + kwargs[_LITELLM_METADATA_KEY] = metadata + + @staticmethod + def _get_function_parameters() -> CodeExecutionFunctionParameters: + return { + "type": "object", + "properties": {"code": {"type": "string"}}, + "required": ["code"], + } + + def _get_function_tool( + self, call_type: CallTypes | None + ) -> CodeExecutionFunctionTool: + description = "Execute python code in a sandbox and return stdout." + if call_type in (CallTypes.completion, CallTypes.acompletion): + return { + "type": "function", + "function": { + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "description": description, + "parameters": self._get_function_parameters(), + }, + } + return { + "type": "function", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "description": description, + "parameters": self._get_function_parameters(), + } + + @staticmethod + def _get_function_tool_choice( + call_type: CallTypes | None, + ) -> CodeExecutionFunctionToolChoice: + if call_type in (CallTypes.completion, CallTypes.acompletion): + return { + "type": "function", + "function": {"name": LITELLM_CODE_EXECUTION_TOOL_NAME}, + } + return { + "type": "function", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + } + @staticmethod def _tool_choice_targets_code_interpreter(tool_choice: Any) -> bool: if not isinstance(tool_choice, dict): return False + function = tool_choice.get("function") return ( tool_choice.get("type") == "code_interpreter" or tool_choice.get("name") == "code_interpreter" + or tool_choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME + or ( + isinstance(function, dict) + and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME + ) ) def _resolve_provider(self, kwargs: dict[str, Any]) -> str | None: @@ -188,7 +337,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger): ): return False, {} - tool_calls = self._extract_code_execution_tool_calls(response=response) + tool_calls = ( + self._extract_chat_completion_code_execution_tool_calls(response=response) + if kwargs.get("_agentic_loop_api_surface") + == CHAT_COMPLETION_AGENTIC_SURFACE + else self._extract_code_execution_tool_calls(response=response) + ) if not tool_calls: return False, {} @@ -206,15 +360,24 @@ class CodeInterpreterInterceptionLogger(CustomLogger): stream: bool, kwargs: dict, ) -> AgenticLoopPlan: + if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE: + return await self._build_chat_completion_agentic_loop_plan( + tools=tools, + model=model, + messages=messages, + optional_params=anthropic_messages_optional_request_params, + kwargs=kwargs, + ) + await self._prune_expired_cache() - tool_calls = cast(list[dict[str, Any]], tools.get("tool_calls", [])) + tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", [])) sandbox_key = kwargs.get(_SANDBOX_KEY) container, params = await self._get_or_create_container(cache_key=sandbox_key) try: - container_id = getattr(container, "id", None) + container_id = cast(str | None, getattr(container, "id", None)) input_list = self._normalize_messages(messages) - code_interpreter_calls = [] + code_interpreter_calls: list[CodeInterpreterCall] = [] for tool_call in tool_calls: arguments = tool_call.get("arguments", "") code = self._parse_code(arguments) @@ -256,9 +419,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger): request_patch = AgenticLoopRequestPatch( model=model, messages=input_list, - tools=optional_params.get("tools"), - optional_params={k: v for k, v in optional_params.items() if k != "tools"}, - kwargs={k: v for k, v in kwargs.items() if k != "litellm_logging_obj"}, + tools=self._get_followup_tools( + tools=optional_params.get("tools"), + call_type=CallTypes.responses, + ), + optional_params=self._get_followup_optional_params(optional_params), + kwargs=self._filter_agentic_loop_kwargs(kwargs), ) return AgenticLoopPlan( @@ -271,12 +437,134 @@ class CodeInterpreterInterceptionLogger(CustomLogger): }, ) + async def _build_chat_completion_agentic_loop_plan( + self, + tools: dict[str, object], + model: str, + messages: list[dict], + optional_params: dict[str, object], + kwargs: dict[str, object], + ) -> AgenticLoopPlan: + await self._prune_expired_cache() + tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", [])) + sandbox_key = cast(str | None, kwargs.get(_SANDBOX_KEY)) + container, params = await self._get_or_create_container(cache_key=sandbox_key) + + try: + container_id = cast(str | None, getattr(container, "id", None)) + tool_results = [ + await self._build_chat_completion_tool_result( + container=container, + params=params, + tool_call=tool_call, + container_id=container_id, + ) + for tool_call in tool_calls + ] + except Exception: + await self._delete_container_for_cache_key(sandbox_key) + raise + tool_messages = [result[0] for result in tool_results] + code_interpreter_calls = [result[1] for result in tool_results] + + request_patch = AgenticLoopRequestPatch( + model=model, + messages=list(messages) + + [self._build_chat_completion_assistant_message(tool_calls)] + + tool_messages, + tools=self._get_followup_tools( + tools=optional_params.get("tools"), + call_type=CallTypes.completion, + ), + optional_params=self._get_followup_optional_params(optional_params), + kwargs=self._filter_agentic_loop_kwargs(kwargs), + ) + + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=request_patch, + metadata={ + "tool_type": "code_interpreter", + "sandbox_key": sandbox_key or "", + "code_interpreter_calls": code_interpreter_calls, + "response_format": "openai", + }, + ) + + async def _build_chat_completion_tool_result( + self, + container: object, + params: dict[str, Any] | None, + tool_call: CodeExecutionToolCall, + container_id: str | None, + ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]: + arguments = tool_call.get("arguments", "") + code = self._parse_code(arguments) + stdout = await self._run_tool_call( + container=container, params=params, arguments=arguments + ) + tool_call_id = ( + tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex + ) + return ( + { + "role": "tool", + "tool_call_id": tool_call_id, + "content": stdout, + }, + { + "id": f"ci_{uuid.uuid4().hex}", + "type": "code_interpreter_call", + "status": "completed", + "code": code, + "container_id": container_id, + "outputs": [{"type": "logs", "logs": stdout}] if stdout else [], + }, + ) + async def async_agentic_loop_cleanup_hook( self, plan: AgenticLoopPlan, kwargs: dict ) -> None: metadata = plan.metadata or {} if plan else {} await self._delete_container_for_cache_key(metadata.get("sandbox_key")) + @staticmethod + def _filter_agentic_loop_kwargs(kwargs: dict[str, object]) -> dict[str, object]: + return { + k: v + for k, v in kwargs.items() + if k not in {"litellm_logging_obj", "acompletion"} + and not is_interception_internal_key( + k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES + ) + } + + def _get_followup_tools( + self, tools: object, call_type: CallTypes | None + ) -> list[dict[str, Any]] | None: + if not isinstance(tools, list): + return None + return [ + ( + self._get_function_tool(call_type=call_type) + if isinstance(tool, dict) and tool.get("type") == "code_interpreter" + else tool + ) + for tool in tools + ] + + def _get_followup_optional_params( + self, optional_params: dict[str, object] + ) -> dict[str, object]: + drop_tool_choice = self._tool_choice_targets_code_interpreter( + optional_params.get("tool_choice") + ) + return { + k: v + for k, v in optional_params.items() + if k != "tools" and not (k == "tool_choice" and drop_tool_choice) + } + async def async_post_agentic_loop_response_hook( self, response: Any, plan: AgenticLoopPlan, kwargs: dict ) -> Any: @@ -420,7 +708,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger): return list(messages) return [] - def _extract_code_execution_tool_calls(self, response: Any) -> list[dict[str, Any]]: + def _extract_code_execution_tool_calls( + self, response: object + ) -> list[CodeExecutionToolCall]: if isinstance(response, dict): output = response.get("output", []) else: @@ -446,6 +736,82 @@ class CodeInterpreterInterceptionLogger(CustomLogger): if self._is_code_execution_call(item) ] + def _extract_chat_completion_code_execution_tool_calls( + self, response: ModelResponse | dict[str, Any] + ) -> list[CodeExecutionToolCall]: + model_response = self._to_model_response(response) + if model_response is None: + return [] + choices = model_response.choices or [] + if not choices: + return [] + message = choices[0].message + tool_calls = message.tool_calls or [] + + return [ + normalized + for tool_call in tool_calls + if (normalized := self._normalize_chat_completion_tool_call(tool_call)) + is not None + ] + + @staticmethod + def _normalize_chat_completion_tool_call( + tool_call: ChatCompletionMessageToolCall, + ) -> CodeExecutionToolCall | None: + if ( + tool_call.type != "function" + or tool_call.function.name != LITELLM_CODE_EXECUTION_TOOL_NAME + ): + return None + + arguments = tool_call.function.arguments + if isinstance(arguments, dict): + arguments = json.dumps(arguments) + elif not isinstance(arguments, str): + arguments = "" if arguments is None else str(arguments) + + return { + "id": tool_call.id, + "call_id": tool_call.id, + "type": "function", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": arguments, + } + + @staticmethod + def _build_chat_completion_assistant_message( + tool_calls: list[CodeExecutionToolCall], + ) -> ChatCompletionAssistantMessage: + return { + "role": "assistant", + "tool_calls": [ + cast( + ChatCompletionAssistantToolCall, + { + "id": tool_call.get("id"), + "type": "function", + "function": { + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": tool_call.get("arguments", ""), + }, + }, + ) + for tool_call in tool_calls + ], + } + + @staticmethod + def _to_model_response( + response: ModelResponse | dict[str, Any], + ) -> ModelResponse | None: + if isinstance(response, ModelResponse): + return response + try: + return ModelResponse(**response) + except (TypeError, ValidationError): + return None + def _is_code_execution_call(self, item: Any) -> bool: if isinstance(item, dict): return ( diff --git a/litellm/integrations/focus/destinations/mavvrik_destination.py b/litellm/integrations/focus/destinations/mavvrik_destination.py index 1e3c98b9a70..659f608a3e1 100644 --- a/litellm/integrations/focus/destinations/mavvrik_destination.py +++ b/litellm/integrations/focus/destinations/mavvrik_destination.py @@ -3,6 +3,7 @@ Flow: 1. GET /metrics/agent/ai/{connection_id}/upload-url → GCS signed URL 2. PUT with CSV content + 3. PATCH /metrics/agent/ai/{connection_id} → advance metricsMarker """ from __future__ import annotations @@ -127,8 +128,6 @@ class FocusMavvrikDestination(FocusDestination): timeout=30.0, ) if resp.status_code == 410: - # Connector has been disconnected in Mavvrik — reset flag so next - # delivery attempt re-registers after it becomes active again. self._registered = False raise RuntimeError( "Mavvrik FOCUS destination: connector is disconnected (410). " @@ -273,14 +272,35 @@ class FocusMavvrikDestination(FocusDestination): pass raise + async def _update_metrics_marker(self, date_epoch: int) -> None: + """PATCH agent endpoint to advance metricsMarker after a successful upload.""" + resp = await self._http.client.request( + method="PATCH", + url=self._agent_url, + headers=self._auth_headers, + json={"metricsMarker": date_epoch}, + timeout=30.0, + ) + if resp.status_code == 410: + self._registered = False + raise RuntimeError( + "Mavvrik FOCUS destination: connector is disconnected (410). " + "Re-enable the connection in the Mavvrik dashboard." + ) + if resp.status_code >= 400: + verbose_logger.warning( + "Mavvrik FOCUS destination: failed to update metricsMarker (%s): %s", + resp.status_code, + resp.text[:200], + ) + return + verbose_logger.debug( + "Mavvrik FOCUS destination: metricsMarker advanced to %s", date_epoch + ) + async def get_metrics_marker(self) -> Optional[int]: """Register with Mavvrik and return the current metricsMarker. - The metricsMarker is a Unix timestamp (seconds) representing the last - date Mavvrik has successfully ingested. Called on every scheduled run - so the logger can detect and catch up any dates missed due to previous - export failures. - Always calls the Mavvrik register API — unlike deliver() which skips registration once _registered is True, catch-up requires a fresh marker value on every run. @@ -328,6 +348,7 @@ class FocusMavvrikDestination(FocusDestination): return date_str = time_window.start_time.strftime("%Y-%m-%d") + date_epoch = int(time_window.start_time.timestamp()) verbose_logger.debug( "Mavvrik FOCUS destination: uploading %d bytes for date=%s (%s)", @@ -339,6 +360,7 @@ class FocusMavvrikDestination(FocusDestination): await self._ensure_registered() signed_url = await self._get_signed_url(date_str) await self._upload_to_gcs(signed_url, content) + await self._update_metrics_marker(date_epoch) verbose_logger.debug( "Mavvrik FOCUS destination: upload complete for date=%s", date_str diff --git a/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py b/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py index 47d3e1da7bc..bbc9d1a6330 100644 --- a/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py +++ b/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py @@ -149,8 +149,8 @@ class MavvrikFocusLogger(FocusLogger): On each run: 1. Register with Mavvrik → get metricsMarker (last successfully ingested date) - 2. If metricsMarker is behind yesterday, catch up missed dates (capped at - _MAX_CATCHUP_DAYS to avoid runaway loops on long outages) + 2. If metricsMarker is behind yesterday (or 0/None for a fresh connector), + catch up missed dates (capped at _MAX_CATCHUP_DAYS) 3. Export yesterday (today's daily window) This ensures a failed export on day N is automatically retried on day N+1 @@ -177,13 +177,21 @@ class MavvrikFocusLogger(FocusLogger): last_ingested = _parse_metrics_marker(marker) - # Catch up missed dates, capped at _MAX_CATCHUP_DAYS - if last_ingested and last_ingested < yesterday: - # Never go further back than _MAX_CATCHUP_DAYS from yesterday - earliest_catchup = yesterday - timedelta(days=self._MAX_CATCHUP_DAYS - 1) - catch_up_date = max(last_ingested + timedelta(days=1), earliest_catchup) + # Catch up missed dates, capped at _MAX_CATCHUP_DAYS. + # last_ingested=None means metricsMarker=0 (fresh connector, never ingested) — + # treat the same as being _MAX_CATCHUP_DAYS behind so we export all available history. + earliest_catchup = yesterday - timedelta(days=self._MAX_CATCHUP_DAYS - 1) + if last_ingested is None or last_ingested < yesterday: + catch_up_date = ( + earliest_catchup + if last_ingested is None + else max(last_ingested + timedelta(days=1), earliest_catchup) + ) - if last_ingested + timedelta(days=1) < earliest_catchup: + if ( + last_ingested is not None + and last_ingested + timedelta(days=1) < earliest_catchup + ): verbose_proxy_logger.warning( "Mavvrik FOCUS export: metricsMarker is more than %d days behind " "(%s). Catching up from %s only; earlier data will not be re-exported.", @@ -197,18 +205,24 @@ class MavvrikFocusLogger(FocusLogger): "Mavvrik FOCUS export: catching up missed date %s", catch_up_date.date(), ) + # Use now as end_time for catch-up windows too — rows for old dates + # may have been flushed to DB well after their calendar day ended. + catch_up_end = min(catch_up_date + timedelta(days=1), now) window = FocusTimeWindow( start_time=catch_up_date, - end_time=catch_up_date + timedelta(days=1), + end_time=catch_up_end, frequency="daily", ) await self._export_window(window=window, limit=None) catch_up_date += timedelta(days=1) - # Export yesterday's window (the normal daily run) + # Export yesterday's window (the normal daily run). + # Use `now` as end_time so spend rows flushed after midnight are included. + # LiteLLM's DailyUserSpend rows for a given date keep getting updated_at + # bumped as the flush job runs; capping at midnight would miss those updates. window = FocusTimeWindow( start_time=yesterday, - end_time=yesterday + timedelta(days=1), + end_time=now, frequency="daily", ) await self._export_window(window=window, limit=None) @@ -253,6 +267,21 @@ class MavvrikFocusLogger(FocusLogger): ) if type(cb) is MavvrikFocusLogger ] + if not loggers and "mavvrik" in litellm.callbacks: + # The logger is registered as the string "mavvrik" but hasn't been + # instantiated yet (lazy init happens on first LLM call). Force it now + # so the scheduler can register the daily export job at startup. + from litellm.litellm_core_utils.litellm_logging import ( # noqa: PLC0415 + _init_custom_logger_compatible_class, + ) + + instance = _init_custom_logger_compatible_class( + logging_integration="mavvrik", + internal_usage_cache=None, + llm_router=None, + ) + if isinstance(instance, MavvrikFocusLogger): + loggers = [instance] if not loggers: verbose_proxy_logger.debug( "No MavvrikFocusLogger registered; skipping scheduler" diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index a109ba898ff..991b156ae64 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -1,6 +1,7 @@ """Typed configuration for the OpenTelemetry instrumentation.""" from enum import Enum +from functools import lru_cache from typing import Any, List from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator @@ -47,7 +48,12 @@ class _OTelV2Flag(BaseSettings): enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV)) +@lru_cache(maxsize=1) def is_otel_v2_enabled() -> bool: + # Resolved once at startup and cached: constructing the pydantic-settings + # model re-scans the environment and cost ~28us, which on the proxy hot path + # (auth, logging-callback setup) compounded into a measurable throughput + # regression. Tests that toggle the env must call ``is_otel_v2_enabled.cache_clear()``. return _OTelV2Flag().enabled diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py new file mode 100644 index 00000000000..938e892bd50 --- /dev/null +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -0,0 +1,332 @@ +# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api + +import json +from typing import cast + +from litellm._logging import verbose_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.integrations.custom_logger import ( + CHAT_COMPLETION_AGENTIC_SURFACE, + NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + AgenticLoopPlan, + AgenticLoopRequestPatch, + is_interception_internal_key, +) +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +_FOLLOWUP_INTERNAL_PARAMS = frozenset( + ( + "acompletion", + "litellm_logging_obj", + "custom_llm_provider", + "model_alias_map", + "stream_response", + "custom_prompt_dict", + "_agentic_loop_api_surface", + ) +) + + +def _gate_overridden(callback: CustomLogger) -> bool: + base = CustomLogger.async_should_run_agentic_loop + func = type(callback).async_should_run_agentic_loop + return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + + +def _build_plan_overridden(callback: CustomLogger) -> bool: + base = CustomLogger.async_build_agentic_loop_plan + func = type(callback).async_build_agentic_loop_plan + return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + + +def _post_hook_overridden(callback: CustomLogger) -> bool: + base = CustomLogger.async_post_agentic_loop_response_hook + func = type(callback).async_post_agentic_loop_response_hook + return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + + +def _coerce_int(value: object, default: int) -> int: + return int(value) if isinstance(value, (int, str)) else default + + +def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]: + depth = _coerce_int(kwargs.get("_agentic_loop_depth"), 0) + max_loops = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1) + raw_fingerprints = kwargs.get("_agentic_loop_fingerprints") + fingerprints = ( + [str(fp) for fp in raw_fingerprints] + if isinstance(raw_fingerprints, list) + else [] + ) + return depth, max_loops, fingerprints + + +def _fingerprint_tools(tool_calls: object) -> str: + try: + return json.dumps(tool_calls, sort_keys=True, default=str) + except Exception: + return str(tool_calls) + + +def _check_agentic_loop_safety( + tool_calls: object, + fingerprints: list[str], + depth: int, + max_loops: int, + model: str, +) -> str: + fingerprint = _fingerprint_tools(tool_calls) + if fingerprint in fingerprints: + raise ValueError( + "Agentic loop detected repeated tool-call fingerprint; aborting rerun" + ) + if depth >= max_loops: + raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + return fingerprint + + +def _wrap_response_as_fake_stream(response: object) -> object: + if getattr(response, "object", None) == "chat.completion.chunk": + return response + if not hasattr(response, "choices"): + return response + from litellm.llms.base_llm.base_model_iterator import ( + convert_model_response_to_streaming, + ) + + return convert_model_response_to_streaming(cast(ModelResponse, response)) + + +def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None: + metadata = kwargs_for_followup.get("litellm_metadata") + metadata = dict(metadata) if isinstance(metadata, dict) else {} + for key, value in kwargs_for_followup.items(): + if ( + key.startswith("_agentic_loop") + or key == "max_agentic_loops" + or is_interception_internal_key(key) + ): + metadata[key] = value + kwargs_for_followup["litellm_metadata"] = metadata + + +def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]: + return { + k: v + for k, v in source.items() + if not is_interception_internal_key( + k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES + ) + and k not in _FOLLOWUP_INTERNAL_PARAMS + } + + +async def _execute_chat_completion_agentic_plan( + *, + plan: AgenticLoopPlan, + callback: CustomLogger, + model: str, + optional_params: dict[str, object], + kwargs: dict[str, object], + logging_obj: object, + custom_llm_provider: str, + depth: int, + max_loops: int, + fingerprints: list[str], + fingerprint: str, +) -> object: + import litellm + + patch = plan.request_patch or AgenticLoopRequestPatch() + if patch.messages is None: + raise ValueError("Agentic loop plan missing patched messages") + + full_model_name = patch.model or model + if "/" not in full_model_name: + full_model_name = f"{custom_llm_provider}/{full_model_name}" + + optional_params_for_followup = {**optional_params, **patch.optional_params} + if patch.tools is not None: + optional_params_for_followup["tools"] = patch.tools + if "tool_choice" not in patch.optional_params: + optional_params_for_followup.pop("tool_choice", None) + + kwargs_for_followup = _filter_followup_kwargs(kwargs) + kwargs_for_followup.update( + { + k: v + for k, v in _filter_followup_kwargs(patch.kwargs).items() + if k not in optional_params_for_followup + } + ) + kwargs_for_followup["_agentic_loop_depth"] = depth + 1 + kwargs_for_followup["max_agentic_loops"] = max_loops + kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] + _add_agentic_loop_metadata(kwargs_for_followup) + + try: + response_followup = await litellm.acompletion( + model=full_model_name, + messages=patch.messages, + **optional_params_for_followup, + **kwargs_for_followup, + ) + if _post_hook_overridden(callback): + try: + response_followup = ( + await callback.async_post_agentic_loop_response_hook( + response=response_followup, plan=plan, kwargs=kwargs + ) + ) + except Exception as e: + _call_id = getattr(logging_obj, "litellm_call_id", "unknown") + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in " + "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s", + _call_id, + model, + str(e), + ) + if kwargs.get("_code_interpreter_interception_converted_stream") and not depth: + return _wrap_response_as_fake_stream(response_followup) + return response_followup + finally: + try: + await callback.async_agentic_loop_cleanup_hook(plan=plan, kwargs=kwargs) + except Exception as e: + _call_id = getattr(logging_obj, "litellm_call_id", "unknown") + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in " + "async_agentic_loop_cleanup_hook [call_id=%s model=%s]: %s", + _call_id, + model, + str(e), + ) + + +async def maybe_run_chat_completion_agentic_loop( + *, + response: ModelResponse, + model: str, + messages: list, + optional_params: dict, + kwargs: dict, + logging_obj: object, + custom_llm_provider: str, + stream: bool, +) -> ModelResponse | CustomStreamWrapper | None: + import litellm + + callbacks = litellm.callbacks + ( + getattr(logging_obj, "dynamic_success_callbacks", None) or [] + ) + depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) + tools = optional_params.get("tools", []) + + for callback in callbacks: + if not isinstance(callback, CustomLogger): + continue + if not _gate_overridden(callback): + continue + + gate_kwargs = { + **kwargs, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + "custom_llm_provider": custom_llm_provider, + } + try: + should_run, tool_calls = await callback.async_should_run_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=gate_kwargs, + ) + except Exception as e: + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in chat completion agentic gate: %s", + str(e), + ) + continue + + if not should_run: + continue + + fingerprint = _check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + + try: + plan_kwargs = { + **kwargs, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + "custom_llm_provider": custom_llm_provider, + } + if not _build_plan_overridden(callback): + return await callback.async_run_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=plan_kwargs, + ) + + plan = await callback.async_build_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=plan_kwargs, + ) + + if plan.response_override is not None: + return plan.response_override + if plan.terminate: + return response + if not plan.run_agentic_loop: + continue + + return await _execute_chat_completion_agentic_plan( + plan=plan, + callback=callback, + model=model, + optional_params=optional_params, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) + except Exception as e: + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", + str(e), + ) + + if ( + kwargs.get("_code_interpreter_interception_converted_stream") + and not depth + and hasattr(response, "choices") + ): + return cast( + "ModelResponse | CustomStreamWrapper", + _wrap_response_as_fake_stream(response), + ) + return None diff --git a/litellm/litellm_core_utils/completion_timeout.py b/litellm/litellm_core_utils/completion_timeout.py index 5350d88e593..70c6896a323 100644 --- a/litellm/litellm_core_utils/completion_timeout.py +++ b/litellm/litellm_core_utils/completion_timeout.py @@ -6,10 +6,7 @@ from typing import Callable, Optional, Union import httpx -from litellm.constants import ( - COMPLETION_HTTP_FALLBACK_SECONDS, - DEFAULT_REQUEST_TIMEOUT_SECONDS, -) +from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS class CompletionTimeout: @@ -22,17 +19,13 @@ class CompletionTimeout: """ Used when ``model_timeout`` and kwargs timeouts are all unset. - ``global_timeout`` is :attr:`litellm.request_timeout` (numeric / string), not - :class:`httpx.Timeout`. - - If it equals :data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS` (6000), - return :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`. Same if - ``None``. Otherwise return ``float(global_timeout)``. + ``global_timeout`` is the explicitly-configured ``litellm.request_timeout`` + (numeric / string) or ``None`` when it was never set. ``None`` falls back to + :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`; any explicit value + (including ``6000``) is honored. """ if global_timeout is None: return COMPLETION_HTTP_FALLBACK_SECONDS - if float(global_timeout) == float(DEFAULT_REQUEST_TIMEOUT_SECONDS): - return COMPLETION_HTTP_FALLBACK_SECONDS return float(global_timeout) @staticmethod @@ -50,11 +43,10 @@ class CompletionTimeout: 1. ``model_timeout`` (call argument / merged ``litellm_params``) 2. ``kwargs["timeout"]`` 3. ``kwargs["request_timeout"]`` - 4. Fallback from ``global_timeout`` (:attr:`litellm.request_timeout`) — if it is - the package default (6000), use 600 instead. + 4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``), + or 600 when nothing was configured. Coerce :class:`httpx.Timeout` when the provider does not support it. - Explicit ``6000`` on the model or in kwargs is kept as ``6000``. """ resolved: Union[float, str, httpx.Timeout] if model_timeout is not None: diff --git a/litellm/litellm_core_utils/request_timeout_resolver.py b/litellm/litellm_core_utils/request_timeout_resolver.py new file mode 100644 index 00000000000..146c39ce9f3 --- /dev/null +++ b/litellm/litellm_core_utils/request_timeout_resolver.py @@ -0,0 +1,29 @@ +"""Single source of truth for whether ``litellm.request_timeout`` was configured. + +``litellm.request_timeout`` always holds a value (the package default, +:data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS`), so a bare read can't +tell "user asked for this" from "nobody set it". This resolver answers that: + +* ``request_timeout_explicitly_set`` is the authoritative signal, set when the + value comes from the ``REQUEST_TIMEOUT`` env var or ``litellm_settings``. +* A runtime value that differs from the package default (e.g. ``litellm.request_timeout + = 300`` in SDK code) is also treated as explicit, for backwards compatibility. +""" + +from __future__ import annotations + +from typing import Optional + +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS + + +def get_configured_request_timeout() -> Optional[float]: + """Return the explicitly-configured ``litellm.request_timeout``, else ``None``.""" + import litellm + + timeout = float(litellm.request_timeout) + if litellm.request_timeout_explicitly_set: + return timeout + if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS): + return timeout + return None diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 5f474b0800e..e278483d689 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2005,11 +2005,29 @@ class CustomStreamWrapper: except StopIteration: if self.sent_last_chunk is True: - complete_streaming_response = litellm.stream_chunk_builder( - chunks=self.chunks, - messages=self.messages, - logging_obj=self.logging_obj, - ) + try: + complete_streaming_response = litellm.stream_chunk_builder( + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, + ) + except Exception as e: + # stream_chunk_builder can re-raise (as APIError) on large agentic + # streams. The raise originates inside this except-StopIteration block, + # so the sibling `except Exception` below does not catch it; it would + # escape __next__ and drop the request from SpendLogs. Recover + # best-effort usage from the raw chunks so cost is still tracked + verbose_logger.warning( + "stream_chunk_builder raised at end-of-stream (%s); logging " + "best-effort usage from chunks.", + str(e), + ) + try: + complete_streaming_response = self.model_response_creator( + chunk={"usage": calculate_total_usage(chunks=self.chunks)} + ) + except Exception: + complete_streaming_response = None response = self.model_response_creator() if complete_streaming_response is not None: @@ -2234,11 +2252,27 @@ class CustomStreamWrapper: except (StopAsyncIteration, StopIteration): if self.sent_last_chunk is True: # log the final chunk with accurate streaming values - complete_streaming_response = litellm.stream_chunk_builder( - chunks=self.chunks, - messages=self.messages, - logging_obj=self.logging_obj, - ) + try: + complete_streaming_response = litellm.stream_chunk_builder( + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, + ) + except Exception as e: + # see sync __next__: a raise from stream_chunk_builder inside this + # except handler escapes __anext__ and drops the request from SpendLogs. + # Recover best-effort usage from the raw chunks so cost is still tracked + verbose_logger.warning( + "stream_chunk_builder raised at end-of-stream (%s); logging " + "best-effort usage from chunks.", + str(e), + ) + try: + complete_streaming_response = self.model_response_creator( + chunk={"usage": calculate_total_usage(chunks=self.chunks)} + ) + except Exception: + complete_streaming_response = None response = self.model_response_creator() if complete_streaming_response is not None: diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 5d14f3cc4ae..d2e5b89371e 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -629,6 +629,7 @@ class ModelResponseIterator: Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], Dict[str, Any], + Optional[str], ]: """ Helper function to handle the content block delta @@ -636,6 +637,7 @@ class ModelResponseIterator: text = "" tool_use: Optional[ChatCompletionToolCallChunk] = None provider_specific_fields = {} + reasoning_content: Optional[str] = None content_block = ContentBlockDelta(**chunk) # type: ignore thinking_blocks: List[ Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] @@ -670,14 +672,24 @@ class ModelResponseIterator: thinking_content = content_block["delta"].get("thinking") if isinstance(thinking_content, str) and thinking_content: self.reasoning_content_chunks.append(thinking_content) - thinking_blocks = [ - ChatCompletionThinkingBlock( - type="thinking", - thinking=thinking_content or "", - signature=str(content_block["delta"].get("signature") or ""), - ) - ] - provider_specific_fields["thinking_blocks"] = thinking_blocks + reasoning_content = thinking_content + + signature = content_block["delta"].get("signature") + if isinstance(signature, str) and signature: + thinking_blocks = [ + ChatCompletionThinkingBlock( + type="thinking", + thinking="".join( + cast(str, block["delta"].get("thinking")) + for block in self.content_blocks + if isinstance(block["delta"].get("thinking"), str) + ), + signature=signature, + ) + ] + provider_specific_fields["thinking_blocks"] = thinking_blocks + if reasoning_content is None: + reasoning_content = "" elif ( "content" in content_block["delta"] and content_block["delta"].get("type") == "compaction_delta" @@ -688,25 +700,13 @@ class ModelResponseIterator: "content": content_block["delta"]["content"], } - return text, tool_use, thinking_blocks, provider_specific_fields - - def _handle_reasoning_content( - self, - thinking_blocks: List[ - Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] - ], - ) -> Optional[str]: - """ - Handle the reasoning content - """ - reasoning_content = None - for block in thinking_blocks: - thinking_content = cast(Optional[str], block.get("thinking")) - if reasoning_content is None: - reasoning_content = "" - if thinking_content is not None: - reasoning_content += thinking_content - return reasoning_content + return ( + text, + tool_use, + thinking_blocks, + provider_specific_fields, + reasoning_content, + ) def _handle_redacted_thinking_content( self, @@ -802,11 +802,8 @@ class ModelResponseIterator: tool_use, thinking_blocks, provider_specific_fields, + reasoning_content, ) = self._content_block_delta_helper(chunk=chunk) - if thinking_blocks: - reasoning_content = self._handle_reasoning_content( - thinking_blocks=thinking_blocks - ) elif type_chunk == "content_block_start": """ event: content_block_start diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c24c990f356..822b75b37f4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -229,6 +229,11 @@ DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING = ( "Sonnet 4.6+, and Mythos Preview." ) +DROP_UNSUPPORTED_SPEED_WARNING = ( + "Dropping unsupported `speed` for model=%s " + "(drop_params=True). Fast mode is only supported on select Opus models." +) + class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ @@ -374,6 +379,51 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): for level in ("low", "minimal", "medium", "high", "xhigh", "max") ) + @staticmethod + def _model_supports_speed_param( + model: str, custom_llm_provider: Optional[str] = None + ) -> bool: + """Whether the model accepts Anthropic's ``speed`` parameter (fast mode). + + Fast mode is direct Anthropic API-only (not Bedrock, Vertex, or Azure). + Those providers strip their prefix before this shared transform runs, so a + bare ``claude-opus-4-8`` would otherwise resolve to the direct-API entry; + the routed provider is checked explicitly to keep them out. + """ + if custom_llm_provider is not None and custom_llm_provider != "anthropic": + return False + return ( + AnthropicModelInfo._get_exact_model_capability(model, "supports_speed") + is True + ) + + @staticmethod + def _maybe_drop_speed_param( + model: str, + optional_params: dict, + drop_params: bool, + custom_llm_provider: Optional[str] = None, + ) -> None: + if "speed" not in optional_params: + return + if AnthropicConfig._model_supports_speed_param(model, custom_llm_provider): + return + if not (litellm.drop_params or drop_params): + speed_value = optional_params.get("speed") + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} does not support speed={speed_value!r}. " + "To drop unsupported params, set " + "`litellm.drop_params = True`." + ), + status_code=400, + ) + litellm.verbose_logger.warning( + DROP_UNSUPPORTED_SPEED_WARNING, + model, + ) + optional_params.pop("speed", None) + @staticmethod def _raise_invalid_reasoning_effort( model: str, value: Any, llm_provider: str @@ -1569,8 +1619,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_context_management ) elif param == "speed" and isinstance(value, str): - # Pass through Anthropic-specific speed parameter for fast mode optional_params["speed"] = value + AnthropicConfig._maybe_drop_speed_param( + model=model, + optional_params=optional_params, + drop_params=drop_params, + custom_llm_provider=self.custom_llm_provider, + ) elif param == "cache_control" and isinstance(value, dict): # Pass through top-level cache_control for automatic prompt caching optional_params["cache_control"] = value @@ -1875,6 +1930,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "has no thinking_blocks. The model won't use extended thinking for this turn." ) + AnthropicConfig._maybe_drop_speed_param( + model=model, + optional_params=optional_params, + drop_params=litellm.drop_params + or litellm_params.get("drop_params") is True, + custom_llm_provider=self.custom_llm_provider, + ) + headers = self.update_headers_with_optional_anthropic_beta( headers=headers, optional_params=optional_params ) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 5741513903c..0e41ef619ba 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -367,6 +367,16 @@ class AnthropicModelInfo(BaseLLMModelInfo): pass return None + @staticmethod + def _get_exact_model_capability(model: str, key: str) -> Optional[bool]: + """Read boolean capability ``key`` from the exact model-map entry only. + + Unlike ``_get_model_capability``, does not walk stripped provider aliases. + Use when a feature is tied to a specific host (e.g. Anthropic API fast mode). + """ + value = litellm.model_cost.get(model, {}).get(key) + return value if isinstance(value, bool) else None + @staticmethod def _supports_model_capability(model: str, key: str) -> bool: """Check a boolean capability ``key`` in the model map. diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index a3ac465c463..7b10a447bc8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -507,7 +507,10 @@ def anthropic_messages_handler( local_vars.update(kwargs) anthropic_messages_optional_request_params = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( - params=local_vars + params=local_vars, + model=model, + drop_params=litellm_params.get("drop_params") is True, + custom_llm_provider=custom_llm_provider, ) ) if is_reasoning_auto_summary_enabled(): diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py index c7c110ff3e3..8714939f025 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py @@ -84,8 +84,14 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): ) # Optional routing overrides for the advisor sub-call (e.g. proxy routing). # If not set in the tool definition, litellm resolves from env vars. - advisor_api_key: Optional[str] = advisor_tool.get("api_key") - advisor_api_base: Optional[str] = advisor_tool.get("api_base") + # The advisor tool is caller-controlled; only honor a client-supplied + # api_base/api_key when the proxy has enabled clientside credentials, + # otherwise let litellm resolve from server config. + advisor_api_key: Optional[str] = None + advisor_api_base: Optional[str] = None + if _allow_client_side_advisor_credentials(): + advisor_api_key = advisor_tool.get("api_key") + advisor_api_base = advisor_tool.get("api_base") # Build the synthetic tool definition the provider will receive. synthetic_advisor_tool = _make_synthetic_advisor_tool() @@ -181,6 +187,20 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): # --------------------------------------------------------------------------- +def _allow_client_side_advisor_credentials() -> bool: + """Whether a caller-supplied advisor api_base/api_key may be honored. + + Gated on the proxy's ``allow_client_side_credentials`` opt-in. When the + interceptor runs outside the proxy (SDK use), there is no admin boundary + to protect, so client-supplied routing is allowed. + """ + try: + from litellm.proxy.proxy_server import general_settings + except (ImportError, ModuleNotFoundError): + return True + return general_settings.get("allow_client_side_credentials") is True + + def _make_synthetic_advisor_tool() -> Dict: """Build a regular tool definition the executor provider can understand.""" return { diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index 88832fb3f63..42167e0fdaa 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -23,12 +23,19 @@ class AnthropicMessagesRequestUtils: @staticmethod def get_requested_anthropic_messages_optional_param( params: Dict[str, Any], + *, + model: str | None = None, + drop_params: bool = False, + custom_llm_provider: str | None = None, ) -> AnthropicMessagesRequestOptionalParams: """ Filter parameters to only include those defined in AnthropicMessagesRequestOptionalParams. Args: params: Dictionary of parameters to filter + model: Resolved model id; when set, unsupported params may be dropped + drop_params: Per-request drop_params flag (also respects litellm.drop_params) + custom_llm_provider: Routed provider; fast mode is gated to direct Anthropic Returns: AnthropicMessagesRequestOptionalParams instance with only the valid parameters @@ -37,6 +44,15 @@ class AnthropicMessagesRequestUtils: filtered_params = { k: v for k, v in params.items() if k in valid_keys and v is not None } + if model is not None: + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + AnthropicConfig._maybe_drop_speed_param( + model=model, + optional_params=filtered_params, + drop_params=drop_params, + custom_llm_provider=custom_llm_provider, + ) return cast(AnthropicMessagesRequestOptionalParams, filtered_params) diff --git a/litellm/llms/apiserpent/search/transformation.py b/litellm/llms/apiserpent/search/transformation.py index 1eb7d34c875..bc11875ba12 100644 --- a/litellm/llms/apiserpent/search/transformation.py +++ b/litellm/llms/apiserpent/search/transformation.py @@ -53,7 +53,13 @@ class APISerpentSearchConfig(BaseSearchConfig): api_base: Optional[str] = None, **kwargs, ) -> Dict: - api_key = api_key or get_secret_str("APISERPENT_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("APISERPENT_API_KEY",), + base_env_var="APISERPENT_API_BASE", + default_api_base=APISERPENT_BASE, + ) if not api_key: raise ValueError( "APISERPENT_API_KEY is not set. Set `APISERPENT_API_KEY` environment variable." diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 8f9d5cad7c4..4f7e98af780 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -377,6 +377,17 @@ class BaseConfig(ABC): ) -> "ModelResponse": pass + def transform_parsed_response_dict(self, parsed_response: dict) -> dict: + """ + Repair a parsed OpenAI-format response dict before generic conversion. + + Providers routed through the OpenAI SDK handler bypass transform_response, + which calls convert_to_model_response_object directly on the SDK's parsed + output. Override this to normalize a malformed response (e.g. github_copilot + returning empty choices for Anthropic-native Claude responses). + """ + return parsed_response + @abstractmethod def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index 166f876ba04..6603c64142b 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -85,6 +85,7 @@ class BaseRerankConfig(ABC): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: pass diff --git a/litellm/llms/base_llm/sandbox/transformation.py b/litellm/llms/base_llm/sandbox/transformation.py index 6ad945f47a3..1c012a15fdb 100644 --- a/litellm/llms/base_llm/sandbox/transformation.py +++ b/litellm/llms/base_llm/sandbox/transformation.py @@ -8,10 +8,14 @@ run code -> delete container; `code_interpreter_tool` combines all three. from typing import Any, Union +import httpx + from pydantic import Field, PrivateAttr from litellm.types.llms.base import LiteLLMPydanticObjectBase +SANDBOX_MAX_OUTPUT_BYTES = 10 * 1024 * 1024 + class ContainerHandle(LiteLLMPydanticObjectBase): """A live sandbox container. Carries everything needed to reach it again.""" @@ -53,7 +57,7 @@ class BaseSandboxConfig: *, template: str | None = None, timeout: int | None = None, - allow_internet_access: bool = True, + allow_internet_access: bool | None = None, api_key: str | None = None, **kwargs, ) -> ContainerHandle: @@ -77,3 +81,16 @@ class BaseSandboxConfig: **kwargs, ) -> bool: raise NotImplementedError("adelete_sandbox must be implemented by provider") + + async def _read_capped_lines(self, response: httpx.Response) -> list[str]: + lines: list[str] = [] + total = 0 + async for line in response.aiter_lines(): + total += len(line.encode("utf-8")) + if total > SANDBOX_MAX_OUTPUT_BYTES: + raise ValueError( + f"Sandbox output exceeded {SANDBOX_MAX_OUTPUT_BYTES} bytes; aborting " + "to avoid unbounded memory use." + ) + lines.append(line) + return lines diff --git a/litellm/llms/base_llm/search/transformation.py b/litellm/llms/base_llm/search/transformation.py index 4dfe86685fb..1581d8bb064 100644 --- a/litellm/llms/base_llm/search/transformation.py +++ b/litellm/llms/base_llm/search/transformation.py @@ -3,11 +3,13 @@ Base Search transformation configuration. """ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union +from urllib.parse import urlsplit import httpx from pydantic import PrivateAttr from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMPydanticObjectBase if TYPE_CHECKING: @@ -16,6 +18,29 @@ else: LiteLLMLoggingObj = Any +def _search_host(url: str) -> str: + return urlsplit(url).netloc.lower() + + +def _is_trusted_search_api_base( + caller_api_base: str, + default_api_base: str | None, + base_env_var: str | None, +) -> bool: + candidate = _search_host(caller_api_base) + if not candidate: + return False + trusted = { + _search_host(base) + for base in ( + default_api_base, + get_secret_str(base_env_var) if base_env_var else None, + ) + if base + } + return candidate in trusted + + class SearchResult(LiteLLMPydanticObjectBase): """Single search result.""" @@ -86,6 +111,60 @@ class BaseSearchConfig: "max_tokens_per_page", } + def _assert_trusted_api_base_for_server_credential( + self, + caller_api_base: str | None, + default_api_base: str | None, + base_env_var: str | None, + credential_name: str, + ) -> None: + """ + Block sending a server-managed credential to a caller-chosen host. + + A caller-supplied api_base is honored when constructing the request URL, so + falling back to a server-configured secret while the caller controls the host + leaks that secret. The provider default and the operator's own api_base + override are the only trusted destinations for a server-managed credential. + """ + if not caller_api_base: + return + if _is_trusted_search_api_base(caller_api_base, default_api_base, base_env_var): + return + raise ValueError( + f"Refusing to send the server-configured {credential_name} to the " + f"caller-supplied api_base '{caller_api_base}'. Pass an explicit api_key " + f"when overriding api_base for this search provider." + ) + + def resolve_server_api_key( + self, + *, + caller_api_key: str | None, + caller_api_base: str | None, + key_env_vars: tuple[str, ...], + base_env_var: str | None, + default_api_base: str | None, + ) -> str | None: + """ + Resolve a single-secret search API key, falling back to a server-managed + secret only when the request targets a trusted host. + + Returns the caller's key when provided, otherwise the first set + server-managed secret (or None when none is set, for keyless providers). + """ + if caller_api_key: + return caller_api_key + server_key = next( + (key for key in (get_secret_str(var) for var in key_env_vars) if key), + None, + ) + if server_key is None: + return None + self._assert_trusted_api_base_for_server_credential( + caller_api_base, default_api_base, base_env_var, key_env_vars[0] + ) + return server_key + def validate_environment( self, headers: Dict, diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 2c9ea187912..c31462a735b 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -10,7 +10,6 @@ from typing import ( Callable, ClassVar, Dict, - List, Literal, Optional, Tuple, @@ -210,32 +209,11 @@ class BaseAWSLLM: """ Return a boto3.Credentials object """ - ## CHECK IS 'os.environ/' passed in - params_to_check: List[Optional[str]] = [ - aws_access_key_id, - aws_secret_access_key, - aws_session_token, - aws_region_name, - aws_session_name, - aws_profile_name, - aws_role_name, - aws_web_identity_token, - aws_sts_endpoint, - aws_external_id, - ] - - # Iterate over parameters and update if needed - for i, param in enumerate(params_to_check): - if param and param.startswith("os.environ/"): - _v = get_secret(param) - if _v is not None and isinstance(_v, str): - params_to_check[i] = _v - elif param is None: # check if uppercase value in env - key = self.aws_authentication_params[i] - if key.upper() in os.environ: - params_to_check[i] = os.getenv(key.upper()) - - # Assign updated values back to parameters + # Only config-sourced credentials are expanded against the environment. + # os.environ/ references in the model config are resolved at load time, + # so any reference still present at this point is caller-supplied input and is + # left as-is rather than expanded into a process environment variable. Each + # unset param falls back to its matching fixed AWS_* ambient env var. ( aws_access_key_id, aws_secret_access_key, @@ -247,7 +225,21 @@ class BaseAWSLLM: aws_web_identity_token, aws_sts_endpoint, aws_external_id, - ) = params_to_check + ) = tuple( + value if value is not None else os.getenv(env_var) + for value, env_var in ( + (aws_access_key_id, "AWS_ACCESS_KEY_ID"), + (aws_secret_access_key, "AWS_SECRET_ACCESS_KEY"), + (aws_session_token, "AWS_SESSION_TOKEN"), + (aws_region_name, "AWS_REGION_NAME"), + (aws_session_name, "AWS_SESSION_NAME"), + (aws_profile_name, "AWS_PROFILE_NAME"), + (aws_role_name, "AWS_ROLE_NAME"), + (aws_web_identity_token, "AWS_WEB_IDENTITY_TOKEN"), + (aws_sts_endpoint, "AWS_STS_ENDPOINT"), + (aws_external_id, "AWS_EXTERNAL_ID"), + ) + ) verbose_logger.debug( "in get credentials\n" @@ -845,6 +837,20 @@ class BaseAWSLLM: f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}" ) + # get_secret() expands environment-variable references (an os.environ/ + # prefix, or a bare name matching an environment variable). Config-sourced + # references are expanded at load time, so such a reference reaching here is + # caller-supplied input; reject it rather than expanding a process-environment + # value for use as the token. + if ( + aws_web_identity_token.startswith("os.environ/") + or aws_web_identity_token in os.environ + ): + raise AwsAuthError( + message="Invalid web identity token reference.", + status_code=400, + ) + oidc_token = get_secret(aws_web_identity_token) if oidc_token is None: diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index cbed2232be5..93306025b02 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, List, Optional from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) +from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.types.llms.openai import AllMessageValues if TYPE_CHECKING: @@ -21,10 +22,6 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -MANTLE_ENDPOINT_TEMPLATE = ( - "https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages" -) - class AmazonMantleConfig(AmazonAnthropicClaudeConfig): """ @@ -46,7 +43,13 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): stream: Optional[bool] = None, ) -> str: region = self._get_aws_region_name(optional_params=optional_params, model=model) - return MANTLE_ENDPOINT_TEMPLATE.format(region=region) + return build_mantle_messages_url( + api_base=api_base, + aws_bedrock_runtime_endpoint=optional_params.get( + "aws_bedrock_runtime_endpoint" + ), + region=region, + ) def validate_environment( self, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9f58e5c0f1c..0c4e7cf2568 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -601,6 +601,15 @@ def extract_model_name_from_bedrock_arn(model: str) -> str: return model +def is_bedrock_application_inference_profile_arn(model: str) -> bool: + """ + An application inference profile ARN ends in an opaque id with no provider + substring, so the invoke path cannot resolve a provider from it. Such ARNs + must use the converse route, which needs no provider. + """ + return ":application-inference-profile/" in model + + def strip_bedrock_routing_prefix(model: str) -> str: """Strip LiteLLM routing prefixes from model name.""" for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "nova-2/", "nova/"]: @@ -622,6 +631,31 @@ def strip_bedrock_throughput_suffix(model: str) -> str: return model +MANTLE_MESSAGES_PATH = "/anthropic/v1/messages" + + +def build_mantle_messages_url( + api_base: Optional[str], + aws_bedrock_runtime_endpoint: Optional[str], + region: str, +) -> str: + """Build the bedrock-mantle Anthropic /messages URL. + + Honors an explicit endpoint override (``api_base``, then + ``aws_bedrock_runtime_endpoint``) so private VPC / VPCE / GovCloud Mantle + endpoints are reachable; otherwise falls back to the public regional host. + The mantle messages path is appended unless the override already carries it, + so callers can pass either the host or the full messages URL. + """ + override = api_base or aws_bedrock_runtime_endpoint + if override: + base = override.rstrip("/") + if base.endswith(MANTLE_MESSAGES_PATH): + return base + return f"{base}{MANTLE_MESSAGES_PATH}" + return f"https://bedrock-mantle.{region}.api.aws{MANTLE_MESSAGES_PATH}" + + def get_bedrock_base_model(model: str) -> str: """ Get the base model from the given model name. @@ -891,6 +925,9 @@ class BedrockModelInfo(BaseLLMModelInfo): ) or _model_after_bedrock.startswith("nova/"): return "converse" + if is_bedrock_application_inference_profile_arn(model): + return "converse" + base_model = BedrockModelInfo.get_base_model(model) alt_model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model) if ( diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 900d9aa97d8..94e7f90b719 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -8,6 +8,7 @@ stripping that are specific to the bedrock-mantle endpoint. from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from litellm.llms.bedrock.common_utils import build_mantle_messages_url from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) @@ -20,10 +21,6 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -MANTLE_ENDPOINT_TEMPLATE = ( - "https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages" -) - class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): """ @@ -43,7 +40,13 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): stream: Optional[bool] = None, ) -> str: region = self._get_aws_region_name(optional_params=optional_params, model=model) - return MANTLE_ENDPOINT_TEMPLATE.format(region=region) + return build_mantle_messages_url( + api_base=api_base, + aws_bedrock_runtime_endpoint=optional_params.get( + "aws_bedrock_runtime_endpoint" + ), + region=region, + ) def validate_anthropic_messages_environment( self, diff --git a/litellm/llms/brave/search/transformation.py b/litellm/llms/brave/search/transformation.py index 9dfcd6bc75a..8ffe7dcb126 100644 --- a/litellm/llms/brave/search/transformation.py +++ b/litellm/llms/brave/search/transformation.py @@ -115,7 +115,13 @@ class BraveSearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("BRAVE_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("BRAVE_API_KEY",), + base_env_var="BRAVE_API_BASE", + default_api_base=self.BRAVE_API_BASE, + ) if not api_key: raise ValueError( diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index 66e253f304d..68f08741cc5 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -1,26 +1,15 @@ -import json -import time -from typing import AsyncIterator, Iterator, List, Optional, Union +from typing import List, Optional, Union import httpx -import litellm -from litellm.litellm_core_utils.url_utils import encode_url_path_segments -from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.chat.transformation import ( - BaseConfig, - BaseLLMException, - LiteLLMLoggingObj, +from litellm._logging import verbose_logger +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import ( + get_secret_str, + normalize_nonempty_secret_str, ) -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import ( - ChatCompletionToolCallChunk, - ChatCompletionUsageBlock, - GenericStreamingChunk, - ModelResponse, - Usage, -) class CloudflareError(BaseLLMException): @@ -34,26 +23,46 @@ class CloudflareError(BaseLLMException): message=message, request=self.request, response=self.response, - ) # Call the base class constructor with the parameters it needs + ) -class CloudflareChatConfig(BaseConfig): - max_tokens: Optional[int] = None - stream: Optional[bool] = None - - def __init__( +class CloudflareChatConfig(OpenAIGPTConfig): + def get_complete_url( self, - max_tokens: Optional[int] = None, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, stream: Optional[bool] = None, - ) -> None: - locals_ = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: - setattr(self.__class__, key, value) + ) -> str: + return super().get_complete_url( + api_base=self._resolve_api_base(api_base), + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + stream=stream, + ) - @classmethod - def get_config(cls): - return super().get_config() + @staticmethod + def _resolve_api_base(api_base: Optional[str]) -> str: + if not api_base: + account_id = normalize_nonempty_secret_str( + get_secret_str("CLOUDFLARE_ACCOUNT_ID") + ) + if account_id is None: + raise ValueError( + "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID in the environment or pass api_base explicitly" + ) + return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1" + trimmed = api_base.rstrip("/") + if trimmed.endswith("/ai/run"): + verbose_logger.warning( + "Cloudflare api_base ending in '/ai/run' is the legacy Workers AI path and no longer serves OpenAI-compatible requests; rewriting to the '/ai/v1' endpoint" + ) + return f"{trimmed[: -len('/ai/run')]}/ai/v1" + return api_base def validate_environment( self, @@ -67,107 +76,18 @@ class CloudflareChatConfig(BaseConfig): ) -> dict: if api_key is None: raise ValueError( - "Missing CloudflareError API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params" + "Missing Cloudflare API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params" ) - headers = { - "accept": "application/json", - "content-type": "apbplication/json", - "Authorization": "Bearer " + api_key, - } - return headers - - def get_complete_url( - self, - api_base: Optional[str], - api_key: Optional[str], - model: str, - optional_params: dict, - litellm_params: dict, - stream: Optional[bool] = None, - ) -> str: - if api_base is None: - account_id = get_secret_str("CLOUDFLARE_ACCOUNT_ID") - api_base = ( - f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/" - ) - encoded_model = encode_url_path_segments(model, field_name="model") - return api_base + encoded_model - - def get_supported_openai_params(self, model: str) -> List[str]: - return [ - "stream", - "max_tokens", - ] - - def map_openai_params( - self, - non_default_params: dict, - optional_params: dict, - model: str, - drop_params: bool, - ) -> dict: - supported_openai_params = self.get_supported_openai_params(model=model) - for param, value in non_default_params.items(): - if param == "max_completion_tokens": - optional_params["max_tokens"] = value - elif param in supported_openai_params: - optional_params[param] = value - return optional_params - - def transform_request( - self, - model: str, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: - config = litellm.CloudflareChatConfig.get_config() - for k, v in config.items(): - if k not in optional_params: - optional_params[k] = v - - data = { - "messages": messages, - **optional_params, - } - return data - - def transform_response( - self, - model: str, - raw_response: httpx.Response, - model_response: ModelResponse, - logging_obj: LiteLLMLoggingObj, - request_data: dict, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - encoding: str, - api_key: Optional[str] = None, - json_mode: Optional[bool] = None, - ) -> ModelResponse: - completion_response = raw_response.json() - - # Support both "response" and "response_text" keys (newer models like Nemotron use "response_text") - result = completion_response["result"] - model_response.choices[0].message.content = result.get("response") if result.get("response") is not None else result.get("response_text", "") # type: ignore - - prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model) - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) + return super().validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, ) - model_response.created = int(time.time()) - model_response.model = "cloudflare/" + model - usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - setattr(model_response, "usage", usage) - return model_response - def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: @@ -175,48 +95,3 @@ class CloudflareChatConfig(BaseConfig): status_code=status_code, message=error_message, ) - - def get_model_response_iterator( - self, - streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], - sync_stream: bool, - json_mode: Optional[bool] = False, - ): - return CloudflareChatResponseIterator( - streaming_response=streaming_response, - sync_stream=sync_stream, - json_mode=json_mode, - ) - - -class CloudflareChatResponseIterator(BaseModelResponseIterator): - def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: - try: - text = "" - tool_use: Optional[ChatCompletionToolCallChunk] = None - is_finished = False - finish_reason = "" - usage: Optional[ChatCompletionUsageBlock] = None - provider_specific_fields = None - - index = int(chunk.get("index", 0)) - - if "response" in chunk and chunk["response"] is not None: - text = chunk["response"] - elif "response_text" in chunk and chunk["response_text"] is not None: - text = chunk["response_text"] - - returned_chunk = GenericStreamingChunk( - text=text, - tool_use=tool_use, - is_finished=is_finished, - finish_reason=finish_reason, - usage=usage, - index=index, - provider_specific_fields=provider_specific_fields, - ) - - return returned_chunk - - except json.JSONDecodeError: - raise ValueError(f"Failed to decode JSON from chunk: {chunk}") diff --git a/litellm/llms/cohere/rerank/guardrail_translation/handler.py b/litellm/llms/cohere/rerank/guardrail_translation/handler.py index e9a5823d2b8..0824e1cca41 100644 --- a/litellm/llms/cohere/rerank/guardrail_translation/handler.py +++ b/litellm/llms/cohere/rerank/guardrail_translation/handler.py @@ -26,11 +26,18 @@ class CohereRerankHandler(BaseTranslation): The handler specifically processes: - The 'query' parameter (string) + - The 'instruction' parameter (string), when present Note: Documents are not processed by guardrails as they are the corpus being searched, not user input. """ + # User-controlled free-text fields that reach the model and must be + # scanned. 'instruction' is folded into the prompt by instruction-aware + # rerankers (e.g. hosted vLLM / Qwen3-Reranker), so it is as sensitive as + # 'query'; omitting it would let a caller smuggle content past guardrails. + _SCANNED_FIELDS = ("query", "instruction") + async def process_input_messages( self, data: dict, @@ -38,42 +45,55 @@ class CohereRerankHandler(BaseTranslation): litellm_logging_obj: Optional[Any] = None, ) -> Any: """ - Process input query by applying guardrails. + Process input text fields ('query' and 'instruction') by applying + guardrails and writing the sanitized values back. Args: - data: Request data dictionary containing 'query' + data: Request data dictionary containing 'query' and optionally + 'instruction' guardrail_to_apply: The guardrail instance to apply Returns: - Modified data with guardrails applied to query only + Modified data with guardrails applied to query/instruction only """ - # Process query only - query = data.get("query") - if query is not None and isinstance(query, str): - inputs = GenericGuardrailAPIInputs(texts=[query]) - # Include model information if available - model = data.get("model") - if model: - inputs["model"] = model - guardrailed_inputs = await guardrail_to_apply.apply_guardrail( - inputs=inputs, - request_data=data, - input_type="request", - logging_obj=litellm_logging_obj, + # Collect every scannable text field in a stable order so the + # guardrailed results can be written back to the right key by index. + fields_to_scan = [ + (key, data[key]) + for key in self._SCANNED_FIELDS + if isinstance(data.get(key), str) + ] + if not fields_to_scan: + verbose_proxy_logger.debug( + "Rerank: No query/instruction to process or not strings" ) - guardrailed_texts = guardrailed_inputs.get("texts", []) - data["query"] = guardrailed_texts[0] if guardrailed_texts else query + return data - verbose_proxy_logger.debug( - "Rerank: Applied guardrail to query. " - "Original length: %d, New length: %d", - len(query), - len(data["query"]), - ) - else: - verbose_proxy_logger.debug( - "Rerank: No query to process or query is not a string" - ) + inputs = GenericGuardrailAPIInputs(texts=[value for _, value in fields_to_scan]) + # Include model information if available + model = data.get("model") + if model: + inputs["model"] = model + guardrailed_inputs = await guardrail_to_apply.apply_guardrail( + inputs=inputs, + request_data=data, + input_type="request", + logging_obj=litellm_logging_obj, + ) + guardrailed_texts = guardrailed_inputs.get("texts", []) + + for idx, (key, original) in enumerate(fields_to_scan): + # Defensive: only write back when the guardrail returned a value for + # this index; otherwise keep the original (never forward unscanned). + if idx < len(guardrailed_texts): + data[key] = guardrailed_texts[idx] + verbose_proxy_logger.debug( + "Rerank: Applied guardrail to %s. " + "Original length: %d, New length: %d", + key, + len(original), + len(data[key]), + ) return data diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index 64ae8e8ffa7..d875f420310 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -57,6 +57,7 @@ class CohereRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: """ Map Cohere rerank params diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index 4c800d6455d..0dcb10d5664 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -49,6 +49,7 @@ class CohereRerankV2Config(CohereRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: """ Map Cohere rerank params diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 01c94476431..1000ab12803 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -42,6 +42,9 @@ from litellm.constants import ( HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, ) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, +) from litellm.types.llms.custom_http import * if TYPE_CHECKING: @@ -134,6 +137,18 @@ _DEFAULT_TIMEOUT = httpx.Timeout( timeout=COMPLETION_HTTP_FALLBACK_SECONDS, connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, ) + + +def _default_cached_client_timeout() -> httpx.Timeout: + """Timeout for cached default httpx clients; honors an explicit litellm.request_timeout.""" + configured = get_configured_request_timeout() + if configured is None: + return _DEFAULT_TIMEOUT + return httpx.Timeout( + timeout=configured, connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS + ) + + _STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0 _STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor( max_workers=50, @@ -1379,7 +1394,7 @@ def get_async_httpx_client( _new_client = AsyncHTTPHandler(**handler_params) else: _new_client = AsyncHTTPHandler( - timeout=_DEFAULT_TIMEOUT, + timeout=_default_cached_client_timeout(), shared_session=shared_session, ) @@ -1428,7 +1443,7 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler: } _new_client = HTTPHandler(**handler_params) else: - _new_client = HTTPHandler(timeout=_DEFAULT_TIMEOUT) + _new_client = HTTPHandler(timeout=_default_cached_client_timeout()) cache.set_cache( key=_cache_key_name, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 790bd0519d7..948c90f9f99 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,5 +1,6 @@ import json import ssl +from functools import lru_cache from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from typing import ( TYPE_CHECKING, @@ -13,6 +14,7 @@ from typing import ( Tuple, Union, cast, + get_type_hints, ) import httpx # type: ignore @@ -26,6 +28,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -101,6 +104,7 @@ from litellm.types.llms.openai import ( HttpxBinaryResponseContent, OpenAIFileObject, ResponseInputParam, + ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ) from litellm.types.rerank import RerankResponse @@ -135,6 +139,7 @@ from litellm.utils import ( ImageResponse, ModelResponse, ProviderConfigManager, + async_pre_call_deployment_hook, ) from .http_handler import get_shared_realtime_ssl_context @@ -184,6 +189,47 @@ def _google_genai_streaming_hidden_params( } +@lru_cache(maxsize=None) +def _responses_api_optional_request_param_names() -> frozenset[str]: + return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys()) + + +def _custom_logger_callbacks(logging_obj: Any) -> list[Any]: + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.litellm_logging import ( + get_custom_logger_compatible_class, + ) + + dynamic_success_callbacks = getattr(logging_obj, "dynamic_success_callbacks", None) + callbacks = list(litellm.callbacks) + if isinstance(dynamic_success_callbacks, (list, tuple)): + callbacks.extend(dynamic_success_callbacks) + + custom_loggers: list[Any] = [] + for cb in callbacks: + if isinstance(cb, str): + resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type] + if resolved is None: + continue + cb = resolved + if isinstance(cb, CustomLogger): + custom_loggers.append(cb) + return custom_loggers + + +def _has_pre_call_deployment_hook(logging_obj: Any) -> bool: + from litellm.integrations.custom_logger import CustomLogger + + base_func = CustomLogger.async_pre_call_deployment_hook + for cb in _custom_logger_callbacks(logging_obj): + cb_func = getattr(type(cb), "async_pre_call_deployment_hook", base_func) + if getattr(cb_func, "__func__", cb_func) is not getattr( + base_func, "__func__", base_func + ): + return True + return False + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1833,6 +1879,9 @@ class BaseLLMHTTPHandler: data = provider_config.transform_search_request( query=query, optional_params=optional_params, + api_key=api_key, + api_base=api_base, + headers=headers or {}, ) # Get complete URL (pass data for providers that need request body for URL construction) @@ -2224,12 +2273,92 @@ class BaseLLMHTTPHandler: ) raise ValueError("anthropic_messages_handler is not implemented for sync calls") + def _run_sync_responses_pre_call_deployment_hook( + self, + *, + model: str, + input: Union[str, ResponseInputParam], + custom_llm_provider: str, + response_api_optional_request_params: dict[str, Any], + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + ) -> tuple[ + str, + Union[str, ResponseInputParam], + str, + dict[str, Any], + GenericLiteLLMParams, + ]: + if not _has_pre_call_deployment_hook(logging_obj): + return ( + model, + input, + custom_llm_provider, + response_api_optional_request_params, + litellm_params, + ) + + modified_kwargs = run_async_function( + async_pre_call_deployment_hook, + { + **dict(litellm_params), + **response_api_optional_request_params, + "model": model, + "input": input, + "custom_llm_provider": custom_llm_provider, + }, + CallTypes.responses.value, + ) + if modified_kwargs is None: + return ( + model, + input, + custom_llm_provider, + response_api_optional_request_params, + litellm_params, + ) + + optional_param_names = _responses_api_optional_request_param_names() + updated_response_params = { + **response_api_optional_request_params, + **{ + key: value + for key, value in modified_kwargs.items() + if key in optional_param_names + }, + } + updated_litellm_params = GenericLiteLLMParams( + **{ + **dict(litellm_params), + **{ + key: value + for key, value in modified_kwargs.items() + if key not in optional_param_names + and key not in {"model", "input", "custom_llm_provider"} + }, + } + ) + return ( + str(modified_kwargs["model"]) if "model" in modified_kwargs else model, + cast( + Union[str, ResponseInputParam], + modified_kwargs["input"] if "input" in modified_kwargs else input, + ), + ( + str(modified_kwargs["custom_llm_provider"]) + if "custom_llm_provider" in modified_kwargs + else custom_llm_provider + ), + updated_response_params, + updated_litellm_params, + ) + def response_api_handler( self, model: str, input: Union[str, ResponseInputParam], responses_api_provider_config: BaseResponsesAPIConfig, - response_api_optional_request_params: Dict, + response_api_optional_request_params: dict[str, Any], custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, @@ -2276,6 +2405,21 @@ class BaseLLMHTTPHandler: shared_session=shared_session, ) + ( + model, + input, + custom_llm_provider, + response_api_optional_request_params, + litellm_params, + ) = self._run_sync_responses_pre_call_deployment_hook( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + logging_obj=logging_obj, + ) + if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( params={"ssl_verify": litellm_params.get("ssl_verify", None)} @@ -2414,9 +2558,27 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) ) - # Responses agentic interception (e.g. code interpreter) runs the follow-up - # loop via the async hook, so it is async-only for now; the sync path returns - # the initial response unchanged. + + if self._has_agentic_completion_hook(logging_obj): + final_response = run_async_function( + self._call_agentic_completion_hooks, + response=initial_response, + model=model, + messages=( + input + if isinstance(input, list) + else [{"role": "user", "content": input}] + ), + anthropic_messages_provider_config=responses_api_provider_config, + anthropic_messages_optional_request_params=response_api_optional_request_params, + logging_obj=logging_obj, + stream=False, + custom_llm_provider=custom_llm_provider, + kwargs=dict(litellm_params), + api_surface="responses", + ) + return final_response if final_response is not None else initial_response + return initial_response async def async_response_api_handler( @@ -4772,22 +4934,9 @@ class BaseLLMHTTPHandler: agentic callback is detected too. """ from litellm.integrations.custom_logger import CustomLogger - from litellm.litellm_core_utils.litellm_logging import ( - get_custom_logger_compatible_class, - ) base_func = CustomLogger.async_should_run_agentic_loop - callbacks = litellm.callbacks + ( - getattr(logging_obj, "dynamic_success_callbacks", None) or [] - ) - for cb in callbacks: - if isinstance(cb, str): - resolved = get_custom_logger_compatible_class(cb) # type: ignore[arg-type] - if resolved is None: - continue - cb = resolved - if not isinstance(cb, CustomLogger): - continue + for cb in _custom_logger_callbacks(logging_obj): cb_func = getattr(type(cb), "async_should_run_agentic_loop", base_func) if getattr(cb_func, "__func__", cb_func) is not getattr( base_func, "__func__", base_func @@ -5537,9 +5686,7 @@ class BaseLLMHTTPHandler: import websockets from websockets.asyncio.client import ClientConnection - url = self._append_query_params( - provider_config.get_complete_url(api_base, model, api_key), query_params - ) + url = provider_config.get_complete_url(api_base, model, api_key) headers = provider_config.validate_environment( headers=headers, model=model, diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 629f3cf4af7..745e85de7e3 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -116,6 +116,7 @@ class DashScopeRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: # qwen3-rerank accepts query/documents/top_n/return_documents. The # rest (rank_fields, max_*_per_doc) are silently dropped. diff --git a/litellm/llms/dataforseo/search/transformation.py b/litellm/llms/dataforseo/search/transformation.py index 27c10d740b5..701db586b72 100644 --- a/litellm/llms/dataforseo/search/transformation.py +++ b/litellm/llms/dataforseo/search/transformation.py @@ -61,9 +61,18 @@ class DataForSEOSearchConfig(BaseSearchConfig): password = get_secret_str("DATAFORSEO_PASSWORD") # If api_key is provided in "login:password" format, use it + caller_supplied_credentials = bool(api_key and ":" in api_key) if api_key and ":" in api_key: login, password = api_key.split(":", 1) + if not caller_supplied_credentials and login and password: + self._assert_trusted_api_base_for_server_credential( + api_base, + self.DATAFORSEO_API_BASE, + "DATAFORSEO_API_BASE", + "DATAFORSEO_LOGIN", + ) + if not login: raise ValueError( "DATAFORSEO_LOGIN is not set. Set `DATAFORSEO_LOGIN` environment variable or pass credentials in api_key parameter." diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index e4bfbcb2513..a5c36ca2e5f 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -104,6 +104,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: # Start with the basic parameters optional_rerank_params = {} diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 7ed3e484535..a316a3b9260 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -146,6 +146,56 @@ class DeepSeekChatConfig(OpenAIGPTConfig): and (optional_params.get("thinking") or {}).get("type") == "enabled" ) + @staticmethod + def _drop_unsupported_tools(optional_params: dict) -> dict: + """ + DeepSeek's /chat/completions only accepts tools of type "function". + + Requests bridged from /v1/responses can carry responses-API-native tool + types (e.g. a Codex CLI tool typed "namespace"); DeepSeek rejects the + whole request with `unknown variant '', expected 'function'` (issue + #30722). Drop the unsupported entries so the function tools still go + through, and drop the now-dangling tool_choice/parallel_tool_calls when + nothing callable survives. + + Only non-`function` tools are ever dropped, so a `tool_choice` that names + a specific function still points at a surviving tool and is left intact; + `tool_choice`/`parallel_tool_calls` are cleared only when no function + tool remains. + """ + tools = optional_params.get("tools") + if not isinstance(tools, list) or not tools: + return optional_params + + def _is_function_tool(tool: object) -> bool: + return isinstance(tool, dict) and tool.get("type") == "function" + + function_tools = [tool for tool in tools if _is_function_tool(tool)] + if len(function_tools) == len(tools): + return optional_params + + dropped_types = sorted( + { + str(tool.get("type")) if isinstance(tool, dict) else type(tool).__name__ + for tool in tools + if not _is_function_tool(tool) + } + ) + litellm.verbose_logger.warning( + "DeepSeek chat completions only supports function tools; dropping " + "unsupported tool type(s) %s before sending the request", + dropped_types, + ) + + cleaned = {k: v for k, v in optional_params.items() if k != "tools"} + if function_tools: + return {**cleaned, "tools": function_tools} + return { + k: v + for k, v in cleaned.items() + if k not in ("tool_choice", "parallel_tool_calls") + } + def transform_request( self, model: str, @@ -163,6 +213,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): (user explicitly enabled it), preventing spurious injection on models like deepseek-v3.2 that support thinking as opt-in but not always-on. """ + optional_params = self._drop_unsupported_tools(optional_params) if self._thinking_mode_active(model=model, optional_params=optional_params): messages = self._fill_reasoning_content(messages) return super().transform_request( @@ -185,6 +236,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): Async equivalent of transform_request — applies the same reasoning_content fix for multi-turn thinking-mode conversations. """ + optional_params = self._drop_unsupported_tools(optional_params) if self._thinking_mode_active(model=model, optional_params=optional_params): messages = self._fill_reasoning_content(messages) return await super().async_transform_request( diff --git a/litellm/llms/e2b/sandbox/transformation.py b/litellm/llms/e2b/sandbox/transformation.py index c279fab22ab..ecfc1642c97 100644 --- a/litellm/llms/e2b/sandbox/transformation.py +++ b/litellm/llms/e2b/sandbox/transformation.py @@ -16,6 +16,7 @@ from litellm.llms.base_llm.sandbox.transformation import ( BaseSandboxConfig, CodeExecutionResult, ContainerHandle, + SANDBOX_MAX_OUTPUT_BYTES, ) from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -29,7 +30,7 @@ E2B_DEFAULT_TEMPLATE = "code-interpreter-v1" E2B_DEFAULT_DOMAIN = "e2b.app" JUPYTER_PORT = 49999 DEFAULT_SANDBOX_TIMEOUT = 300 -MAX_OUTPUT_BYTES = 10 * 1024 * 1024 +MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES class E2BSandboxConfig(BaseSandboxConfig): @@ -49,7 +50,7 @@ class E2BSandboxConfig(BaseSandboxConfig): *, template: str | None = None, timeout: int | None = None, - allow_internet_access: bool = True, + allow_internet_access: bool | None = None, api_key: str | None = None, api_base: str | None = None, metadata: dict | None = None, @@ -62,7 +63,9 @@ class E2BSandboxConfig(BaseSandboxConfig): "templateID": template or E2B_DEFAULT_TEMPLATE, "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT, "secure": True, - "allow_internet_access": allow_internet_access, + "allow_internet_access": ( + True if allow_internet_access is None else allow_internet_access + ), } if metadata: body["metadata"] = metadata @@ -168,20 +171,6 @@ class E2BSandboxConfig(BaseSandboxConfig): handle._hidden_params = {} return handle - @staticmethod - async def _read_capped_lines(response: httpx.Response) -> list[str]: - lines: list[str] = [] - total = 0 - async for line in response.aiter_lines(): - total += len(line.encode("utf-8")) - if total > MAX_OUTPUT_BYTES: - raise ValueError( - f"Sandbox output exceeded {MAX_OUTPUT_BYTES} bytes; aborting to " - "avoid unbounded memory use." - ) - lines.append(line) - return lines - @staticmethod def _parse_lines(lines: list[str]) -> CodeExecutionResult: def _try_parse(stripped: str): @@ -192,10 +181,9 @@ class E2BSandboxConfig(BaseSandboxConfig): messages = tuple( parsed - for stripped in (line.strip() for line in lines) - if stripped - for parsed in (_try_parse(stripped),) - if parsed is not None + for line in lines + if (stripped := line.strip()) + if (parsed := _try_parse(stripped)) is not None ) def of_type(message_type: str): diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 7a34ededa6b..5cfd14aeaa9 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -65,7 +65,13 @@ class ExaAISearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("EXA_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("EXA_API_KEY",), + base_env_var="EXA_API_BASE", + default_api_base=self.EXA_AI_API_BASE, + ) if not api_key: raise ValueError( "EXA_API_KEY is not set. Set `EXA_API_KEY` environment variable." diff --git a/litellm/llms/fastcrw/search/transformation.py b/litellm/llms/fastcrw/search/transformation.py index ce702266e7b..b571a659cac 100644 --- a/litellm/llms/fastcrw/search/transformation.py +++ b/litellm/llms/fastcrw/search/transformation.py @@ -57,7 +57,13 @@ class FastCRWSearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("CRW_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("CRW_API_KEY",), + base_env_var="CRW_API_BASE", + default_api_base=self.FASTCRW_API_BASE, + ) if not api_key: raise ValueError( "CRW_API_KEY is not set. Set `CRW_API_KEY` environment variable." diff --git a/litellm/llms/firecrawl/search/transformation.py b/litellm/llms/firecrawl/search/transformation.py index 18cf1d28c4d..7e01ba58706 100644 --- a/litellm/llms/firecrawl/search/transformation.py +++ b/litellm/llms/firecrawl/search/transformation.py @@ -61,7 +61,13 @@ class FirecrawlSearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("FIRECRAWL_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("FIRECRAWL_API_KEY",), + base_env_var="FIRECRAWL_API_BASE", + default_api_base=self.FIRECRAWL_API_BASE, + ) if not api_key: raise ValueError( "FIRECRAWL_API_KEY is not set. Set `FIRECRAWL_API_KEY` environment variable." diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index 4a7b64b9b77..27309780c86 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -67,6 +67,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict[str, Any]: """ Map Cohere rerank params to Fireworks AI rerank params diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 74f6cd4d831..e153d00e6ab 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -103,6 +103,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # bypassing spend and budget accounting. self._pending_usage_metadata: Optional[dict] = None + def _include_function_response_id(self) -> bool: + """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it.""" + return True + @staticmethod def _usage_detail_alias(details: Any, defaults: Dict[str, int]) -> Dict[str, Any]: if not isinstance(details, dict): @@ -604,10 +608,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) # Build Gemini toolResponse format - function_response = { - "id": call_id, - "response": output_dict, - } + function_response: dict[str, Any] = {"response": output_dict} + if self._include_function_response_id() and call_id: + function_response["id"] = call_id if function_name: function_response["name"] = function_name diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 72dacb59f8a..9880ab1eb6e 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -194,6 +194,88 @@ class GithubCopilotConfig(OpenAIConfig): ) return text_content, tool_calls, thinking_blocks + @staticmethod + def _normalize_anthropic_usage(usage: dict) -> dict: + normalized = dict(usage) + if "input_tokens" in usage and "prompt_tokens" not in usage: + normalized["prompt_tokens"] = usage["input_tokens"] + if "output_tokens" in usage and "completion_tokens" not in usage: + normalized["completion_tokens"] = usage["output_tokens"] + if "total_tokens" not in normalized: + normalized["total_tokens"] = normalized.get( + "prompt_tokens", 0 + ) + normalized.get("completion_tokens", 0) + return normalized + + @classmethod + def _synthesize_choices_for_anthropic_native(cls, response_json: dict) -> dict: + """ + Synthesize a `choices` array from an Anthropic-native Copilot response. + + Newer Copilot Claude models (e.g. opus-4.7, opus-4.8) return content + blocks and `stop_reason` without an OpenAI-style `choices` array, and the + max_tokens=1 probe returns no content at all. Returns the response + unchanged when it already carries choices. + + See: https://github.com/BerriAI/litellm/issues/29391 + """ + if response_json.get("choices"): + return response_json + + content = "" + tool_calls: List[ChatCompletionToolCallChunk] = [] + thinking_blocks: Optional[List[Any]] = None + raw_content = response_json.get("content") + if isinstance(raw_content, list): + content, tool_calls, thinking_blocks = cls._parse_anthropic_native_content( + raw_content + ) + elif isinstance(raw_content, str): + content = raw_content + + stop_reason = response_json.get("stop_reason") + finish_reason_map = { + "end_turn": "stop", + "max_tokens": "length", + "stop_sequence": "stop", + "tool_use": "tool_calls", + } + if tool_calls: + finish_reason = "tool_calls" + elif stop_reason in finish_reason_map: + finish_reason = finish_reason_map[stop_reason] + elif content: + finish_reason = "stop" + else: + finish_reason = "length" + + message: dict = { + "role": "assistant", + "content": content if content or not tool_calls else None, + } + if tool_calls: + message["tool_calls"] = tool_calls + if thinking_blocks: + message["thinking_blocks"] = thinking_blocks + + synthesized = { + **response_json, + "choices": [ + {"index": 0, "message": message, "finish_reason": finish_reason} + ], + } + usage = response_json.get("usage") + if isinstance(usage, dict): + synthesized["usage"] = cls._normalize_anthropic_usage(usage) + return synthesized + + def transform_parsed_response_dict(self, parsed_response: dict) -> dict: + """ + Repair the OpenAI-SDK-parsed response on the handler path that bypasses + transform_response. See: https://github.com/BerriAI/litellm/issues/30927 + """ + return self._synthesize_choices_for_anthropic_native(parsed_response) + def transform_response( self, model: str, @@ -208,15 +290,6 @@ class GithubCopilotConfig(OpenAIConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> "ModelResponse": - """ - Handle newer Copilot models (e.g. claude-opus-4.7, claude-opus-4.8) that - return Anthropic-native format responses without a `choices` array. - - Synthesizes the missing `choices` from Anthropic-native fields, then - delegates to the parent so all standard post-processing applies. - - See: https://github.com/BerriAI/litellm/issues/29391 - """ try: response_json = raw_response.json() except Exception: @@ -235,70 +308,12 @@ class GithubCopilotConfig(OpenAIConfig): ) if not response_json.get("choices"): - content = "" - tool_calls: List[ChatCompletionToolCallChunk] = [] - thinking_blocks: Optional[List[Any]] = None - if "content" in response_json and isinstance( - response_json["content"], list - ): - content, tool_calls, thinking_blocks = ( - self._parse_anthropic_native_content(response_json["content"]) - ) - elif isinstance(response_json.get("content"), str): - content = response_json["content"] - - stop_reason = response_json.get("stop_reason") - finish_reason_map = { - "end_turn": "stop", - "max_tokens": "length", - "stop_sequence": "stop", - "tool_use": "tool_calls", - } - # Prefer tool_calls when blocks were extracted; otherwise map stop_reason. - if tool_calls: - finish_reason = "tool_calls" - elif stop_reason in finish_reason_map: - finish_reason = finish_reason_map[stop_reason] - elif content: - finish_reason = "stop" - else: - finish_reason = "length" - - message: dict = { - "role": "assistant", - "content": content if content or not tool_calls else None, - } - if tool_calls: - message["tool_calls"] = tool_calls - if thinking_blocks: - message["thinking_blocks"] = thinking_blocks - - response_json["choices"] = [ - { - "index": 0, - "message": message, - "finish_reason": finish_reason, - } - ] - - if "usage" in response_json: - usage = response_json["usage"] - if "input_tokens" in usage and "prompt_tokens" not in usage: - usage["prompt_tokens"] = usage["input_tokens"] - if "output_tokens" in usage and "completion_tokens" not in usage: - usage["completion_tokens"] = usage["output_tokens"] - if "total_tokens" not in usage: - usage["total_tokens"] = usage.get("prompt_tokens", 0) + usage.get( - "completion_tokens", 0 - ) - - # Build a patched response so super() sees valid JSON with choices - patched = httpx.Response( + response_json = self._synthesize_choices_for_anthropic_native(response_json) + raw_response = httpx.Response( status_code=raw_response.status_code, headers=raw_response.headers, content=json.dumps(response_json).encode(), ) - raw_response = patched return super().transform_response( model=model, diff --git a/litellm/llms/google_pse/search/transformation.py b/litellm/llms/google_pse/search/transformation.py index a8aa109cbf0..5cd3f2085a8 100644 --- a/litellm/llms/google_pse/search/transformation.py +++ b/litellm/llms/google_pse/search/transformation.py @@ -85,7 +85,13 @@ class GooglePSESearchConfig(BaseSearchConfig): Google PSE uses API key as a query parameter, not in headers. This method is called but headers are not used for authentication. """ - api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("GOOGLE_PSE_API_KEY",), + base_env_var="GOOGLE_PSE_API_BASE", + default_api_base=self.GOOGLE_PSE_API_BASE, + ) if not api_key: raise ValueError( "GOOGLE_PSE_API_KEY is not set. Set `GOOGLE_PSE_API_KEY` environment variable." @@ -137,6 +143,7 @@ class GooglePSESearchConfig(BaseSearchConfig): query: Union[str, List[str]], optional_params: dict, api_key: Optional[str] = None, + api_base: str | None = None, search_engine_id: Optional[str] = None, **kwargs, ) -> Dict: @@ -165,8 +172,16 @@ class GooglePSESearchConfig(BaseSearchConfig): # Google PSE only supports single string queries query = " ".join(query) - # Get API credentials - api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY") + # Get API credentials. The key is sent as a query param to api_base, so + # resolve it host-aware to avoid leaking a server-managed key to a + # caller-supplied host. + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("GOOGLE_PSE_API_KEY",), + base_env_var="GOOGLE_PSE_API_BASE", + default_api_base=self.GOOGLE_PSE_API_BASE, + ) search_engine_id = search_engine_id or get_secret_str("GOOGLE_PSE_ENGINE_ID") if not api_key: diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 60b6dc7d23d..d0c96f8b420 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -61,6 +61,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): "top_n", "rank_fields", "return_documents", + "instruction", ] def map_cohere_rerank_params( @@ -76,6 +77,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: """ Map parameters for Hosted VLLM rerank @@ -83,16 +85,22 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if max_chunks_per_doc is not None: raise ValueError("Hosted VLLM does not support max_chunks_per_doc") - return dict( - OptionalRerankParams( - query=query, - documents=documents, - top_n=top_n, - rank_fields=rank_fields, - return_documents=return_documents, - ) + mapped_params = OptionalRerankParams( + query=query, + documents=documents, + top_n=top_n, + rank_fields=rank_fields, + return_documents=return_documents, ) + # `instruction` is a vLLM-supported passthrough (folded into the model's + # chat_template_kwargs). Only forward it when explicitly set so omitting + # it leaves the request unchanged. + if instruction is not None: + mapped_params["instruction"] = instruction + + return dict(mapped_params) + def validate_environment( self, headers: dict, @@ -135,6 +143,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): top_n=optional_rerank_params.get("top_n", None), rank_fields=optional_rerank_params.get("rank_fields", None), return_documents=optional_rerank_params.get("return_documents", None), + instruction=optional_rerank_params.get("instruction", None), ) return rerank_request.model_dump(exclude_none=True) diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 2c847b617ef..4e409f31ed2 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -100,6 +100,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: optional_rerank_params = {} if non_default_params is not None: diff --git a/litellm/llms/jina_ai/rerank/transformation.py b/litellm/llms/jina_ai/rerank/transformation.py index 56be754fc34..0d48ed5edcd 100644 --- a/litellm/llms/jina_ai/rerank/transformation.py +++ b/litellm/llms/jina_ai/rerank/transformation.py @@ -45,6 +45,7 @@ class JinaAIRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: optional_params = {} supported_params = self.get_supported_cohere_rerank_params(model) diff --git a/litellm/llms/linkup/search/transformation.py b/litellm/llms/linkup/search/transformation.py index 2b17d5642ac..d27ae038f9e 100644 --- a/litellm/llms/linkup/search/transformation.py +++ b/litellm/llms/linkup/search/transformation.py @@ -61,7 +61,13 @@ class LinkupSearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("LINKUP_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("LINKUP_API_KEY",), + base_env_var="LINKUP_API_BASE", + default_api_base=self.LINKUP_API_BASE, + ) if not api_key: raise ValueError( "LINKUP_API_KEY is not set. Set `LINKUP_API_KEY` environment variable." diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index da8687bce72..9399ca88583 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -238,11 +238,11 @@ class MoonshotChatConfig(OpenAIGPTConfig): https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-tool_choice """ - messages.append( + optional_params.pop("tool_choice") + return [ + *messages, { "role": "user", "content": "Please select a tool to handle the current issue.", # Usually, the Kimi large language model understands the intention to invoke a tool and selects one for invocation - } - ) - optional_params.pop("tool_choice") - return messages + }, + ] diff --git a/litellm/llms/nvidia_nim/rerank/transformation.py b/litellm/llms/nvidia_nim/rerank/transformation.py index fc317293acc..8eee188bf46 100644 --- a/litellm/llms/nvidia_nim/rerank/transformation.py +++ b/litellm/llms/nvidia_nim/rerank/transformation.py @@ -117,6 +117,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: """ Map Cohere/OpenAI rerank params to Nvidia NIM format. diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index d009a085fab..dab277a7ba8 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -7,7 +7,10 @@ These models use token-based pricing instead of pixel-based pricing like DALL-E. from typing import Optional from litellm import verbose_logger -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, + generic_cost_per_token, +) from litellm.types.utils import ImageResponse, Usage @@ -16,54 +19,40 @@ def cost_calculator( image_response: ImageResponse, custom_llm_provider: Optional[str] = None, ) -> float: - """ - Calculate cost for OpenAI gpt-image models. - - Uses the same usage format as Responses API, so we reuse the helper - to transform to chat completion format and use generic_cost_per_token. - - Args: - model: The model name (e.g., "gpt-image-1", "gpt-image-2") - image_response: The ImageResponse containing usage data - custom_llm_provider: Optional provider name - - Returns: - float: Total cost in USD - """ + """Calculate cost for OpenAI gpt-image models (token-based pricing).""" usage = getattr(image_response, "usage", None) - if usage is None: verbose_logger.debug( f"No usage data available for {model}, cannot calculate token-based cost" ) return 0.0 - # If usage is already a Usage object with completion_tokens_details set, - # use it directly (it was already transformed in convert_to_image_response) + provider = custom_llm_provider or "openai" + + # A chat Usage with an explicit output breakdown: cost via generic_cost_per_token. if isinstance(usage, Usage) and usage.completion_tokens_details is not None: - chat_usage = usage - else: - # Transform ImageUsage to Usage using the existing helper - # ImageUsage has the same format as ResponseAPIUsage - from litellm.responses.utils import ResponseAPILoggingUtils - - chat_usage = ( - ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=provider ) + return prompt_cost + completion_cost - # Use generic_cost_per_token for cost calculation - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=chat_usage, - custom_llm_provider=custom_llm_provider or "openai", - ) + # ImageUsage / ResponseAPIUsage: reuse the shared helper (same path as + # azure_ai/gemini/vertex_ai). It prices generated output tokens at + # output_cost_per_image_token, classifying them as image tokens when the provider + # does not itemize output and splitting text/image when it does. + if getattr(usage, "input_tokens", None) is not None: + token_based_cost = calculate_image_response_cost_from_usage( + model=model, image_response=image_response, custom_llm_provider=provider + ) + if token_based_cost is not None: + return token_based_cost - total_cost = prompt_cost + completion_cost + # Fallback: a Usage with no output breakdown that the image helper can't read — + # cost via generic_cost_per_token (text rate) instead of returning 0.0. + if isinstance(usage, Usage): + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=provider + ) + return prompt_cost + completion_cost - verbose_logger.debug( - f"OpenAI gpt-image cost calculation for {model}: " - f"prompt_cost=${prompt_cost:.6f}, completion_cost=${completion_cost:.6f}, " - f"total=${total_cost:.6f}" - ) - - return total_cost + return 0.0 diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index ea905d8ebca..8237aaa010a 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -785,7 +785,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) logging_obj.model_call_details["response_headers"] = headers - stringified_response = response.model_dump() + stringified_response = ( + provider_config.transform_parsed_response_dict( + response.model_dump() + ) + ) logging_obj.post_call( input=messages, api_key=api_key, @@ -933,7 +937,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): timeout=timeout, logging_obj=logging_obj, ) - stringified_response = response.model_dump() + stringified_response = provider_config.transform_parsed_response_dict( + response.model_dump() + ) logging_obj.post_call( input=data["messages"], api_key=api_key, diff --git a/litellm/llms/opensandbox/__init__.py b/litellm/llms/opensandbox/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/litellm/llms/opensandbox/__init__.py @@ -0,0 +1 @@ + diff --git a/litellm/llms/opensandbox/sandbox/__init__.py b/litellm/llms/opensandbox/sandbox/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/litellm/llms/opensandbox/sandbox/__init__.py @@ -0,0 +1 @@ + diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py new file mode 100644 index 00000000000..dc9f8440d30 --- /dev/null +++ b/litellm/llms/opensandbox/sandbox/transformation.py @@ -0,0 +1,598 @@ +import asyncio +import json +import time +from typing import Union, cast + +import httpx + +from litellm.constants import ( + OPEN_SANDBOX_API_BASE_ENV_VAR, + OPEN_SANDBOX_API_KEY_ENV_VAR, + OPEN_SANDBOX_DEFAULT_CPU_LIMIT, + OPEN_SANDBOX_DEFAULT_ENTRYPOINT, + OPEN_SANDBOX_DEFAULT_LANGUAGE, + OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT, + OPEN_SANDBOX_DEFAULT_TEMPLATE, + OPEN_SANDBOX_DEFAULT_TIMEOUT, + OPEN_SANDBOX_EXECD_PORT, + OPEN_SANDBOX_POLL_INTERVAL, + OPEN_SANDBOX_READY_TIMEOUT, +) +from litellm.llms.base_llm.sandbox.transformation import ( + BaseSandboxConfig, + CodeExecutionResult, + ContainerHandle, + SANDBOX_MAX_OUTPUT_BYTES, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.custom_http import httpxSpecialProvider + +DEFAULT_SANDBOX_TIMEOUT = OPEN_SANDBOX_DEFAULT_TIMEOUT +DEFAULT_READY_TIMEOUT = OPEN_SANDBOX_READY_TIMEOUT +DEFAULT_POLL_INTERVAL = OPEN_SANDBOX_POLL_INTERVAL +MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES + + +class OpenSandboxSandboxConfig(BaseSandboxConfig): + def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler: + if client is not None: + return client + return get_async_httpx_client(llm_provider=httpxSpecialProvider.Sandbox) + + def validate_environment(self, api_key: str | None = None, **kwargs) -> str: + if api_key is not None: + return api_key + return get_secret_str(OPEN_SANDBOX_API_KEY_ENV_VAR) or "" + + async def acreate_sandbox( + self, + *, + template: str | None = None, + timeout: int | None = None, + allow_internet_access: bool | None = None, + api_key: str | None = None, + api_base: str | None = None, + metadata: dict[str, str] | None = None, + env_vars: dict[str, str] | None = None, + resource_limits: dict[str, str] | None = None, + resource_requests: dict[str, str] | None = None, + entrypoint: list[str] | tuple[str, ...] | None = None, + network_policy: dict[str, object] | None = None, + secure_access: bool = False, + use_server_proxy: bool = False, + ready_timeout: float | None = None, + poll_interval: float | None = None, + client: AsyncHTTPHandler | None = None, + **kwargs, + ) -> ContainerHandle: + key = self.validate_environment(api_key=api_key) + base = self._api_base(api_base) + ready_timeout_seconds = ( + float(ready_timeout) if ready_timeout is not None else DEFAULT_READY_TIMEOUT + ) + poll_interval_seconds = ( + float(poll_interval) if poll_interval is not None else DEFAULT_POLL_INTERVAL + ) + body = self._create_body( + template=template, + timeout=timeout, + allow_internet_access=allow_internet_access, + metadata=metadata, + env_vars=env_vars, + resource_limits=resource_limits, + resource_requests=resource_requests, + entrypoint=entrypoint, + network_policy=network_policy, + secure_access=secure_access, + ) + + response = cast( + httpx.Response, + await self._http(client).post( + url=f"{base}/sandboxes", + headers=self._lifecycle_headers(key), + json=body, + ), + ) + data = response.json() + sandbox_id = str(data["id"]) + + if self._sandbox_state(data) != "Running": + await self._wait_until_running( + sandbox_id=sandbox_id, + api_base=base, + headers=self._lifecycle_headers(key), + client=client, + ready_timeout=ready_timeout_seconds, + poll_interval=poll_interval_seconds, + ) + + endpoint, endpoint_headers = await self._wait_for_execd_endpoint( + sandbox_id=sandbox_id, + api_base=base, + headers=self._lifecycle_headers(key), + use_server_proxy=use_server_proxy, + client=client, + ready_timeout=ready_timeout_seconds, + poll_interval=poll_interval_seconds, + ) + + handle = ContainerHandle(id=sandbox_id, provider="opensandbox", domain=base) + handle._hidden_params = { + "api_base": base, + "api_key": key, + "execd_endpoint": endpoint, + "execd_headers": endpoint_headers, + "use_server_proxy": use_server_proxy, + } + return handle + + async def arun_code( + self, + *, + container: Union[ContainerHandle, str], + code: str, + api_key: str | None = None, + api_base: str | None = None, + language: str = OPEN_SANDBOX_DEFAULT_LANGUAGE, + use_server_proxy: bool = False, + ready_timeout: float | None = None, + poll_interval: float | None = None, + client: AsyncHTTPHandler | None = None, + **kwargs, + ) -> CodeExecutionResult: + handle = await self._ensure_handle( + container=container, + api_key=api_key, + api_base=api_base, + use_server_proxy=use_server_proxy, + ready_timeout=( + float(ready_timeout) + if ready_timeout is not None + else DEFAULT_READY_TIMEOUT + ), + poll_interval=( + float(poll_interval) + if poll_interval is not None + else DEFAULT_POLL_INTERVAL + ), + client=client, + ) + endpoint = str(handle._hidden_params["execd_endpoint"]) + endpoint_headers = self._as_str_dict(handle._hidden_params.get("execd_headers")) + base = str( + handle._hidden_params.get("api_base") + or handle.domain + or self._api_base(api_base) + ) + lines = await self._post_code( + url=f"{self._endpoint_base_url(endpoint, base)}/code", + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + "Cache-Control": "no-cache", + **endpoint_headers, + }, + body={ + "code": code, + "context": {"language": language}, + }, + client=client, + ) + return self._parse_lines(lines) + + async def adelete_sandbox( + self, + *, + container: Union[ContainerHandle, str], + api_key: str | None = None, + api_base: str | None = None, + client: AsyncHTTPHandler | None = None, + **kwargs, + ) -> bool: + handle = self._as_handle(container, api_base=api_base) + base = str(handle._hidden_params.get("api_base") or self._api_base(api_base)) + key = self._api_key(api_key=api_key, handle=handle) + try: + response = cast( + httpx.Response, + await self._http(client).delete( + url=f"{base}/sandboxes/{handle.id}", + headers=self._lifecycle_headers(key), + ), + ) + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + return False + raise + return 200 <= response.status_code < 300 + + async def _ensure_handle( + self, + *, + container: Union[ContainerHandle, str], + api_key: str | None, + api_base: str | None, + use_server_proxy: bool, + ready_timeout: float, + poll_interval: float, + client: AsyncHTTPHandler | None, + ) -> ContainerHandle: + handle = self._as_handle(container, api_base=api_base) + if handle._hidden_params.get("execd_endpoint"): + return handle + + base = str(handle._hidden_params.get("api_base") or self._api_base(api_base)) + key = self._api_key(api_key=api_key, handle=handle) + resolved_use_server_proxy = bool( + handle._hidden_params.get("use_server_proxy", use_server_proxy) + ) + endpoint, endpoint_headers = await self._wait_for_execd_endpoint( + sandbox_id=handle.id, + api_base=base, + headers=self._lifecycle_headers(key), + use_server_proxy=resolved_use_server_proxy, + client=client, + ready_timeout=ready_timeout, + poll_interval=poll_interval, + ) + handle.domain = base + handle._hidden_params = { + **handle._hidden_params, + "api_base": base, + "api_key": key, + "execd_endpoint": endpoint, + "execd_headers": endpoint_headers, + "use_server_proxy": resolved_use_server_proxy, + } + return handle + + async def _wait_until_running( + self, + *, + sandbox_id: str, + api_base: str, + headers: dict[str, str], + client: AsyncHTTPHandler | None, + ready_timeout: float, + poll_interval: float, + ) -> None: + deadline = time.monotonic() + ready_timeout + while True: + response = cast( + httpx.Response, + await self._http(client).get( + url=f"{api_base}/sandboxes/{sandbox_id}", + headers=headers, + ), + ) + data = response.json() + state = self._sandbox_state(data) + if state == "Running": + return + if state in {"Failed", "Stopping", "Terminated"}: + raise ValueError(f"OpenSandbox sandbox {sandbox_id} entered {state}") + if time.monotonic() >= deadline: + raise TimeoutError( + f"OpenSandbox sandbox {sandbox_id} was not Running within " + f"{ready_timeout} seconds" + ) + await asyncio.sleep(poll_interval) + + async def _wait_for_execd_endpoint( + self, + *, + sandbox_id: str, + api_base: str, + headers: dict[str, str], + use_server_proxy: bool, + client: AsyncHTTPHandler | None, + ready_timeout: float, + poll_interval: float, + ) -> tuple[str, dict[str, str]]: + deadline = time.monotonic() + ready_timeout + last_error: Exception | None = None + while True: + try: + return await self._get_execd_endpoint( + sandbox_id=sandbox_id, + api_base=api_base, + headers=headers, + use_server_proxy=use_server_proxy, + client=client, + ) + except httpx.HTTPStatusError as e: + if e.response.status_code != 404: + raise + last_error = e + except ValueError as e: + last_error = e + + if time.monotonic() >= deadline: + raise TimeoutError( + f"OpenSandbox execd endpoint for {sandbox_id} was not ready within " + f"{ready_timeout} seconds" + ) from last_error + await asyncio.sleep(poll_interval) + + async def _get_execd_endpoint( + self, + *, + sandbox_id: str, + api_base: str, + headers: dict[str, str], + use_server_proxy: bool, + client: AsyncHTTPHandler | None, + ) -> tuple[str, dict[str, str]]: + response = cast( + httpx.Response, + await self._http(client).get( + url=f"{api_base}/sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}", + headers=headers, + params={"use_server_proxy": use_server_proxy}, + ), + ) + data = response.json() + endpoint = data.get("endpoint") + if not endpoint: + raise ValueError( + f"OpenSandbox did not return an execd endpoint for {sandbox_id}" + ) + return str(endpoint), self._as_str_dict(data.get("headers")) + + async def _post_code( + self, + *, + url: str, + headers: dict[str, str], + body: dict[str, object], + client: AsyncHTTPHandler | None, + ) -> list[str]: + timeout = httpx.Timeout(connect=30.0, read=None, write=30.0, pool=None) + response = cast( + httpx.Response, + await self._http(client).post( + url=url, + headers=headers, + timeout=timeout, + json=body, + stream=True, + ), + ) + return await self._read_capped_lines(response) + + def _api_key(self, *, api_key: str | None, handle: ContainerHandle) -> str: + if api_key is not None: + return api_key + if "api_key" in handle._hidden_params: + return str(handle._hidden_params["api_key"]) + return self.validate_environment() + + @staticmethod + def _create_body( + *, + template: str | None, + timeout: int | None, + allow_internet_access: bool | None, + metadata: dict[str, str] | None, + env_vars: dict[str, str] | None, + resource_limits: dict[str, str] | None, + resource_requests: dict[str, str] | None, + entrypoint: list[str] | tuple[str, ...] | None, + network_policy: dict[str, object] | None, + secure_access: bool, + ) -> dict[str, object]: + body: dict[str, object] = { + "image": {"uri": template or OPEN_SANDBOX_DEFAULT_TEMPLATE}, + "entrypoint": list(entrypoint or OPEN_SANDBOX_DEFAULT_ENTRYPOINT), + "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT, + "resourceLimits": resource_limits + or OpenSandboxSandboxConfig._default_resource_limits(), + } + if metadata: + body["metadata"] = metadata + if env_vars: + body["env"] = env_vars + if resource_requests: + body["resourceRequests"] = resource_requests + if network_policy is not None: + body["networkPolicy"] = network_policy + elif allow_internet_access is not True: + body["networkPolicy"] = {"defaultAction": "deny", "egress": []} + if secure_access: + body["secureAccess"] = True + return body + + @staticmethod + def _default_resource_limits() -> dict[str, str]: + return { + "cpu": OPEN_SANDBOX_DEFAULT_CPU_LIMIT, + "memory": OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT, + } + + @staticmethod + def _sandbox_state(data: object) -> str | None: + if not isinstance(data, dict): + return None + status = data.get("status") + if not isinstance(status, dict): + return None + state = status.get("state") + return str(state) if state is not None else None + + @staticmethod + def _as_str_dict(value: object) -> dict[str, str]: + if not isinstance(value, dict): + return {} + return {str(k): str(v) for k, v in value.items()} + + @staticmethod + def _api_base(api_base: str | None) -> str: + base = api_base or get_secret_str(OPEN_SANDBOX_API_BASE_ENV_VAR) + if not base: + raise ValueError( + "OpenSandbox api_base is required. Pass api_base or set " + f"{OPEN_SANDBOX_API_BASE_ENV_VAR}." + ) + return str(base).rstrip("/") + + @staticmethod + def _lifecycle_headers(api_key: str) -> dict[str, str]: + headers = {"Content-Type": "application/json"} + if api_key: + headers["OPEN-SANDBOX-API-KEY"] = api_key + return headers + + @staticmethod + def _endpoint_base_url(endpoint: str, api_base: str) -> str: + normalized_endpoint = endpoint.rstrip("/") + if normalized_endpoint.startswith(("http://", "https://")): + return normalized_endpoint + protocol = api_base.split("://", 1)[0] if "://" in api_base else "http" + return f"{protocol}://{normalized_endpoint}" + + @staticmethod + def _as_handle( + container: Union[ContainerHandle, str], *, api_base: str | None + ) -> ContainerHandle: + if isinstance(container, ContainerHandle): + return container + handle = ContainerHandle( + id=str(container), + provider="opensandbox", + domain=OpenSandboxSandboxConfig._api_base(api_base), + ) + handle._hidden_params = {} + return handle + + @staticmethod + def _parse_lines(lines: list[str]) -> CodeExecutionResult: + messages = tuple( + event + for line in lines + if (event := OpenSandboxSandboxConfig._parse_sse_line(line)) is not None + ) + + def of_type(message_type: str): + return (m for m in messages if m.get("type") == message_type) + + error = next( + (OpenSandboxSandboxConfig._normalize_error(m) for m in of_type("error")), + None, + ) + execution_count = next( + ( + OpenSandboxSandboxConfig._as_int(m.get("execution_count")) + for m in of_type("execution_count") + if OpenSandboxSandboxConfig._as_int(m.get("execution_count")) + is not None + ), + None, + ) + + return CodeExecutionResult( + stdout="".join(str(m.get("text", "")) for m in of_type("stdout")), + stderr="".join(str(m.get("text", "")) for m in of_type("stderr")), + results=[ + OpenSandboxSandboxConfig._normalize_result(m) for m in of_type("result") + ], + error=error, + execution_count=execution_count, + ) + + @staticmethod + def _parse_sse_line(line: str) -> dict[str, object] | None: + stripped = line.strip() + if not stripped or stripped.startswith( + ( + ":", + "event:", + "id:", + "retry:", + ) + ): + return None + data = stripped[5:].strip() if stripped.startswith("data:") else stripped + if not data: + return None + try: + parsed = json.loads(data) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + if "type" not in parsed and "code" in parsed and "message" in parsed: + return { + "type": "error", + "error": { + "ename": str(parsed["code"]), + "evalue": str(parsed["message"]), + "traceback": [], + }, + } + return parsed + + @staticmethod + def _normalize_result(message: dict[str, object]) -> dict[str, object]: + results = message.get("results") + if isinstance(results, dict): + return {str(k): v for k, v in results.items()} + return { + str(k): v + for k, v in message.items() + if k not in {"type", "timestamp", "execution_count"} + } + + @staticmethod + def _normalize_error(message: dict[str, object]) -> dict[str, object]: + raw_error = message.get("error") + if isinstance(raw_error, dict): + name = OpenSandboxSandboxConfig._first_non_none_value( + raw_error, "ename", "name", default="" + ) + value = OpenSandboxSandboxConfig._first_non_none_value( + raw_error, "evalue", "value", default="" + ) + traceback = OpenSandboxSandboxConfig._first_non_none_value( + raw_error, "traceback", default=[] + ) + return { + "name": name, + "value": value, + "traceback": traceback, + } + return { + "name": OpenSandboxSandboxConfig._first_non_none_value( + message, "name", default="" + ), + "value": OpenSandboxSandboxConfig._first_non_none_value( + message, "value", "text", default="" + ), + "traceback": OpenSandboxSandboxConfig._first_non_none_value( + message, "traceback", default=[] + ), + } + + @staticmethod + def _as_int(value: object) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None + + @staticmethod + def _first_non_none_value( + values: dict[str, object], *keys: str, default: object + ) -> object: + return next( + (values[key] for key in keys if key in values and values[key] is not None), + default, + ) diff --git a/litellm/llms/parallel_ai/search/transformation.py b/litellm/llms/parallel_ai/search/transformation.py index 85602bf1d86..35a0d84df40 100644 --- a/litellm/llms/parallel_ai/search/transformation.py +++ b/litellm/llms/parallel_ai/search/transformation.py @@ -67,10 +67,12 @@ class ParallelAISearchConfig(BaseSearchConfig): api_base: Optional[str] = None, **kwargs, ) -> Dict: - api_key = ( - api_key - or get_secret_str("PARALLEL_AI_API_KEY") - or get_secret_str("PARALLEL_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"), + base_env_var="PARALLEL_AI_API_BASE", + default_api_base=self.PARALLEL_AI_API_BASE, ) if not api_key: raise ValueError( diff --git a/litellm/llms/perplexity/search/transformation.py b/litellm/llms/perplexity/search/transformation.py index ea96f87957c..55de52c5384 100644 --- a/litellm/llms/perplexity/search/transformation.py +++ b/litellm/llms/perplexity/search/transformation.py @@ -50,7 +50,13 @@ class PerplexitySearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("PERPLEXITYAI_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("PERPLEXITYAI_API_KEY",), + base_env_var="PERPLEXITY_API_BASE", + default_api_base=self.PERPLEXITY_API_BASE, + ) if not api_key: raise ValueError( "PERPLEXITYAI_API_KEY is not set. Set `PERPLEXITYAI_API_KEY` environment variable." diff --git a/litellm/llms/searchapi/search/transformation.py b/litellm/llms/searchapi/search/transformation.py index c04e1377f9c..ae8413684cc 100644 --- a/litellm/llms/searchapi/search/transformation.py +++ b/litellm/llms/searchapi/search/transformation.py @@ -74,7 +74,13 @@ class SearchAPIConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("SEARCHAPI_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("SEARCHAPI_API_KEY",), + base_env_var="SEARCHAPI_API_BASE", + default_api_base=self.SEARCHAPI_API_BASE, + ) if not api_key: raise ValueError( @@ -114,6 +120,7 @@ class SearchAPIConfig(BaseSearchConfig): query: Union[str, List[str]], optional_params: dict, api_key: Optional[str] = None, + api_base: str | None = None, search_engine_id: Optional[str] = None, **kwargs, ) -> Dict: @@ -137,8 +144,16 @@ class SearchAPIConfig(BaseSearchConfig): if isinstance(query, list): query = " ".join(query) - # Get API key from parameter or environment - api_key = api_key or get_secret_str("SEARCHAPI_API_KEY") + # Get API key from parameter or environment. The key is sent as a query + # param to api_base, so resolve it host-aware to avoid leaking a + # server-managed key to a caller-supplied host. + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("SEARCHAPI_API_KEY",), + base_env_var="SEARCHAPI_API_BASE", + default_api_base=self.SEARCHAPI_API_BASE, + ) if not api_key: raise ValueError( "SEARCHAPI_API_KEY is not set. Set `SEARCHAPI_API_KEY` environment variable." diff --git a/litellm/llms/searxng/search/transformation.py b/litellm/llms/searxng/search/transformation.py index ee6f3895721..ff68be5709e 100644 --- a/litellm/llms/searxng/search/transformation.py +++ b/litellm/llms/searxng/search/transformation.py @@ -61,7 +61,13 @@ class SearXNGSearchConfig(BaseSearchConfig): Some instances may require authentication via headers. """ # SearXNG typically doesn't require API keys, but support optional auth - api_key = api_key or get_secret_str("SEARXNG_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("SEARXNG_API_KEY",), + base_env_var="SEARXNG_API_BASE", + default_api_base=None, + ) if api_key: headers["Authorization"] = f"Bearer {api_key}" headers["Content-Type"] = "application/json" diff --git a/litellm/llms/serper/search/transformation.py b/litellm/llms/serper/search/transformation.py index 0daccbe652b..dd43f2d2dc9 100644 --- a/litellm/llms/serper/search/transformation.py +++ b/litellm/llms/serper/search/transformation.py @@ -55,7 +55,13 @@ class SerperSearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("SERPER_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("SERPER_API_KEY",), + base_env_var="SERPER_API_BASE", + default_api_base=self.SERPER_API_BASE, + ) if not api_key: raise ValueError( "SERPER_API_KEY is not set. Set `SERPER_API_KEY` environment variable." diff --git a/litellm/llms/tavily/search/transformation.py b/litellm/llms/tavily/search/transformation.py index ec96db96f36..647cfb5fa84 100644 --- a/litellm/llms/tavily/search/transformation.py +++ b/litellm/llms/tavily/search/transformation.py @@ -64,7 +64,13 @@ class TavilySearchConfig(BaseSearchConfig): """ Validate environment and return headers. """ - api_key = api_key or get_secret_str("TAVILY_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("TAVILY_API_KEY",), + base_env_var="TAVILY_API_BASE", + default_api_base=self.TAVILY_API_BASE, + ) if not api_key: raise ValueError( "TAVILY_API_KEY is not set. Set `TAVILY_API_KEY` environment variable." diff --git a/litellm/llms/tinyfish/search/transformation.py b/litellm/llms/tinyfish/search/transformation.py index c4949380e3a..b92f7ca1aff 100644 --- a/litellm/llms/tinyfish/search/transformation.py +++ b/litellm/llms/tinyfish/search/transformation.py @@ -67,7 +67,13 @@ class TinyfishSearchConfig(BaseSearchConfig): api_base: str | None = None, **kwargs: object, ) -> dict[str, str]: - resolved_key = api_key or get_secret_str("TINYFISH_API_KEY") + resolved_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("TINYFISH_API_KEY",), + base_env_var="TINYFISH_API_BASE", + default_api_base=self.TINYFISH_API_BASE, + ) if not resolved_key: raise ValueError( "TINYFISH_API_KEY is not set. Set `TINYFISH_API_KEY` environment variable." diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index d6441db7856..1fe9f15c9f0 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -32,6 +32,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): self._project = project self._location = location + def _include_function_response_id(self) -> bool: + return False + # ------------------------------------------------------------------ # URL # ------------------------------------------------------------------ diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index 3b84972e946..d2041009efb 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -242,6 +242,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: """ Map Cohere rerank params to Vertex AI format diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index d64450a1211..907e5b7e26b 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -39,6 +39,7 @@ class VoyageRerankConfig(BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: # Voyage AI uses 'top_k' instead of 'top_n' optional_params: Dict[str, Any] = {"query": query, "documents": documents} diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 202760f68a6..a34358a6be3 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -104,6 +104,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, ) -> Dict: """ Map Cohere rerank params to IBM watsonx.ai rerank params diff --git a/litellm/llms/you_com/search/transformation.py b/litellm/llms/you_com/search/transformation.py index 3c94b991735..0c7916e4c05 100644 --- a/litellm/llms/you_com/search/transformation.py +++ b/litellm/llms/you_com/search/transformation.py @@ -64,7 +64,13 @@ class YouComSearchConfig(BaseSearchConfig): endpoint with the `X-API-Key` header. Otherwise fall through to the keyless free tier; no auth header is required. """ - api_key = api_key or get_secret_str("YOUCOM_API_KEY") + api_key = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("YOUCOM_API_KEY",), + base_env_var="YOUCOM_API_BASE", + default_api_base=self.YOU_COM_API_BASE, + ) headers["Content-Type"] = "application/json" # Pin Accept-Encoding to identity: the keyless `api.you.com/v1/agents/search` # endpoint advertises gzip content-encoding but returns body bytes the diff --git a/litellm/main.py b/litellm/main.py index 63c5798e70a..c3d7ca28c49 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -81,11 +81,17 @@ from litellm.constants import ( from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + maybe_run_chat_completion_agentic_loop, +) from litellm.litellm_core_utils.audio_utils.utils import ( calculate_request_duration, get_audio_file_for_health_check, ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, +) from litellm.litellm_core_utils.get_litellm_params import OPTIONAL_KWARGS_KEYS from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -118,6 +124,10 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.realtime_api.main import _realtime_health_check from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.types.completion import ( + _CompletionDispatchContext, + _CompletionDispatchResult, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( CustomPricingLiteLLMParams, @@ -650,6 +660,39 @@ async def acompletion( response_object=response, model_response_object=litellm.ModelResponse(), ) + # Provider-agnostic dispatch point for the chat-completions agentic loop + # (code-interpreter interception, etc). Chat routing forks per provider + # before this (OpenAI goes through the OpenAI SDK in openai.py, others + # through the shared httpx handler), so a dispatch inside any single + # provider handler would miss the others. Here is where every fork + # reconverges, so the loop runs once for all providers. Responses needs + # no equivalent: every provider already funnels through one shared + # handler where the loop is dispatched. + if isinstance(response, litellm.ModelResponse): + looped = await maybe_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + optional_params={ + k: v + for k, v in completion_kwargs.items() + if v is not None + and k + not in ( + "model", + "messages", + "stream", + "acompletion", + "deployment_id", + ) + }, + kwargs=kwargs, + logging_obj=kwargs.get("litellm_logging_obj"), + custom_llm_provider=custom_llm_provider, + stream=bool(stream), + ) + if looped is not None: + response = looped if isinstance(response, CustomStreamWrapper): response.set_logging_event_loop( loop=loop @@ -1084,6 +1127,3825 @@ def _build_custom_pricing_entry( return entry +def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + _azure_detection_model = ctx._azure_detection_model + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + max_retries = ctx.max_retries + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + dynamic_params = False + if client is not None and ( + isinstance(client, openai.AzureOpenAI) + or isinstance(client, openai.AsyncAzureOpenAI) + ): + dynamic_params = _check_dynamic_azure_params( + azure_client_params={"api_version": api_version}, + azure_client=client, + ) + + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version + or litellm.api_version + or get_secret_str("AZURE_API_VERSION") + or litellm.AZURE_DEFAULT_API_VERSION + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret_str("AZURE_AD_TOKEN") + + azure_ad_token_provider = litellm_params.get("azure_ad_token_provider", None) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + if max_retries is not None: + optional_params["max_retries"] = max_retries + + if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIO1Config.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + response = azure_o1_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + custom_llm_provider=custom_llm_provider, + ) + else: + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client = ctx.client + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + api_type = get_secret_str("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + + if api_base is None: + raise ValueError( + "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." + ) + + api_version = ( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret_str("AZURE_AD_TOKEN") + + azure_ad_token_provider = litellm_params.get("azure_ad_token_provider", None) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=cast(str, api_version), + api_type=api_type, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response + + +def _complete_deepseek(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_azure_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + + azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model) + + # Check if this is an agents route - model format: azure_ai/agents/ + if azure_ai_route == "agents": + from litellm.llms.azure_ai.agents import AzureAIAgentsConfig + + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure AI Agents requests require an api_base. " + "Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + response = AzureAIAgentsConfig.completion( + model=model, + messages=messages, + api_base=api_base, + api_key=api_key, + model_response=model_response, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + acompletion=acompletion, + stream=stream, + headers=headers or litellm.headers, + ) + + # Check if this is a Claude model - route to Azure Anthropic handler + elif "claude" in model.lower(): + # Use Azure Anthropic handler for Claude models + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure Anthropic requests require an api_base. " + "Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + # Ensure the URL ends with /v1/messages for Anthropic + if api_base: + api_base = api_base.rstrip("/") + if not api_base.endswith("/v1/messages"): + if "/anthropic" in api_base: + parts = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + api_base = api_base + "/anthropic" + api_base = api_base + "/v1/messages" + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + else: + # Non-Claude models use standard Azure AI flow + api_base = AzureFoundryModelInfo.get_api_base(api_base) + # set API KEY + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## FOR COHERE + if "command-r" in model: # make sure tool call in messages are str + messages = stringify_json_tool_call_content(messages=messages) + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, # pass AsyncOpenAI, OpenAI client + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response + + +def _complete_text_completion_openai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + text_completion = ctx.text_completion + timeout = ctx.timeout + + openai.api_type = "openai" + + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + openai.api_version = None + # set API KEY + + api_key = ( + api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.OpenAITextCompletionConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: + openai.organization = litellm.organization + + ## COMPLETION CALL + _response = openai_text_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + acompletion=acompletion, + client=client, # pass AsyncOpenAI, OpenAI client + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False + ): + # convert to chat completion response + _response = ( + litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_fireworks_ai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_ragflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_xai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("GROQ_API_BASE") + or "https://api.groq.com/openai/v1" + ) + + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.groq_key + or get_secret("GROQ_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.GroqChatConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def _complete_bedrock_mantle( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") + api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") + headers = headers or litellm.headers + config = litellm.BedrockMantleChatConfig.get_config() + for k, v in config.items(): + if k not in optional_params: + optional_params[k] = v + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + +def _complete_a2a(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + ( + api_base, + api_key, + headers, + ) = litellm.A2AConfig.resolve_agent_config_from_registry( + model=model, + api_base=api_base, + api_key=api_key, + headers=headers, + optional_params=optional_params, + ) + + # Fall back to environment variables and defaults + api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") + + if api_base is None: + raise Exception( + "api_base is required for A2A provider. " + "Either provide api_base parameter, set A2A_API_BASE environment variable, " + "or register the agent in the proxy with model='a2a/'." + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + + +def _complete_gigachat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret("GIGACHAT_API_KEY") + or get_secret("GIGACHAT_CREDENTIALS") + ) + + headers = headers or litellm.headers or {} + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + headers = headers or litellm.headers + ## LOAD CONFIG - if set + config = litellm.GenAIHubOrchestrationConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + return sap_gen_ai_hub_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + shared_session=shared_session, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + api_key=api_key, + api_base=api_base, + stream=stream, + ) + + +def _complete_aiohttp_openai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + return base_llm_aiohttp_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + +def _complete_cometapi(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.cometapi_key + or get_secret_str("COMETAPI_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COMETAPI_API_BASE") + or "https://api.cometapi.com/v1" + ) + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + ## LOGGING + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def _complete_minimax(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("MINIMAX_API_BASE") + or "https://api.minimax.io/v1" + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def _complete_hosted_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def _complete_custom_openai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + metadata = ctx.metadata + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + organization = ctx.organization + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + organization + or litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + openai.organization = organization + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + # Add GitHub Copilot headers (same as /responses endpoint does) + if custom_llm_provider == "github_copilot": + from litellm.llms.github_copilot.authenticator import Authenticator + from litellm.llms.github_copilot.common_utils import ( + get_copilot_default_headers, + ) + + copilot_auth = Authenticator() + copilot_api_key = copilot_auth.get_api_key() + copilot_headers = get_copilot_default_headers(copilot_api_key) + if extra_headers: + copilot_headers.update(extra_headers) + extra_headers = copilot_headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + if ( + litellm.enable_preview_features and metadata is not None + ): # [PREVIEW] allow metadata to be passed to OPENAI + openai_metadata = get_requester_metadata(metadata) + if openai_metadata is not None: + optional_params["metadata"] = openai_metadata + + ## LOAD CONFIG - if set + config = litellm.OpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + use_base_llm_http_handler = get_secret_bool( + "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER" + ) + + try: + if use_base_llm_http_handler: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + else: + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, + custom_llm_provider=custom_llm_provider, + shared_session=shared_session, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_mistral(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") + api_base = ( + api_base + or litellm.api_base + or get_secret("MISTRAL_API_BASE") + or "https://api.mistral.ai/v1" + ) + + return base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + + +def _complete_replicate(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + replicate_key = ( + api_key + or litellm.replicate_key + or litellm.api_key + or get_secret("REPLICATE_API_KEY") + or get_secret("REPLICATE_API_TOKEN") + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("REPLICATE_API_BASE") + or "https://api.replicate.com/v1" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = replicate_chat_completion( # type: ignore + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), # for calculating input/output tokens + api_key=replicate_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + acompletion=acompletion, + headers=headers, + ) + + if optional_params.get("stream", False) is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=replicate_key, + original_response=model_response, + ) + + return model_response + + +def _complete_anthropic_text( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.anthropic_key + or litellm.api_key + or os.environ.get("ANTHROPIC_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + api_base = cast( + Optional[str], + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/complete", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if ( + api_base is not None + and not disable_url_suffix + and not api_base.endswith("/v1/complete") + ): + api_base += "/v1/complete" + elif disable_url_suffix: + verbose_logger.debug( + "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix" + ) + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="anthropic_text", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def _complete_anthropic(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.anthropic_key + or litellm.api_key + or os.environ.get("ANTHROPIC_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # call /messages + # default route for all anthropic models + api_base = cast( + Optional[str], + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/messages", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if ( + api_base is not None + and not disable_url_suffix + and not api_base.endswith("/v1/messages") + ): + api_base += "/v1/messages" + elif disable_url_suffix: + verbose_logger.debug( + "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix" + ) + + response = anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + return response + + +def _complete_nlp_cloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + nlp_cloud_key = ( + api_key + or litellm.nlp_cloud_key + or get_secret("NLP_CLOUD_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("NLP_CLOUD_API_BASE") + or "https://api.nlpcloud.io/v1/gpu/" + ) + + response = nlp_cloud_chat_completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=nlp_cloud_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="nlp_cloud", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_aleph_alpha(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + api_base = ctx.api_base + api_key = ctx.api_key + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + aleph_alpha_key = ( + api_key + or litellm.aleph_alpha_key + or get_secret("ALEPH_ALPHA_API_KEY") + or get_secret("ALEPHALPHA_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("ALEPH_ALPHA_API_BASE") + or "https://api.aleph-alpha.com/complete" + ) + + model_response = aleph_alpha.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + default_max_tokens_to_sample=litellm.max_tokens, + api_key=aleph_alpha_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="aleph_alpha", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + cohere_key = ( + api_key + or litellm.cohere_key + or get_secret_str("COHERE_API_KEY") + or get_secret_str("CO_API_KEY") + or litellm.api_key + ) + + cohere_route = CohereModelInfo.get_cohere_route(model) + verbose_logger.debug(f"Cohere route: {cohere_route}") + # Set API base based on route + if cohere_route == "v2": + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COHERE_API_BASE") + or "https://api.cohere.com/v2/chat" + ) + # Remove v2/ prefix from model name for the actual API call + if "v2/" in model: + model = model.replace("v2/", "") + else: + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COHERE_API_BASE") + or "https://api.cohere.ai/v1/chat" + ) + + headers = headers or litellm.headers or {} + if headers is None: + headers = {} + + if extra_headers is not None: + headers.update(extra_headers) + + verbose_logger.debug(f"Model: {model}, API Base: {api_base}") + verbose_logger.debug(f"Provider Config: {provider_config}") + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cohere_chat", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=cohere_key, + provider_config=provider_config, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def _complete_maritalk(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + maritalk_key = ( + api_key + or litellm.maritalk_key + or get_secret("MARITALK_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("MARITALK_API_BASE") + or "https://chat.maritaca.ai/api" + ) + + return openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=maritalk_key, + logging_obj=logging, + custom_llm_provider="maritalk", + custom_prompt_dict=custom_prompt_dict, + ) + + +def _complete_amazon_nova(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + api_base = ctx.api_base + api_key = ctx.api_key + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.amazon_nova_api_key + or get_secret_str("AMAZON_NOVA_API_KEY") + or litellm.api_key + ) + api_base = ( + api_base + or litellm.api_base + or get_secret_str("AMAZON_NOVA_API_BASE") + or "https://api.nova.amazon.com/v1" + ) + return openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + ) + + +def _complete_huggingface(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + huggingface_key = ( + api_key + or litellm.huggingface_key + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_API_KEY") + or litellm.api_key + ) + hf_headers = headers or litellm.headers + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=hf_headers, + model_response=model_response, + api_key=huggingface_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + +def _complete_oci(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + +def _complete_compactifai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + stream = ctx.stream + timeout = ctx.timeout + + api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key + + api_base = api_base or "https://api.compactif.ai/v1" + + ## COMPLETION CALL + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + +def _complete_oobabooga(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + api_base = ctx.api_base + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + model_response = oobabooga.completion( + model=model, + messages=messages, + model_response=model_response, + api_base=api_base, # type: ignore + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="oobabooga", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_databricks(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for databricks we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("DATABRICKS_API_BASE") + ) + + # set API KEY + api_key = ( + api_key + or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there + or litellm.databricks_key + or get_secret("DATABRICKS_API_KEY") + ) + + headers = headers or litellm.headers + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="databricks", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response + + +def _complete_datarobot(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + stream = ctx.stream + timeout = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + +def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENROUTER_API_BASE") + or "https://openrouter.ai/api/v1" + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret_str("OPENROUTER_API_KEY") + or get_secret_str("OR_API_KEY") + ) + + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + + _headers = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) + + headers = openrouter_headers + + ## Load Config + config = litellm.OpenrouterConfig.get_config() + for k, v in config.items(): + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="openrouter", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call( + input=messages, api_key=openai.api_key, original_response=response + ) + + return response + + +def _complete_vercel_ai_gateway( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") + or "https://ai-gateway.vercel.sh/v1" + ) + + api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") + + vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" + vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM" + + vercel_headers = { + "http-referer": vercel_site_url, + "x-title": vercel_app_name, + } + + _headers = headers or litellm.headers + if _headers: + vercel_headers.update(_headers) + + headers = vercel_headers + + ## Load Config + config = litellm.VercelAIGatewayConfig.get_config() + for k, v in config.items(): + if k == "extra_body": + # we use openai 'extra_body' to pass vercel specific params - providerOptions + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="vercel_ai_gateway", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call( + input=messages, api_key=openai.api_key, original_response=response + ) + + return response + + +def _complete_vertex_ai_beta( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + gemini_api_key = ( + api_key + or get_api_key_from_env() + or get_secret("PALM_API_KEY") # older palm api key should also work + or litellm.api_key + ) + + api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") + new_params = safe_deep_copy(optional_params or {}) + return vertex_chat_completion.completion( # type: ignore + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=gemini_api_key, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, # type: ignore + client=client, + api_base=api_base, + extra_headers=headers, + ) + + +def _complete_vertex_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") + + new_params = safe_deep_copy(optional_params or {}) + model_route = get_vertex_ai_model_route(model=model, litellm_params=litellm_params) + + if model_route == VertexAIModelRoute.PARTNER_MODELS: + model_response = vertex_partner_models_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.GEMINI: + model_response = vertex_chat_completion.completion( # type: ignore + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=None, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, # type: ignore + client=client, + api_base=api_base, + extra_headers=headers, + ) + elif model_route == VertexAIModelRoute.GEMMA: + # Vertex Gemma Models with custom prediction endpoint + model_response = vertex_gemma_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.MODEL_GARDEN: + # Vertex Model Garden - OpenAI compatible models + model_response = vertex_model_garden_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.AGENT_ENGINE: + # Vertex AI Agent Engine (Reasoning Engines) + from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + ) + + vertex_agent_engine_config = VertexAgentEngineConfig() + + # Update litellm_params with vertex credentials + litellm_params["vertex_project"] = vertex_ai_project + litellm_params["vertex_location"] = vertex_ai_location + litellm_params["vertex_credentials"] = vertex_credentials + + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + model_response=model_response, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + encoding=_get_encoding(), + api_key=None, + api_base=api_base, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, + custom_llm_provider="vertex_ai", + provider_config=vertex_agent_engine_config, + headers=headers or {}, + ) + else: # VertexAIModelRoute.NON_GEMINI + model_response = vertex_ai_non_gemini.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] is True + and acompletion is False + ): + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vertex_ai", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_predibase(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + tenant_id = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + if tenant_id is None: + raise ValueError( + "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." + ) + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.predibase_key + or get_secret("PREDIBASE_API_KEY") + ) + + _model_response = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + timeout=timeout, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] is True + and acompletion is False + ): + return _model_response + return _model_response + + +def _complete_text_completion_codestral( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or "https://codestral.mistral.ai/v1/fim/completions" + ) + + api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") + + text_completion_model_response = litellm.TextCompletionResponse(stream=stream) + + _model_response = codestral_text_completions.completion( # type: ignore + model=model, + messages=messages, + model_response=text_completion_model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + timeout=timeout, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] is True + and acompletion is False + ): + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_text_completion_inception( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + text_completion = ctx.text_completion + timeout = ctx.timeout + + passed_api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + ) + api_base = ( + passed_api_base + or get_secret_str("INCEPTION_API_BASE") + or "https://api.inceptionlabs.ai/v1" + ) + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = api_base.rstrip("/") + if not api_base.endswith("/fim"): + api_base += "/fim" + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = ( + api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + ) + + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, # type: ignore[arg-type] + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False + ): + _response = ( + litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + ) + + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_sagemaker_chat( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def _complete_sagemaker(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + custom_prompt_dict = ctx.custom_prompt_dict + hf_model_name = ctx.hf_model_name + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + return sagemaker_llm.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + custom_prompt_dict=custom_prompt_dict, + hf_model_name=hf_model_name, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + acompletion=acompletion, + ) + + +def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + if "aws_bedrock_client" in optional_params: + verbose_logger.warning( + "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." + ) + # Extract credentials for legacy boto3 client and pass thru to httpx + aws_bedrock_client = optional_params.pop("aws_bedrock_client") + creds = aws_bedrock_client._get_credentials().get_frozen_credentials() + + if creds.access_key: + optional_params["aws_access_key_id"] = creds.access_key + if creds.secret_key: + optional_params["aws_secret_access_key"] = creds.secret_key + if creds.token: + optional_params["aws_session_token"] = creds.token + if ( + "aws_region_name" not in optional_params + or optional_params["aws_region_name"] is None + ): + optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name + + bedrock_route = BedrockModelInfo.get_bedrock_route(model) + if bedrock_route == "claude_platform": + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders.BEDROCK, + ) + model = BedrockModelInfo.get_claude_platform_model(model) + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + elif bedrock_route == "converse": + model = model.replace("converse/", "") + response = bedrock_converse_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + extra_headers=headers, # Use merged headers instead of original extra_headers + timeout=timeout, + acompletion=acompletion, + client=client, + api_base=api_base, + api_key=api_key, + ) + elif bedrock_route == "converse_like": + model = model.replace("converse_like/", "") + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + else: + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + return response + + +def _complete_watsonx(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + return watsonx_chat_completion.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=_get_encoding(), + custom_llm_provider="watsonx", + ) + + +def _complete_watsonx_text( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or optional_params.pop("apikey", None) + or get_secret_str("WATSONX_APIKEY") + or get_secret_str("WATSONX_API_KEY") + or get_secret_str("WX_API_KEY") + ) + + api_base = ( + api_base + or optional_params.pop( + "url", + optional_params.pop("api_base", optional_params.pop("base_url", None)), + ) + or get_secret_str("WATSONX_API_BASE") + or get_secret_str("WATSONX_URL") + or get_secret_str("WX_URL") + or get_secret_str("WML_URL") + ) + + wx_credentials = optional_params.pop( + "wx_credentials", + optional_params.pop( + "watsonx_credentials", None + ), # follow {provider}_credentials, same as vertex ai + ) + + token: Optional[str] = None + if wx_credentials is not None: + api_base = wx_credentials.get("url", api_base) + api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key)) + token = wx_credentials.get( + "token", + wx_credentials.get( + "watsonx_token", None + ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' + ) + + if token is not None: + optional_params["token"] = token + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="watsonx_text", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def _complete_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = vllm_handler.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vllm", + logging_obj=logging, + ) + + ## RESPONSE OBJECT + return model_response + + +def _complete_ollama(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + litellm.api_base + or api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def _complete_ollama_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + litellm.api_base + or api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) + + api_key = ( + api_key + or litellm.ollama_key + or os.environ.get("OLLAMA_API_KEY") + or litellm.api_key + ) + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama_chat", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def _complete_triton(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = litellm.api_base or api_base + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + ) + + +def _complete_cloudflare(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.cloudflare_api_key + or litellm.api_key + or get_secret("CLOUDFLARE_API_KEY") + ) + api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE") + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cloudflare", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def _complete_petals(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + api_base = ctx.api_base + client = ctx.client + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + + api_base = api_base or litellm.api_base + + stream = optional_params.pop("stream", False) + model_response = petals_handler.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + client=client, + ) + if stream is True: ## [BETA] + # Fake streaming for petals + resp_string = model_response["choices"][0]["message"]["content"] + return CustomStreamWrapper( + resp_string, + model, + custom_llm_provider="petals", + logging_obj=logging, + ) + return model_response + + +def _complete_snowflake(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + client = ( + HTTPHandler(timeout=timeout) if stream is False else None + ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_gradient_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = litellm.api_base or api_base + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="gradient_ai", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + ) + + +def _complete_bytez(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.bytez_key + or get_secret_str("BYTEZ_API_KEY") + or litellm.api_key + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=bytez_transformation, + ) + + pass + + return response + + +def _complete_lemonade(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.lemonade_key + or get_secret_str("LEMONADE_API_KEY") + or litellm.api_key + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=lemonade_transformation, + ) + + pass + + return response + + +def _complete_ovhcloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.ovhcloud_key + or get_secret_str("OVHCLOUD_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OVHCLOUD_API_BASE") + or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=ovhcloud_transformation, + ) + + pass + + return response + + +def _complete_custom(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + api_base = ctx.api_base + headers = ctx.headers + kwargs = ctx.kwargs + max_tokens = ctx.max_tokens + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + temperature = ctx.temperature + top_p = ctx.top_p + + url = litellm.api_base or api_base or "" + if url is None or url == "": + raise ValueError( + "api_base not set. Set api_base or litellm.api_base for custom endpoints" + ) + + """ + assume input to custom LLM api bases follow this format: + resp = litellm.module_level_client.post( + api_base, + json={ + 'model': 'meta-llama/Llama-2-13b-hf', # model name + 'params': { + 'prompt': ["The capital of France is P"], + 'max_tokens': 32, + 'temperature': 0.7, + 'top_p': 1.0, + 'top_k': 40, + } + } + ) + + """ + prompt = " ".join([message["content"] for message in messages]) # type: ignore + resp = litellm.module_level_client.post( + url, + headers=headers, + json={ + "model": model, + "params": { + "prompt": [prompt], + "max_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "top_k": kwargs.get("top_k"), + }, + **kwargs.get("extra_body", {}), + }, + ) + response_json = resp.json() + """ + assume all responses from custom api_bases of this format: + { + 'data': [ + { + 'prompt': 'The capital of France is P', + 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], + 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], + 'message': 'ok' + } + ] + } + """ + string_response = response_json["data"][0]["output"][0] + ## RESPONSE OBJECT + model_response.choices[0].message.content = string_response # type: ignore + model_response.created = int(time.time()) + model_response.model = model + return model_response + + +def _complete_custom_providers( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + custom_handler: Optional[CustomLLM] = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider( + model=model, custom_llm_provider=custom_llm_provider + ) + + ## ROUTE LLM CALL ## + handler_fn = custom_chat_llm_router( + async_fn=acompletion, stream=stream, custom_llm=custom_handler + ) + + headers = headers or litellm.headers or {} + + ## CALL FUNCTION + response = handler_fn( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=_get_encoding(), + ) + if stream is True: + return CustomStreamWrapper( + completion_stream=response, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def _complete_langgraph(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + from litellm.llms.langgraph.chat.transformation import LangGraphConfig + + ( + api_base, + api_key, + ) = LangGraphConfig()._get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + +def _complete_langflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + from litellm.llms.langflow.chat.transformation import LangFlowConfig + + ( + api_base, + api_key, + ) = LangFlowConfig()._get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + @tracer.wrap() @client def completion( # type: ignore @@ -1215,9 +5077,7 @@ def completion( # type: ignore if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway( tools=tools_for_mcp ): - # Return coroutine - acompletion will await it - # completion() can return a coroutine when MCP tools are present, which acompletion() awaits - return acompletion_with_mcp( # type: ignore[return-value] + return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it model=model, messages=messages, functions=functions, @@ -1389,12 +5249,16 @@ def completion( # type: ignore logging: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, litellm_logging_obj) fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: - return completion_with_fallbacks(**args) + return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime + **args + ) if model_list is not None: deployments = [ m["litellm_params"] for m in model_list if m["model_name"] == model ] - return litellm.batch_completion_models(deployments=deployments, **args) + return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type + deployments=deployments, **args + ) if litellm.model_alias_map and model in litellm.model_alias_map: model = litellm.model_alias_map[ model @@ -1454,7 +5318,7 @@ def completion( # type: ignore timeout, kwargs, custom_llm_provider, - global_timeout=getattr(litellm, "request_timeout", None), + global_timeout=get_configured_request_timeout(), supports_httpx_timeout=supports_httpx_timeout, ) @@ -1716,7 +5580,7 @@ def completion( # type: ignore else: optional_params["reasoning_effort"] = {"summary": rs_val} - return responses_api_bridge.completion( + return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller model=model, messages=messages, headers=headers, @@ -1746,375 +5610,52 @@ def completion( # type: ignore optional_params ) + _dispatch_ctx = _CompletionDispatchContext( + _azure_detection_model=_azure_detection_model, + acompletion=acompletion, + api_base=api_base, + api_key=api_key, + api_version=api_version, + client=client, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + extra_headers=extra_headers, + headers=headers, + hf_model_name=hf_model_name, + kwargs=kwargs, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging=logging, + max_retries=max_retries, + max_tokens=max_tokens, + messages=messages, + metadata=metadata, + model=model, + model_response=model_response, + optional_params=optional_params, + organization=organization, + provider_config=provider_config, + shared_session=shared_session, + stream=stream, + temperature=temperature, + text_completion=text_completion, + timeout=timeout, + top_p=top_p, + ) if custom_llm_provider == "azure": # azure configs ## check dynamic params ## - dynamic_params = False - if client is not None and ( - isinstance(client, openai.AzureOpenAI) - or isinstance(client, openai.AsyncAzureOpenAI) - ): - dynamic_params = _check_dynamic_azure_params( - azure_client_params={"api_version": api_version}, - azure_client=client, - ) - - api_type = get_secret("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") - - api_version = ( - api_version - or litellm.api_version - or get_secret_str("AZURE_API_VERSION") - or litellm.AZURE_DEFAULT_API_VERSION - ) - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) - - azure_ad_token = optional_params.get("extra_body", {}).pop( - "azure_ad_token", None - ) or get_secret_str("AZURE_AD_TOKEN") - - azure_ad_token_provider = litellm_params.get( - "azure_ad_token_provider", None - ) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - if max_retries is not None: - optional_params["max_retries"] = max_retries - - if litellm.AzureOpenAIO1Config().is_o_series_model( - model=_azure_detection_model - ): - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIO1Config.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = azure_o1_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, # type: ignore - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - custom_llm_provider=custom_llm_provider, - ) - else: - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response = azure_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - api_type=api_type, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, # type: ignore - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) + response = _complete_azure(_dispatch_ctx) elif custom_llm_provider == "azure_text": # azure configs - api_type = get_secret_str("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - - if api_base is None: - raise ValueError( - "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." - ) - - api_version = ( - api_version - or litellm.api_version - or get_secret_str("AZURE_API_VERSION") - ) - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) - - azure_ad_token = optional_params.get("extra_body", {}).pop( - "azure_ad_token", None - ) or get_secret_str("AZURE_AD_TOKEN") - - azure_ad_token_provider = litellm_params.get( - "azure_ad_token_provider", None - ) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response = azure_text_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=cast(str, api_version), - api_type=api_type, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) + response = _complete_azure_text(_dispatch_ctx) elif custom_llm_provider == "deepseek": ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_deepseek(_dispatch_ctx) elif custom_llm_provider == "azure_ai": - from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo - - azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model) - - # Check if this is an agents route - model format: azure_ai/agents/ - if azure_ai_route == "agents": - from litellm.llms.azure_ai.agents import AzureAIAgentsConfig - - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure AI Agents requests require an api_base. " - "Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - response = AzureAIAgentsConfig.completion( - model=model, - messages=messages, - api_base=api_base, - api_key=api_key, - model_response=model_response, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - acompletion=acompletion, - stream=stream, - headers=headers or litellm.headers, - ) - - # Check if this is a Claude model - route to Azure Anthropic handler - elif "claude" in model.lower(): - # Use Azure Anthropic handler for Claude models - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure Anthropic requests require an api_base. " - "Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - # Ensure the URL ends with /v1/messages for Anthropic - if api_base: - api_base = api_base.rstrip("/") - if not api_base.endswith("/v1/messages"): - if "/anthropic" in api_base: - parts = api_base.split("/anthropic", 1) - api_base = parts[0] + "/anthropic" - else: - api_base = api_base + "/anthropic" - api_base = api_base + "/v1/messages" - - response = azure_anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - response = response - else: - # Non-Claude models use standard Azure AI flow - api_base = AzureFoundryModelInfo.get_api_base(api_base) - # set API KEY - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - ## FOR COHERE - if "command-r" in model: # make sure tool call in messages are str - messages = stringify_json_tool_call_content(messages=messages) - - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, # pass AsyncOpenAI, OpenAI client - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) + response = _complete_azure_ai(_dispatch_ctx) elif ( custom_llm_provider == "text-completion-openai" or "ft:babbage-002" in model @@ -2123,535 +5664,42 @@ def completion( # type: ignore in litellm.openai_text_completion_compatible_providers and kwargs.get("text_completion") is True ): - openai.api_type = "openai" - - api_base = ( - api_base - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - - openai.api_version = None - # set API KEY - - api_key = ( - api_key - or litellm.api_key - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - - ## LOAD CONFIG - if set - config = litellm.OpenAITextCompletionConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - if litellm.organization: - openai.organization = litellm.organization - - if ( - len(messages) > 0 - and "content" in messages[0] - and isinstance(messages[0]["content"], list) - ): - # text-davinci-003 can accept a string or array, if it's an array, assume the array is set in messages[0]['content'] - # https://platform.openai.com/docs/api-reference/completions/create - prompt = messages[0]["content"] - else: - prompt = " ".join([message["content"] for message in messages]) # type: ignore - - ## COMPLETION CALL - _response = openai_text_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - acompletion=acompletion, - client=client, # pass AsyncOpenAI, OpenAI client - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - ) - - if ( - optional_params.get("stream", False) is False - and acompletion is False - and text_completion is False - ): - # convert to chat completion response - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - response = _response + response = _complete_text_completion_openai(_dispatch_ctx) elif custom_llm_provider == "fireworks_ai": ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_fireworks_ai(_dispatch_ctx) elif custom_llm_provider == "heroku": - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_heroku(_dispatch_ctx) elif custom_llm_provider == "ragflow": ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_ragflow(_dispatch_ctx) elif custom_llm_provider == "xai": ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_xai(_dispatch_ctx) elif custom_llm_provider == "groq": - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("GROQ_API_BASE") - or "https://api.groq.com/openai/v1" - ) - - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.groq_key - or get_secret("GROQ_API_KEY") - ) - - headers = headers or litellm.headers - - ## LOAD CONFIG - if set - config = litellm.GroqChatConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) + response = _complete_groq(_dispatch_ctx) elif custom_llm_provider == "bedrock_mantle": - api_base = ( - api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") - ) - api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") - headers = headers or litellm.headers - config = litellm.BedrockMantleChatConfig.get_config() - for k, v in config.items(): - if k not in optional_params: - optional_params[k] = v - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) + response = _complete_bedrock_mantle(_dispatch_ctx) elif custom_llm_provider == "a2a": # A2A (Agent-to-Agent) Protocol # Resolve agent configuration from registry if model format is "a2a/" - ( - api_base, - api_key, - headers, - ) = litellm.A2AConfig.resolve_agent_config_from_registry( - model=model, - api_base=api_base, - api_key=api_key, - headers=headers, - optional_params=optional_params, - ) - - # Fall back to environment variables and defaults - api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") - - if api_base is None: - raise Exception( - "api_base is required for A2A provider. " - "Either provide api_base parameter, set A2A_API_BASE environment variable, " - "or register the agent in the proxy with model='a2a/'." - ) - - headers = headers or litellm.headers - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) + response = _complete_a2a(_dispatch_ctx) elif custom_llm_provider == "gigachat": # GigaChat - Sber AI's LLM (Russia) - api_key = ( - api_key - or litellm.api_key - or litellm.gigachat_key - or get_secret("GIGACHAT_API_KEY") - or get_secret("GIGACHAT_CREDENTIALS") - ) - - headers = headers or litellm.headers or {} - - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_gigachat(_dispatch_ctx) elif custom_llm_provider == "sap": - headers = headers or litellm.headers - ## LOAD CONFIG - if set - config = litellm.GenAIHubOrchestrationConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = sap_gen_ai_hub_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - shared_session=shared_session, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - api_key=api_key, - api_base=api_base, - stream=stream, - ) + response = _complete_sap(_dispatch_ctx) elif custom_llm_provider == "aiohttp_openai": # NEW aiohttp provider for 10-100x higher RPS - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - response = base_llm_aiohttp_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) + response = _complete_aiohttp_openai(_dispatch_ctx) elif custom_llm_provider == "cometapi": - api_key = ( - api_key - or litellm.cometapi_key - or get_secret_str("COMETAPI_KEY") - or litellm.api_key - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COMETAPI_API_BASE") - or "https://api.cometapi.com/v1" - ) - - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - ## LOGGING - logging.post_call( - input=messages, api_key=api_key, original_response=response - ) + response = _complete_cometapi(_dispatch_ctx) elif custom_llm_provider == "minimax": - api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("MINIMAX_API_BASE") - or "https://api.minimax.io/v1" - ) - - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call( - input=messages, api_key=api_key, original_response=response - ) + response = _complete_minimax(_dispatch_ctx) elif custom_llm_provider == "hosted_vllm": - api_base = ( - api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") - ) - - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call( - input=messages, api_key=api_key, original_response=response - ) + response = _complete_hosted_vllm(_dispatch_ctx) elif ( model in litellm.open_ai_chat_completion_models or custom_llm_provider == "custom_openai" @@ -2676,205 +5724,17 @@ def completion( # type: ignore ): # allow user to make an openai call with a custom base # note: if a user sets a custom base - we should ensure this works # allow for the setting of dynamic and stateful api-bases - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - organization = ( - organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 - ) - openai.organization = organization - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - - # Add GitHub Copilot headers (same as /responses endpoint does) - if custom_llm_provider == "github_copilot": - from litellm.llms.github_copilot.authenticator import Authenticator - from litellm.llms.github_copilot.common_utils import ( - get_copilot_default_headers, - ) - - copilot_auth = Authenticator() - copilot_api_key = copilot_auth.get_api_key() - copilot_headers = get_copilot_default_headers(copilot_api_key) - if extra_headers: - copilot_headers.update(extra_headers) - extra_headers = copilot_headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - if ( - litellm.enable_preview_features and metadata is not None - ): # [PREVIEW] allow metadata to be passed to OPENAI - openai_metadata = get_requester_metadata(metadata) - if openai_metadata is not None: - optional_params["metadata"] = openai_metadata - - ## LOAD CONFIG - if set - config = litellm.OpenAIConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - use_base_llm_http_handler = get_secret_bool( - "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER" - ) - - try: - if use_base_llm_http_handler: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - else: - response = openai_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - organization=organization, - custom_llm_provider=custom_llm_provider, - shared_session=shared_session, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) + response = _complete_custom_openai(_dispatch_ctx) elif custom_llm_provider == "mistral": - api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") - api_base = ( - api_base - or litellm.api_base - or get_secret("MISTRAL_API_BASE") - or "https://api.mistral.ai/v1" - ) - - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) + response = _complete_mistral(_dispatch_ctx) elif ( "replicate" in model or custom_llm_provider == "replicate" or model in litellm.replicate_models ): # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") - replicate_key = ( - api_key - or litellm.replicate_key - or litellm.api_key - or get_secret("REPLICATE_API_KEY") - or get_secret("REPLICATE_API_TOKEN") - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret("REPLICATE_API_BASE") - or "https://api.replicate.com/v1" - ) - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - - model_response = replicate_chat_completion( # type: ignore - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=replicate_key, - logging_obj=logging, - custom_prompt_dict=custom_prompt_dict, - acompletion=acompletion, - headers=headers, - ) - - if optional_params.get("stream", False) is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=replicate_key, - original_response=model_response, - ) - - response = model_response + response = _complete_replicate(_dispatch_ctx) elif ( "clarifai" in model or custom_llm_provider == "clarifai" @@ -2882,614 +5742,36 @@ def completion( # type: ignore ): pass # Deprecated - handled in the openai compatible provider section above elif custom_llm_provider == "anthropic_text": - api_key = ( - api_key - or litellm.anthropic_key - or litellm.api_key - or os.environ.get("ANTHROPIC_API_KEY") - ) - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - api_base = ( - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/complete" - ) - - # Check if we should disable automatic URL suffix appending - disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if ( - api_base is not None - and not disable_url_suffix - and not api_base.endswith("/v1/complete") - ): - api_base += "/v1/complete" - elif disable_url_suffix: - verbose_logger.debug( - "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix" - ) - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="anthropic_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) + response = _complete_anthropic_text(_dispatch_ctx) elif custom_llm_provider == "anthropic": - api_key = ( - api_key - or litellm.anthropic_key - or litellm.api_key - or os.environ.get("ANTHROPIC_API_KEY") - ) - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - # call /messages - # default route for all anthropic models - api_base = ( - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/messages" - ) - - # Check if we should disable automatic URL suffix appending - disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if ( - api_base is not None - and not disable_url_suffix - and not api_base.endswith("/v1/messages") - ): - api_base += "/v1/messages" - elif disable_url_suffix: - verbose_logger.debug( - "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix" - ) - - response = anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - response = response + response = _complete_anthropic(_dispatch_ctx) elif custom_llm_provider == "nlp_cloud": - nlp_cloud_key = ( - api_key - or litellm.nlp_cloud_key - or get_secret("NLP_CLOUD_API_KEY") - or litellm.api_key - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret("NLP_CLOUD_API_BASE") - or "https://api.nlpcloud.io/v1/gpu/" - ) - - response = nlp_cloud_chat_completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=nlp_cloud_key, - logging_obj=logging, - ) - - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - response, - model, - custom_llm_provider="nlp_cloud", - logging_obj=logging, - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - - response = response + response = _complete_nlp_cloud(_dispatch_ctx) elif custom_llm_provider == "aleph_alpha": - aleph_alpha_key = ( - api_key - or litellm.aleph_alpha_key - or get_secret("ALEPH_ALPHA_API_KEY") - or get_secret("ALEPHALPHA_API_KEY") - or litellm.api_key - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret("ALEPH_ALPHA_API_BASE") - or "https://api.aleph-alpha.com/complete" - ) - - model_response = aleph_alpha.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - default_max_tokens_to_sample=litellm.max_tokens, - api_key=aleph_alpha_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="aleph_alpha", - logging_obj=logging, - ) - return response - response = model_response + response = _complete_aleph_alpha(_dispatch_ctx) elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": - cohere_key = ( - api_key - or litellm.cohere_key - or get_secret_str("COHERE_API_KEY") - or get_secret_str("CO_API_KEY") - or litellm.api_key - ) - - cohere_route = CohereModelInfo.get_cohere_route(model) - verbose_logger.debug(f"Cohere route: {cohere_route}") - # Set API base based on route - if cohere_route == "v2": - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COHERE_API_BASE") - or "https://api.cohere.com/v2/chat" - ) - # Remove v2/ prefix from model name for the actual API call - if "v2/" in model: - model = model.replace("v2/", "") - else: - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COHERE_API_BASE") - or "https://api.cohere.ai/v1/chat" - ) - - headers = headers or litellm.headers or {} - if headers is None: - headers = {} - - if extra_headers is not None: - headers.update(extra_headers) - - verbose_logger.debug(f"Model: {model}, API Base: {api_base}") - verbose_logger.debug(f"Provider Config: {provider_config}") - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cohere_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=cohere_key, - provider_config=provider_config, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) + response = _complete_cohere_chat(_dispatch_ctx) elif custom_llm_provider == "maritalk": - maritalk_key = ( - api_key - or litellm.maritalk_key - or get_secret("MARITALK_API_KEY") - or litellm.api_key - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret("MARITALK_API_BASE") - or "https://chat.maritaca.ai/api" - ) - - model_response = openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=maritalk_key, - logging_obj=logging, - custom_llm_provider="maritalk", - custom_prompt_dict=custom_prompt_dict, - ) - - response = model_response + response = _complete_maritalk(_dispatch_ctx) elif custom_llm_provider == "amazon_nova": - api_key = ( - api_key - or litellm.amazon_nova_api_key - or get_secret_str("AMAZON_NOVA_API_KEY") - or litellm.api_key - ) - api_base = ( - api_base - or litellm.api_base - or get_secret_str("AMAZON_NOVA_API_BASE") - or "https://api.nova.amazon.com/v1" - ) - response = openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - custom_prompt_dict=custom_prompt_dict, - ) + response = _complete_amazon_nova(_dispatch_ctx) elif custom_llm_provider == "huggingface": - huggingface_key = ( - api_key - or litellm.huggingface_key - or os.environ.get("HF_TOKEN") - or os.environ.get("HUGGINGFACE_API_KEY") - or litellm.api_key - ) - hf_headers = headers or litellm.headers - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=hf_headers, - model_response=model_response, - api_key=huggingface_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) + response = _complete_huggingface(_dispatch_ctx) elif custom_llm_provider == "oci": - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) + response = _complete_oci(_dispatch_ctx) elif custom_llm_provider == "compactifai": - api_key = ( - api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key - ) - - api_base = api_base or "https://api.compactif.ai/v1" - - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) + response = _complete_compactifai(_dispatch_ctx) elif custom_llm_provider == "oobabooga": - custom_llm_provider = "oobabooga" - model_response = oobabooga.completion( - model=model, - messages=messages, - model_response=model_response, - api_base=api_base, # type: ignore - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - api_key=None, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="oobabooga", - logging_obj=logging, - ) - return response - response = model_response + response = _complete_oobabooga(_dispatch_ctx) elif custom_llm_provider == "databricks": - api_base = ( - api_base # for databricks we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or os.getenv("DATABRICKS_API_BASE") - ) - - # set API KEY - api_key = ( - api_key - or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there - or litellm.databricks_key - or get_secret("DATABRICKS_API_KEY") - ) - - headers = headers or litellm.headers - - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="databricks", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) + response = _complete_databricks(_dispatch_ctx) elif custom_llm_provider == "datarobot": - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) + response = _complete_datarobot(_dispatch_ctx) elif custom_llm_provider == "openrouter": - api_base = ( - api_base - or litellm.api_base - or get_secret_str("OPENROUTER_API_BASE") - or "https://openrouter.ai/api/v1" - ) - - api_key = ( - api_key - or litellm.api_key - or litellm.openrouter_key - or get_secret_str("OPENROUTER_API_KEY") - or get_secret_str("OR_API_KEY") - ) - - openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" - - openrouter_headers = { - "HTTP-Referer": openrouter_site_url, - "X-Title": openrouter_app_name, - } - - _headers = headers or litellm.headers - if _headers: - openrouter_headers.update(_headers) - - headers = openrouter_headers - - ## Load Config - config = litellm.OpenrouterConfig.get_config() - for k, v in config.items(): - if k == "extra_body": - # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - - data = {"model": model, "messages": messages, **optional_params} - - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="openrouter", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call( - input=messages, api_key=openai.api_key, original_response=response - ) + response = _complete_openrouter(_dispatch_ctx) elif custom_llm_provider == "vercel_ai_gateway": - api_base = ( - api_base - or litellm.api_base - or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") - or "https://ai-gateway.vercel.sh/v1" - ) - - api_key = ( - api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") - ) - - vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" - vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM" - - vercel_headers = { - "http-referer": vercel_site_url, - "x-title": vercel_app_name, - } - - _headers = headers or litellm.headers - if _headers: - vercel_headers.update(_headers) - - headers = vercel_headers - - ## Load Config - config = litellm.VercelAIGatewayConfig.get_config() - for k, v in config.items(): - if k == "extra_body": - # we use openai 'extra_body' to pass vercel specific params - providerOptions - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - - data = {"model": model, "messages": messages, **optional_params} - - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="vercel_ai_gateway", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call( - input=messages, api_key=openai.api_key, original_response=response - ) + response = _complete_vercel_ai_gateway(_dispatch_ctx) elif ( custom_llm_provider == "together_ai" or ("togethercomputer" in model) @@ -3504,1114 +5786,75 @@ def completion( # type: ignore "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" ) elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini": - vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) - - gemini_api_key = ( - api_key - or get_api_key_from_env() - or get_secret("PALM_API_KEY") # older palm api key should also work - or litellm.api_key - ) - - api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") - new_params = safe_deep_copy(optional_params or {}) - response = vertex_chat_completion.completion( # type: ignore - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=gemini_api_key, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, # type: ignore - client=client, - api_base=api_base, - extra_headers=headers, - ) + response = _complete_vertex_ai_beta(_dispatch_ctx) elif custom_llm_provider == "vertex_ai": - vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) - - api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") - - new_params = safe_deep_copy(optional_params or {}) - model_route = get_vertex_ai_model_route( - model=model, litellm_params=litellm_params - ) - - if model_route == VertexAIModelRoute.PARTNER_MODELS: - model_response = vertex_partner_models_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.GEMINI: - model_response = vertex_chat_completion.completion( # type: ignore - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=None, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, # type: ignore - client=client, - api_base=api_base, - extra_headers=headers, - ) - elif model_route == VertexAIModelRoute.GEMMA: - # Vertex Gemma Models with custom prediction endpoint - model_response = vertex_gemma_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.MODEL_GARDEN: - # Vertex Model Garden - OpenAI compatible models - model_response = vertex_model_garden_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.AGENT_ENGINE: - # Vertex AI Agent Engine (Reasoning Engines) - from litellm.llms.vertex_ai.agent_engine.transformation import ( - VertexAgentEngineConfig, - ) - - vertex_agent_engine_config = VertexAgentEngineConfig() - - # Update litellm_params with vertex credentials - litellm_params["vertex_project"] = vertex_ai_project - litellm_params["vertex_location"] = vertex_ai_location - litellm_params["vertex_credentials"] = vertex_credentials - - model_response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - model_response=model_response, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - encoding=_get_encoding(), - api_key=None, - api_base=api_base, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, - custom_llm_provider="vertex_ai", - provider_config=vertex_agent_engine_config, - headers=headers or {}, - ) - else: # VertexAIModelRoute.NON_GEMINI - model_response = vertex_ai_non_gemini.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - ) - - if ( - "stream" in optional_params - and optional_params["stream"] is True - and acompletion is False - ): - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vertex_ai", - logging_obj=logging, - ) - return response - response = model_response + response = _complete_vertex_ai(_dispatch_ctx) elif custom_llm_provider == "predibase": - tenant_id = ( - optional_params.pop("tenant_id", None) - or optional_params.pop("predibase_tenant_id", None) - or litellm.predibase_tenant_id - or get_secret("PREDIBASE_TENANT_ID") - ) - - if tenant_id is None: - raise ValueError( - "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." - ) - - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or get_secret("PREDIBASE_API_BASE") - ) - - api_key = ( - api_key - or litellm.api_key - or litellm.predibase_key - or get_secret("PREDIBASE_API_KEY") - ) - - _model_response = predibase_chat_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - tenant_id=tenant_id, - timeout=timeout, - ) - - if ( - "stream" in optional_params - and optional_params["stream"] is True - and acompletion is False - ): - return _model_response - response = _model_response + response = _complete_predibase(_dispatch_ctx) elif custom_llm_provider == "text-completion-codestral": - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or "https://codestral.mistral.ai/v1/fim/completions" - ) - - api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") - - text_completion_model_response = litellm.TextCompletionResponse( - stream=stream - ) - - _model_response = codestral_text_completions.completion( # type: ignore - model=model, - messages=messages, - model_response=text_completion_model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - timeout=timeout, - ) - - if ( - "stream" in optional_params - and optional_params["stream"] is True - and acompletion is False - ): - return _model_response - response = _model_response + response = _complete_text_completion_codestral(_dispatch_ctx) elif custom_llm_provider == "text-completion-inception": - passed_api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - ) - api_base = ( - passed_api_base - or get_secret_str("INCEPTION_API_BASE") - or "https://api.inceptionlabs.ai/v1" - ) - # FIM is served at `/v1/fim/completions`; the OpenAI client appends - # `/completions`, so point it at the `/v1/fim` base. - api_base = api_base.rstrip("/") - if not api_base.endswith("/fim"): - api_base += "/fim" - - # Don't forward the server-managed Inception key to a caller-supplied - # api_base; only resolve it for the default/server base, or when the - # caller passes their own key. - if passed_api_base is None or api_key: - api_key = ( - api_key - or litellm.inception_key - or get_secret_str("INCEPTION_API_KEY") - ) - - _response = openai_text_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, # type: ignore[arg-type] - custom_llm_provider="text-completion-inception", - api_base=api_base, - acompletion=acompletion, - client=client, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - ) - - if ( - optional_params.get("stream", False) is False - and acompletion is False - and text_completion is False - ): - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - - if optional_params.get("stream", False) or acompletion is True: - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - response = _response + response = _complete_text_completion_inception(_dispatch_ctx) elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): # boto3 reads keys from .env # sagemaker_chat: HF Messages API endpoints # sagemaker_nova: Nova models on SageMaker (OpenAI-compatible) - model_response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - ## RESPONSE OBJECT - response = model_response + response = _complete_sagemaker_chat(_dispatch_ctx) elif custom_llm_provider == "sagemaker": # boto3 reads keys from .env - model_response = sagemaker_llm.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - custom_prompt_dict=custom_prompt_dict, - hf_model_name=hf_model_name, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - ) - - ## RESPONSE OBJECT - response = model_response + response = _complete_sagemaker(_dispatch_ctx) elif custom_llm_provider == "bedrock": # boto3 reads keys from .env - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - - if "aws_bedrock_client" in optional_params: - verbose_logger.warning( - "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." - ) - # Extract credentials for legacy boto3 client and pass thru to httpx - aws_bedrock_client = optional_params.pop("aws_bedrock_client") - creds = aws_bedrock_client._get_credentials().get_frozen_credentials() - - if creds.access_key: - optional_params["aws_access_key_id"] = creds.access_key - if creds.secret_key: - optional_params["aws_secret_access_key"] = creds.secret_key - if creds.token: - optional_params["aws_session_token"] = creds.token - if ( - "aws_region_name" not in optional_params - or optional_params["aws_region_name"] is None - ): - optional_params["aws_region_name"] = ( - aws_bedrock_client.meta.region_name - ) - - bedrock_route = BedrockModelInfo.get_bedrock_route(model) - if bedrock_route == "claude_platform": - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=LlmProviders.BEDROCK, - ) - model = BedrockModelInfo.get_claude_platform_model(model) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - return response - elif bedrock_route == "converse": - model = model.replace("converse/", "") - response = bedrock_converse_chat_completion.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - extra_headers=headers, # Use merged headers instead of original extra_headers - timeout=timeout, - acompletion=acompletion, - client=client, - api_base=api_base, - api_key=api_key, - ) - elif bedrock_route == "converse_like": - model = model.replace("converse_like/", "") - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - else: - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) + response = _complete_bedrock(_dispatch_ctx) elif custom_llm_provider == "watsonx": - response = watsonx_chat_completion.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - custom_llm_provider="watsonx", - ) + response = _complete_watsonx(_dispatch_ctx) elif custom_llm_provider == "watsonx_text": - api_key = ( - api_key - or optional_params.pop("apikey", None) - or get_secret_str("WATSONX_APIKEY") - or get_secret_str("WATSONX_API_KEY") - or get_secret_str("WX_API_KEY") - ) - - api_base = ( - api_base - or optional_params.pop( - "url", - optional_params.pop( - "api_base", optional_params.pop("base_url", None) - ), - ) - or get_secret_str("WATSONX_API_BASE") - or get_secret_str("WATSONX_URL") - or get_secret_str("WX_URL") - or get_secret_str("WML_URL") - ) - - wx_credentials = optional_params.pop( - "wx_credentials", - optional_params.pop( - "watsonx_credentials", None - ), # follow {provider}_credentials, same as vertex ai - ) - - token: Optional[str] = None - if wx_credentials is not None: - api_base = wx_credentials.get("url", api_base) - api_key = wx_credentials.get( - "apikey", wx_credentials.get("api_key", api_key) - ) - token = wx_credentials.get( - "token", - wx_credentials.get( - "watsonx_token", None - ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' - ) - - if token is not None: - optional_params["token"] = token - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="watsonx_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) + response = _complete_watsonx_text(_dispatch_ctx) elif custom_llm_provider == "vllm": - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - model_response = vllm_handler.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - - if ( - "stream" in optional_params and optional_params["stream"] is True - ): ## [BETA] - # don't try to access stream object, - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vllm", - logging_obj=logging, - ) - return response - - ## RESPONSE OBJECT - response = model_response + response = _complete_vllm(_dispatch_ctx) elif custom_llm_provider == "ollama": - api_base = ( - litellm.api_base - or api_base - or get_secret("OLLAMA_API_BASE") - or "http://localhost:11434" - ) - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) + response = _complete_ollama(_dispatch_ctx) elif custom_llm_provider == "ollama_chat": - api_base = ( - litellm.api_base - or api_base - or get_secret("OLLAMA_API_BASE") - or "http://localhost:11434" - ) - - api_key = ( - api_key - or litellm.ollama_key - or os.environ.get("OLLAMA_API_KEY") - or litellm.api_key - ) - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) + response = _complete_ollama_chat(_dispatch_ctx) elif custom_llm_provider == "triton": - api_base = litellm.api_base or api_base - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) + response = _complete_triton(_dispatch_ctx) elif custom_llm_provider == "cloudflare": - api_key = ( - api_key - or litellm.cloudflare_api_key - or litellm.api_key - or get_secret("CLOUDFLARE_API_KEY") - ) - account_id = get_secret("CLOUDFLARE_ACCOUNT_ID") - api_base = ( - api_base - or litellm.api_base - or get_secret("CLOUDFLARE_API_BASE") - or f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/" - ) - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cloudflare", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) + response = _complete_cloudflare(_dispatch_ctx) elif custom_llm_provider == "petals" or model in litellm.petals_models: - api_base = api_base or litellm.api_base - - custom_llm_provider = "petals" - stream = optional_params.pop("stream", False) - model_response = petals_handler.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - client=client, - ) - if stream is True: ## [BETA] - # Fake streaming for petals - resp_string = model_response["choices"][0]["message"]["content"] - response = CustomStreamWrapper( - resp_string, - model, - custom_llm_provider="petals", - logging_obj=logging, - ) - return response - response = model_response + response = _complete_petals(_dispatch_ctx) elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: - try: - client = ( - HTTPHandler(timeout=timeout) if stream is False else None - ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + response = _complete_snowflake(_dispatch_ctx) elif custom_llm_provider == "gradient_ai": - api_base = litellm.api_base or api_base - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="gradient_ai", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) + response = _complete_gradient_ai(_dispatch_ctx) elif custom_llm_provider == "bytez": - api_key = ( - api_key - or litellm.bytez_key - or get_secret_str("BYTEZ_API_KEY") - or litellm.api_key - ) - - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=bytez_transformation, - ) - - pass + response = _complete_bytez(_dispatch_ctx) elif custom_llm_provider == "lemonade": - api_key = ( - api_key - or litellm.lemonade_key - or get_secret_str("LEMONADE_API_KEY") - or litellm.api_key - ) - - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=lemonade_transformation, - ) - - pass + response = _complete_lemonade(_dispatch_ctx) elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models: - api_key = ( - api_key - or litellm.ovhcloud_key - or get_secret_str("OVHCLOUD_API_KEY") - or litellm.api_key - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("OVHCLOUD_API_BASE") - or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" - ) - - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=ovhcloud_transformation, - ) - - pass + response = _complete_ovhcloud(_dispatch_ctx) elif custom_llm_provider == "custom": - url = litellm.api_base or api_base or "" - if url is None or url == "": - raise ValueError( - "api_base not set. Set api_base or litellm.api_base for custom endpoints" - ) - - """ - assume input to custom LLM api bases follow this format: - resp = litellm.module_level_client.post( - api_base, - json={ - 'model': 'meta-llama/Llama-2-13b-hf', # model name - 'params': { - 'prompt': ["The capital of France is P"], - 'max_tokens': 32, - 'temperature': 0.7, - 'top_p': 1.0, - 'top_k': 40, - } - } - ) - - """ - prompt = " ".join([message["content"] for message in messages]) # type: ignore - resp = litellm.module_level_client.post( - url, - headers=headers, - json={ - "model": model, - "params": { - "prompt": [prompt], - "max_tokens": max_tokens, - "temperature": temperature, - "top_p": top_p, - "top_k": kwargs.get("top_k"), - }, - **kwargs.get("extra_body", {}), - }, - ) - response_json = resp.json() - """ - assume all responses from custom api_bases of this format: - { - 'data': [ - { - 'prompt': 'The capital of France is P', - 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], - 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], - 'message': 'ok' - } - ] - } - """ - string_response = response_json["data"][0]["output"][0] - ## RESPONSE OBJECT - model_response.choices[0].message.content = string_response # type: ignore - model_response.created = int(time.time()) - model_response.model = model - response = model_response + response = _complete_custom(_dispatch_ctx) elif ( custom_llm_provider in litellm._custom_providers ): # Assume custom LLM provider # Get the Custom Handler - custom_handler: Optional[CustomLLM] = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] - - if custom_handler is None: - raise LiteLLMUnknownProvider( - model=model, custom_llm_provider=custom_llm_provider - ) - - ## ROUTE LLM CALL ## - handler_fn = custom_chat_llm_router( - async_fn=acompletion, stream=stream, custom_llm=custom_handler - ) - - headers = headers or litellm.headers or {} - - ## CALL FUNCTION - response = handler_fn( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - ) - if stream is True: - return CustomStreamWrapper( - completion_stream=response, - model=model, - custom_llm_provider=custom_llm_provider, - logging_obj=logging, - ) + response = _complete_custom_providers(_dispatch_ctx) elif custom_llm_provider == "langgraph": # LangGraph - Agent Runtime Provider - from litellm.llms.langgraph.chat.transformation import LangGraphConfig - - ( - api_base, - api_key, - ) = LangGraphConfig()._get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) - - headers = headers or litellm.headers - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) + response = _complete_langgraph(_dispatch_ctx) elif custom_llm_provider == "langflow": # LangFlow - Visual AI Agent Platform - from litellm.llms.langflow.chat.transformation import LangFlowConfig - - ( - api_base, - api_key, - ) = LangFlowConfig()._get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) - - headers = headers or litellm.headers - - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) + response = _complete_langflow(_dispatch_ctx) else: raise LiteLLMUnknownProvider( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4f022e1f882..6ebac7efc8d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10443,7 +10443,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "supports_speed": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -10476,7 +10477,8 @@ "fast": 6.0 }, "supports_max_reasoning_effort": true, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -10511,7 +10513,8 @@ "us": 1.1, "fast": 6.0 }, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -10546,7 +10549,8 @@ "us": 1.1, "fast": 6.0 }, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -10615,7 +10619,8 @@ "us": 1.1, "fast": 2.0 }, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -10684,6 +10689,268 @@ "mode": "chat", "output_cost_per_token": 1.923e-06 }, + "cloudflare/@cf/openai/gpt-oss-120b": { + "input_cost_per_token": 3.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/google/gemma-2b-it-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/meta/llama-3.2-3b-instruct": { + "input_cost_per_token": 5.09e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 80000, + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 3.35e-07 + }, + "cloudflare/@cf/meta/llama-guard-3-8b": { + "input_cost_per_token": 4.84e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3e-08 + }, + "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 15000, + "max_output_tokens": 15000, + "max_tokens": 15000, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": { + "input_cost_per_token": 4.97e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 80000, + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 4.881e-06, + "supports_reasoning": true + }, + "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": { + "input_cost_per_token": 1.52e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.87e-07 + }, + "cloudflare/@cf/meta/llama-3.2-1b-instruct": { + "input_cost_per_token": 2.7e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 60000, + "max_output_tokens": 60000, + "max_tokens": 60000, + "mode": "chat", + "output_cost_per_token": 2.01e-07 + }, + "cloudflare/@cf/moonshotai/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/zai-org/glm-4.7-flash": { + "input_cost_per_token": 6.05e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/meta-llama/llama-2-7b-chat-hf-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { + "input_cost_per_token": 2.93e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 24000, + "max_output_tokens": 24000, + "max_tokens": 24000, + "mode": "chat", + "output_cost_per_token": 2.253e-06, + "supports_function_calling": true + }, + "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": { + "input_cost_per_token": 1.7e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 131000, + "max_output_tokens": 131000, + "max_tokens": 131000, + "mode": "chat", + "output_cost_per_token": 1.12e-07, + "supports_function_calling": true + }, + "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": { + "input_cost_per_token": 6.6e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1e-06 + }, + "cloudflare/@cf/zai-org/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "cloudflare", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/nvidia/nemotron-3-120b-a12b": { + "input_cost_per_token": 5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/aisingapore/gemma-sea-lion-v4-27b-it": { + "input_cost_per_token": 3.51e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.55e-07 + }, + "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": { + "input_cost_per_token": 5.09e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3.35e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/google/gemma-7b-it-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 3500, + "max_output_tokens": 3500, + "max_tokens": 3500, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 3e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/mistralai/mistral-small-3.1-24b-instruct": { + "input_cost_per_token": 3.51e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.55e-07, + "supports_function_calling": true + }, + "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": { + "input_cost_per_token": 4.85e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6.76e-07, + "supports_vision": true + }, + "cloudflare/@cf/openai/gpt-oss-20b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/meta/llama-4-scout-17b-16e-instruct": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 131000, + "max_output_tokens": 131000, + "max_tokens": 131000, + "mode": "chat", + "output_cost_per_token": 8.5e-07, + "supports_function_calling": true + }, + "cloudflare/@cf/qwen/qwq-32b": { + "input_cost_per_token": 6.6e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 24000, + "max_output_tokens": 24000, + "max_tokens": 24000, + "mode": "chat", + "output_cost_per_token": 1e-06, + "supports_reasoning": true + }, "codestral/codestral-2405": { "input_cost_per_token": 0.0, "litellm_provider": "codestral", @@ -20088,8 +20355,6 @@ "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_priority": 1.4e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -20163,8 +20428,6 @@ "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, "output_cost_per_token_priority": 2.8e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -20238,8 +20501,6 @@ "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 8e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -20311,8 +20572,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.7e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20354,8 +20613,6 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20377,8 +20634,6 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20667,8 +20922,6 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -21372,8 +21625,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21767,6 +22018,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21815,6 +22068,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21859,6 +22114,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21903,6 +22160,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21951,6 +22210,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21998,6 +22259,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22038,6 +22301,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22081,6 +22346,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22126,6 +22393,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22172,6 +22441,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22215,6 +22486,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22258,6 +22531,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22296,8 +22571,6 @@ "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -22704,8 +22977,6 @@ "output_cost_per_token": 2e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22787,8 +23058,6 @@ "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_flex": 2e-07, diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b27082c361a..3a9ef8db804 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -10,7 +10,7 @@ import os import re from functools import partial from io import IOBase -from typing import Any, Coroutine, Dict, Optional, Union +from typing import Any, Callable, Coroutine, Dict, Optional, Union, cast import httpx @@ -20,6 +20,7 @@ from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.ocr.rust_bridge import RustOcr, load_rust_ocr, rust_ocr_enabled from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -28,6 +29,82 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# +def _timeout_to_seconds( + timeout: Optional[Union[float, httpx.Timeout]], +) -> Optional[float]: + """Convert the Python OCR timeout to a single seconds value for the Rust bridge. + + The Rust HTTP client takes one duration; ``httpx.Timeout`` carries separate + connect/read/write/pool values, so pick the read deadline as the closest + analog to a total-request timeout. + """ + if timeout is None: + return None + if isinstance(timeout, httpx.Timeout): + return timeout.read + return float(timeout) + + +def _run_rust_ocr( + rust_ocr: RustOcr, + logging_obj: LiteLLMLoggingObj, + provider_config: BaseOCRConfig, + resolve_api_key: Callable[[str], Optional[str]], + model: str, + document: dict[str, object], + api_key: Optional[str], + api_base: Optional[str], + optional_params: dict[str, object], + litellm_params: dict[str, object], + timeout_seconds: Optional[float], +) -> OCRResponse: + """Run the Mistral OCR call through the Rust bridge and wrap the result. + + Resolves the key the same way the Python path does so secret-manager backends + (AWS/Azure/GCP/Vault) work; the Rust bridge's own fallback only reads the + process environment. The request that Rust actually sends (resolved URL and + headers) is mirrored into pre_call so logs match the wire. Dependencies are + injected so this stays unit-testable without patching module globals. + """ + resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY") + resolved_headers = provider_config.validate_environment( + headers={}, + model=model, + api_key=resolved_api_key, + api_base=api_base, + litellm_params=litellm_params, + ) + resolved_complete_url = provider_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + logging_obj.pre_call( + input="OCR document processing", + api_key=resolved_api_key, + additional_args={ + "complete_input_dict": { + "model": model, + "document": document, + **optional_params, + }, + "api_base": resolved_complete_url, + "headers": resolved_headers, + }, + ) + return OCRResponse.model_validate( + rust_ocr( + model=model, + document=document, + api_key=resolved_api_key, + api_base=api_base, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + ) + + @client async def aocr( model: str, @@ -220,7 +297,7 @@ def ocr( """ local_vars = locals() try: - litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")) litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True @@ -261,7 +338,6 @@ def ocr( if dynamic_api_base: api_base = dynamic_api_base - # Get provider config ocr_provider_config: Optional[BaseOCRConfig] = ( ProviderConfigManager.get_provider_ocr_config( model=model, @@ -278,17 +354,14 @@ def ocr( f"OCR call - model: {model}, provider: {custom_llm_provider}" ) - # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) - # Extract OCR-specific parameters from kwargs supported_params = ocr_provider_config.get_supported_ocr_params(model=model) non_default_params = {} for param in supported_params: if param in kwargs: non_default_params[param] = kwargs.pop(param) - # Map parameters to provider-specific format optional_params = ocr_provider_config.map_ocr_params( non_default_params=non_default_params, optional_params={}, @@ -297,7 +370,8 @@ def ocr( verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") - # Pre Call logging + effective_timeout = timeout or request_timeout + litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model, @@ -309,12 +383,35 @@ def ocr( custom_llm_provider=custom_llm_provider, ) - # Call the handler - pass document dict directly + # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. + if custom_llm_provider == "mistral" and rust_ocr_enabled(): + rust_ocr = load_rust_ocr() + if rust_ocr is None: + verbose_logger.debug( + "Rust OCR bridge unavailable; falling back to Python path" + ) + else: + from litellm.secret_managers.main import get_secret_str + + return _run_rust_ocr( + rust_ocr=rust_ocr, + logging_obj=litellm_logging_obj, + provider_config=ocr_provider_config, + resolve_api_key=get_secret_str, + model=model, + document=document, + api_key=api_key, + api_base=api_base, + optional_params=optional_params, + litellm_params=dict(litellm_params), + timeout_seconds=_timeout_to_seconds(effective_timeout), + ) + response = base_llm_http_handler.ocr( model=model, - document=document, # Pass the entire document dict + document=document, optional_params=optional_params, - timeout=timeout or request_timeout, + timeout=effective_timeout, logging_obj=litellm_logging_obj, api_key=api_key, api_base=api_base, diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py new file mode 100644 index 00000000000..61f9e8ca69a --- /dev/null +++ b/litellm/ocr/rust_bridge.py @@ -0,0 +1,74 @@ +""" +Optional Rust-backed OCR path. + +Enable with ``litellm.use_litellm_rust()``; the sync ``litellm.ocr()`` entrypoint +then routes supported Mistral calls through the compiled ``litellm_python_bridge`` +extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust. + +No module-level ``litellm`` imports keep this a leaf so ``litellm/ocr/main.py`` +can import it statically without forming an import cycle. +""" + +from __future__ import annotations + +from typing import Final, Protocol, cast + + +class RustOcr(Protocol): + """Signature of the compiled ``litellm_python_bridge.ocr`` entrypoint.""" + + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: ... + + +class _Unset: + """Sentinel type so ``ocr=None`` can clear a prior injection while omission preserves it.""" + + +_UNSET: Final[_Unset] = _Unset() + +_rust_ocr_enabled = False +_rust_ocr_impl: RustOcr | None = None + + +def use_litellm_rust( + enabled: bool = True, *, ocr: RustOcr | None | _Unset = _UNSET +) -> None: + """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension. + + ``ocr`` injects the bridge callable; when omitted the compiled extension is + loaded on demand and any previously injected bridge is preserved. Pass + ``ocr=None`` explicitly to clear a prior injection. + """ + global _rust_ocr_enabled, _rust_ocr_impl + _rust_ocr_enabled = enabled + if not isinstance(ocr, _Unset): + _rust_ocr_impl = ocr + + +def rust_ocr_enabled() -> bool: + """Whether the Rust OCR path has been turned on via ``use_litellm_rust()``.""" + return _rust_ocr_enabled + + +def load_rust_ocr() -> RustOcr | None: + """Return the Rust OCR callable, or ``None`` when no bridge is available. + + Prefers an injected implementation, otherwise loads the compiled + ``litellm_python_bridge`` extension; a missing extension yields ``None`` so + the caller can fall back to the Python path instead of hard-failing. + """ + if _rust_ocr_impl is not None: + return _rust_ocr_impl + try: + import litellm_python_bridge + except ImportError: + return None + return cast(RustOcr, litellm_python_bridge.ocr) diff --git a/litellm/proxy/_experimental/mcp_server/AGENTS.md b/litellm/proxy/_experimental/mcp_server/AGENTS.md new file mode 100644 index 00000000000..8eebc3ea3b3 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/AGENTS.md @@ -0,0 +1,95 @@ +# Experimental MCP Server Change Guidelines + +Read @../../../../CLAUDE.md and @CLAUDE.md before changing this package. + +This directory owns the proxy-hosted MCP server implementation. Keep changes +inside the module that owns the behavior, and only reach outside this package +when the public type contract, database schema, dashboard, or cross-proxy route +wiring must change with it. + +## File Structure + +Respect the current package boundaries: + +```text +litellm/proxy/_experimental/mcp_server/ + AGENTS.md + CLAUDE.md + server.py # ASGI/MCP route handling, sessions, tool calls [PR7: 7-arm only — move BYOK/OAuth pre-fetch into resolver] + mcp_server_manager.py # upstream server registry, clients, tool routing [PR7: _create_mcp_client swaps resolve_mcp_auth -> resolve_credentials] + auth/ + user_api_key_auth_mcp.py # LiteLLM admission auth and MCP request headers + token_exchange.py # OAuth token exchange handling [unchanged; V1TokenExchangeAdapter delegates here] + litellm_auth_handler.py # authenticated-user adapter for MCP sessions + outbound_credentials/ # NEW — typed upstream-credential resolution (resolve_credentials + arms) + __init__.py # public surface: resolve_credentials, the configs, CredError + result.py # Ok | Error union (pure stdlib) + types.py # AuthConfig union, CredError, Subject, ServerSpec + httpx_auth.py # NoOpAuth, StaticHeaderAuth (every mode -> one httpx.Auth) + resolver.py # resolve_credentials(): exhaustive per-mode match + assert_never + seams.py # injected Protocols (one per cache-touching mode) + v1_adapters.py # v1-backed seam bodies; delegate to auth/oauth2/db owners + adapter.py # to_subject / to_server_spec / raise_public (v1 <-> v2 boundary) + discoverable_endpoints.py # MCP OAuth metadata, authorize, token, callback + byok_oauth_endpoints.py # BYOK OAuth UI/API flow + oauth_utils.py # redirect URI and proxy base URL validation + oauth2_token_cache.py # OAuth2 and per-user token resolution/cache [PR7: resolve_mcp_auth removed; cache class stays, V1OAuth2CacheAdapter delegates to async_get_token] + db.py # MCP server, credential, env var, submission DB access [unchanged; V1ByokStore delegates to _get_byok_credential / get_user_credential] + toolset_db.py # MCP toolset DB access + rest_endpoints.py # proxy REST facade for listing/calling MCP tools [PR7: 7-arm only — pass identity + inbound token down instead of mcp_auth_header] + openapi_to_mcp_generator.py# OpenAPI spec to MCP tool generation + sampling_handler.py # MCP sampling to LiteLLM completion flow + elicitation_handler.py # MCP elicitation relay flow + semantic_tool_filter.py # semantic filtering of available MCP tools + guardrail_translation/ + handler.py # MCP guardrail result translation + sse_transport.py # SSE transport implementation + mcp_context.py # contextvars for MCP request/session metadata + mcp_debug.py # debug helpers + tool_registry.py # in-memory MCP tool registry helpers + cost_calculator.py # MCP tool cost calculation + ui_session_utils.py # dashboard session auth context helpers + utils.py # shared primitives used by several modules +``` + +Do not add broad catch-all modules. Prefer the existing owner above, and add a +new file only for a distinct capability that would otherwise make an existing +module materially harder to understand. + +## Implementation Rules + +- Preserve the boundary between LiteLLM admission auth and upstream MCP auth. + Admission belongs in `auth/user_api_key_auth_mcp.py`; upstream token exchange, + delegated auth, per-user OAuth, BYOK, and raw header forwarding belong in the + dedicated OAuth/header modules. +- Treat `none`, bearer/API key, OAuth, OAuth token exchange, delegated upstream + auth, SSE, streamable HTTP, and stdio as separate flows. Do not collapse them + behind a single generic branch unless tests prove every mode still behaves + correctly. +- Be especially careful with `available_on_public_internet: false` combined with + `delegate_auth_to_upstream: true`. The local `CLAUDE.md` explains the anonymous + upstream PKCE path that must remain intentional. +- Keep database-backed fields in sync across migrations, typed models under + `litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this + package, and dashboard state when the field is user-visible. +- Use the official MCP SDK types and established LiteLLM Pydantic models where + they exist. Avoid untyped protocol dictionaries at package boundaries. +- Keep security-sensitive logic easy to audit. Header forwarding, IP filtering, + public internet checks, token storage, env var interpolation, and credential + encryption need focused tests for both allowed and rejected paths. +- Avoid adding comments to new code unless they explain non-obvious security or + protocol behavior. Prefer clear names and small functions. + +## Tests + +Mirror this package under `tests/test_litellm/proxy/_experimental/mcp_server/`. +For regressions, extend the existing mapped test file instead of creating a new +one. Use subdirectories that match the implementation path, such as +`auth/test_token_exchange.py` for `auth/token_exchange.py` and +`guardrail_translation/test_mcp_guardrail_handler.py` for +`guardrail_translation/handler.py`. + +Use `tests/mcp_tests/` only when extending an existing broader MCP integration +scenario that already lives there. Route, auth, tool listing, tool execution, +OAuth, sampling, elicitation, DB, and dashboard-session changes should have +focused coverage in the mirrored `tests/test_litellm/...` path first. diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index e47fc84b533..90108de25c3 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -12,6 +12,7 @@ from litellm.proxy._types import ( LiteLLM_TeamTable, ProxyException, SpecialHeaders, + SpecialMCPServerNames, UserAPIKeyAuth, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils @@ -642,6 +643,15 @@ class MCPRequestHandler: user_api_key_auth ) ) + + # The key explicitly opted out of every MCP server. This overrides + # team inheritance and additive grants (mirrors no-default-models). + if ( + SpecialMCPServerNames.no_mcp_servers.value + in allowed_mcp_servers_for_key + ): + return [] + allowed_mcp_servers_for_team = ( await MCPRequestHandler._get_allowed_mcp_servers_for_team( user_api_key_auth @@ -1058,6 +1068,13 @@ class MCPRequestHandler: if key_object_permission is None: return [] + # Sentinel opt-out: surface it unexpanded so the caller can short-circuit + # to zero servers instead of inheriting the team. + if SpecialMCPServerNames.no_mcp_servers.value in ( + key_object_permission.mcp_servers or [] + ): + return [SpecialMCPServerNames.no_mcp_servers.value] + # Permission entries may be server_ids OR names/aliases — expand to ids. direct_mcp_servers = global_mcp_server_manager.expand_permission_list( key_object_permission.mcp_servers or [] diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index afec884cd96..5e704b889ae 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -80,6 +80,7 @@ from litellm.proxy._types import ( MCPEnvVar, MCPTransport, MCPTransportType, + SpecialMCPServerNames, UserAPIKeyAuth, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils @@ -1349,6 +1350,17 @@ class MCPServerManager: allow_all_server_ids = self.get_allow_all_keys_server_ids() try: + # The key explicitly opted out of every MCP server. Return zero before + # layering on allow_all_keys servers so the opt-out is absolute. + key_object_permission = ( + user_api_key_auth.object_permission if user_api_key_auth else None + ) + if key_object_permission is not None and ( + SpecialMCPServerNames.no_mcp_servers.value + in (key_object_permission.mcp_servers or []) + ): + return [] + # Check if object_permission.mcp_servers is explicitly set has_explicit_object_permission = False if user_api_key_auth and user_api_key_auth.object_permission: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py new file mode 100644 index 00000000000..73166a45d6e --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py @@ -0,0 +1,73 @@ +"""Typed upstream-credential resolution for MCP servers. + +This subpackage houses the typed credential vocabulary and the ``resolve_credentials`` +dispatch. A server declares one per-mode config from the ``AuthConfig`` discriminated union; +``UpstreamCredentialProvider.resolve_credentials`` selects one arm and returns an ``httpx.Auth`` +or a typed ``CredError``. Failures are modeled as values via :mod:`.result` (``Result[T, +CredError]``) rather than raised, so every seam is total. Nothing here is wired onto a live +request path yet. +""" + +from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import ( + NoOpAuth, + StaticHeaderAuth, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import ( + UpstreamCredentialProvider, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( + Error, + Ok, + Result, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + Ambient, + ApiKeyConfig, + ApiKeySource, + AssumeRole, + AuthConfig, + AuthorizationCodeConfig, + AuthSpecKind, + AwsCredentialSource, + AwsSigV4Config, + Byok, + ClientCredentialsConfig, + CredError, + NoneConfig, + PassthroughConfig, + ServerSpec, + SharedKey, + StaticKeys, + Subject, + TokenExchangeConfig, + parse_auth_spec_kind, +) + +__all__ = [ + "Ok", + "Error", + "Result", + "NoOpAuth", + "StaticHeaderAuth", + "UpstreamCredentialProvider", + "AuthSpecKind", + "CredError", + "Subject", + "ServerSpec", + "AuthConfig", + "parse_auth_spec_kind", + "AuthorizationCodeConfig", + "ClientCredentialsConfig", + "TokenExchangeConfig", + "ApiKeyConfig", + "ApiKeySource", + "SharedKey", + "Byok", + "PassthroughConfig", + "NoneConfig", + "AwsSigV4Config", + "AwsCredentialSource", + "StaticKeys", + "AssumeRole", + "Ambient", +] diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py new file mode 100644 index 00000000000..2345fa98123 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py @@ -0,0 +1,45 @@ +"""Concrete `httpx.Auth` objects the resolver returns for the self-contained modes. + +These are the egress credential as the SDK consumes it: an `httpx.Auth` attached to the +upstream `AsyncClient`. The OAuth-flow modes (`authorization_code`, `client_credentials`, +`token_exchange`) return SDK-provided auth objects instead and land later. + +`auth_flow` mutating the outbound request is the `httpx.Auth` contract, not a house-style +violation: the request is httpx's object, and these carry no state of their own. +""" + +from __future__ import annotations + +from collections.abc import Generator + +import httpx +from pydantic import SecretStr + + +class NoOpAuth(httpx.Auth): + """Attaches nothing — the `none` mode (and the seam-level default).""" + + def auth_flow( + self, request: httpx.Request + ) -> Generator[httpx.Request, httpx.Response, None]: + yield request + + +class StaticHeaderAuth(httpx.Auth): + """Sets one fixed header on every request — the `api_key` family and `passthrough`. + + The header value is a live credential (a bearer token, an API key, a forwarded user + token), so it is held as a `SecretStr` and unwrapped only when written onto the request. + That keeps it masked in reprs, `vars()`, tracebacks, and structured logs, matching the + `SecretStr` discipline the config models use. + """ + + def __init__(self, header_value: str, header_name: str = "Authorization") -> None: + self.header_name = header_name + self._header_value = SecretStr(header_value) + + def auth_flow( + self, request: httpx.Request + ) -> Generator[httpx.Request, httpx.Response, None]: + request.headers[self.header_name] = self._header_value.get_secret_value() + yield request diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py new file mode 100644 index 00000000000..7bcdb3e6529 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py @@ -0,0 +1,70 @@ +"""The one credential resolver: dispatch on the declared mode, fail closed. + +`resolve_credentials` selects exactly one arm off the server's typed `config` and either +produces an `httpx.Auth` or returns a typed `CredError`. The `match` is over the `AuthConfig` +variant, so each arm receives its own fully-typed config with no field-presence inference and +no precedence cascade. It is wildcard-free with an `assert_never` tail, so adding a mode without +an arm fails the type gate (basedpyright `reportMatchNotExhaustive`); a bypassed gate fails loudly +at runtime instead of returning `None`. + +This skeleton ships every arm as a `not_implemented` stub. Each mode's real body, with its +injected seam, lands in its own follow-up PR; until then the arm returns a typed error rather +than silently producing no credential. Pure v2: no imports from v1. +""" + +from __future__ import annotations + +import httpx +from typing_extensions import assert_never + +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( + Error, + Result, +) +from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( + ApiKeyConfig, + AuthorizationCodeConfig, + AuthSpecKind, + AwsSigV4Config, + ClientCredentialsConfig, + CredError, + NoneConfig, + PassthroughConfig, + ServerSpec, + Subject, + TokenExchangeConfig, +) + + +class UpstreamCredentialProvider: + """Produces the one `httpx.Auth` for a `(subject, upstream)` pair, per declared mode. + + Collaborators (the per-mode credential stores and token fetchers) are injected as each arm + is built; the skeleton needs none, since every arm is a stub. + """ + + async def resolve_credentials( + self, subject: Subject, server: ServerSpec + ) -> Result[httpx.Auth, CredError]: + match server.config: + case NoneConfig(): + return _not_implemented(AuthSpecKind.none) + case ApiKeyConfig(): + return _not_implemented(AuthSpecKind.api_key) + case PassthroughConfig(): + return _not_implemented(AuthSpecKind.passthrough) + case ClientCredentialsConfig(): + return _not_implemented(AuthSpecKind.client_credentials) + case TokenExchangeConfig(): + return _not_implemented(AuthSpecKind.token_exchange) + case AuthorizationCodeConfig(): + return _not_implemented(AuthSpecKind.authorization_code) + case AwsSigV4Config(): + return _not_implemented(AuthSpecKind.aws_sigv4) + assert_never(server.config) + + +def _not_implemented(kind: AuthSpecKind) -> Result[httpx.Auth, CredError]: + return Error( + CredError.of_not_implemented(f"{kind.value}: resolver arm not implemented yet") + ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py new file mode 100644 index 00000000000..a612e8510f5 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/result.py @@ -0,0 +1,54 @@ +"""A tagged-union ``Result`` the type checker can actually narrow. + +``Ok`` and ``Error`` are separate frozen classes joined by a ``Union`` alias, so +reaching for ``result.ok`` before eliminating the ``Error`` arm (via ``isinstance`` +or a ``match`` pattern) is a type error rather than a runtime ``AttributeError``. A +single class carrying both payload fields would make that unguarded access invisible +to the type checker. + +Both variants are covariant and frozen; the absent side defaults to ``Never`` so a +bare ``Ok(value)`` or ``Error(err)`` infers fully and is assignable to any ``Result`` +whose matching side fits. + +``is_ok`` / ``is_error`` are runtime predicates that also narrow via their ``Literal`` +returns; inside strictly typed code, discriminate with ``match`` or ``isinstance``. + +This is the shared ``Result`` shape for the ``outbound_credentials`` resolver: every +seam returns ``Result[T, CredError]`` instead of raising, so each failure is a value +the caller must handle rather than an exception that can slip past the type checker. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Generic, Literal, TypeAlias + +from typing_extensions import Never, TypeVar + +_TOk_co = TypeVar("_TOk_co", covariant=True, default=Never) +_TError_co = TypeVar("_TError_co", covariant=True, default=Never) + + +@dataclass(frozen=True) +class Ok(Generic[_TOk_co, _TError_co]): + ok: _TOk_co + + def is_ok(self) -> Literal[True]: + return True + + def is_error(self) -> Literal[False]: + return False + + +@dataclass(frozen=True) +class Error(Generic[_TOk_co, _TError_co]): + error: _TError_co + + def is_ok(self) -> Literal[False]: + return False + + def is_error(self) -> Literal[True]: + return True + + +Result: TypeAlias = Ok[_TOk_co, _TError_co] | Error[_TOk_co, _TError_co] diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py new file mode 100644 index 00000000000..2088dc77252 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py @@ -0,0 +1,334 @@ +"""The upstream-credential vocabulary — the typed seam the resolver dispatches on. + +This module ships the data types only; the resolver lands in a later PR. It is the contract +the credential build implements and the spec tests assert against. + +Design invariants encoded here: + +- **Mode is the single source of truth.** A server declares exactly one per-mode `config` + (the `AuthConfig` discriminated union); `auth_spec_kind` is *derived* from it, never a + second field that can drift. The resolver dispatches on the config variant, one arm per + mode. No field-presence inference, no precedence cascade. +- **Illegal states unrepresentable.** Each mode's config is its own frozen model holding + only that mode's fields — an `aws_sigv4` server cannot hold OAuth fields, and a config + missing a required field is rejected at construction, not at call time. +- **Fail-closed at the boundary.** A raw mode string can only enter through + `parse_auth_spec_kind()`, which returns a typed `CredError`. +- **Errors as values.** Every seam returns `Result[_, CredError]`; only edge adapters raise. +- **No v1 imports.** This vocabulary stays free of `MCPServer` and the rest of v1; the + v1 -> v2 adapter maps onto these types in a later PR. + +Sum types are Expression `@tagged_union`s discriminated on a `Literal` `tag`, matched via +`self.tag` with an `assert_never` tail; `Result` is this package's vendored `Ok | Error` +union (see `result.py`), not `expression.Result`. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Annotated, Literal + +from expression import case, tag, tagged_union +from pydantic import BaseModel, ConfigDict, Field, SecretStr +from typing_extensions import assert_never + +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import ( + Error, + Ok, + Result, +) + + +class AuthSpecKind(str, Enum): + """The server's statically-declared upstream-auth mode — derived from its `config`. + + Covers v1's full `MCPAuth` surface, not only OAuth grants: the three grant modes, the + collapsed static-header family, client passthrough, no-auth, and AWS request signing. + BYOK is *not* a member: it is the `api_key` mode seeded per-user, a source selector + inside that arm. The static-header schemes v1 splits into separate `MCPAuth` values + (`bearer_token`/`api_key`/`basic`/`token`/`authorization`) collapse into `api_key`; the + scheme is a parameter the arm carries, not its own mode. + """ + + authorization_code = "authorization_code" # per-user 3LO; gateway-stored token + client_credentials = "client_credentials" # gateway service account (M2M) + token_exchange = "token_exchange" # RFC 8693: token endpoint + subject_token (OBO) + api_key = "api_key" # static header, any scheme (BYOK = per-user-seeded source) + passthrough = "passthrough" # client forwards an upstream-audience token + none = "none" # no upstream credential; resolve yields a no-op auth, never an error + aws_sigv4 = "aws_sigv4" # AWS SigV4 per-request signing (e.g. Bedrock AgentCore) + + +@tagged_union(frozen=True) +class CredError: + """Why a credential could not be produced. Fail-closed: an arm yields this or an `httpx.Auth`. + + Discriminated on the `Literal` `tag`; consumers `match self.tag` (see `summary`) so the + type checker can prove exhaustiveness. Construct via the `of_*` factories. + """ + + tag: Literal[ + "unauthorized", + "misconfigured", + "upstream_unavailable", + "unsupported_mode", + "precondition_required", + "not_implemented", + ] = tag() + + unauthorized: str = ( + case() + ) # no usable credential for this (subject, server) -> 401 challenge + misconfigured: str = ( + case() + ) # the declared mode is missing required config -> 5xx (operator) + upstream_unavailable: str = ( + case() + ) # the IdP / token endpoint could not be reached -> 503 + unsupported_mode: str = ( + case() + ) # a raw mode string did not parse into AuthSpecKind (boundary) + precondition_required: str = ( + case() + ) # a required per-user value (e.g. an env var) has not been provided -> 412 + not_implemented: str = ( + case() + ) # the declared mode's resolver arm is not built yet -> 501 (not operator error) + + @staticmethod + def of_unauthorized(detail: str) -> CredError: + return CredError(unauthorized=detail) + + @staticmethod + def of_misconfigured(detail: str) -> CredError: + return CredError(misconfigured=detail) + + @staticmethod + def of_upstream_unavailable(detail: str) -> CredError: + return CredError(upstream_unavailable=detail) + + @staticmethod + def of_unsupported_mode(detail: str) -> CredError: + return CredError(unsupported_mode=detail) + + @staticmethod + def of_precondition_required(detail: str) -> CredError: + return CredError(precondition_required=detail) + + @staticmethod + def of_not_implemented(detail: str) -> CredError: + return CredError(not_implemented=detail) + + @property + def summary(self) -> str: + # Exhaustiveness: every Literal tag has an arm; the trailing assert_never typechecks + # only while that stays true (a `case _` would defeat reportMatchNotExhaustive). + match self.tag: + case "unauthorized": + return f"unauthorized: {self.unauthorized}" + case "misconfigured": + return f"misconfigured: {self.misconfigured}" + case "upstream_unavailable": + return f"upstream unavailable: {self.upstream_unavailable}" + case "unsupported_mode": + return self.unsupported_mode + case "precondition_required": + return f"precondition required: {self.precondition_required}" + case "not_implemented": + return f"not implemented: {self.not_implemented}" + assert_never(self.tag) + + +class AuthorizationCodeConfig(BaseModel): + """Per-user 3LO; the gateway is the OAuth client and stores the user's token. + + Endpoints are discovered (RFC 9728 -> RFC 8414) and the client is registered via DCR + (RFC 7591), so the common case carries none of the fields below; they are optional manual + overrides for IdPs without discovery / DCR. The per-user token is read from the token store + at resolve time, not held here. + """ + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.authorization_code] = AuthSpecKind.authorization_code + scopes: tuple[str, ...] = () + client_id: str | None = None + client_secret: SecretStr | None = None + authorization_url: str | None = None + token_url: str | None = None + + +class ClientCredentialsConfig(BaseModel): + """M2M service account; one upstream identity for every user. + + Fields are optional so the config can be built incomplete: a value may be supplied at + runtime (`token_url` via RFC 8414 discovery, `client_id`/`secret` via DCR), and the + resolver arm raises `CredError.misconfigured` when a needed field is still absent. + """ + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.client_credentials] = AuthSpecKind.client_credentials + client_id: str | None = None + client_secret: SecretStr | None = None + token_url: str | None = None + scopes: tuple[str, ...] = () + + +class TokenExchangeConfig(BaseModel): + """RFC 8693 OBO; swap the caller's live subject_token for a token bound to the upstream's + audience (`server.resource`, RFC 8707). The gateway authenticates to the exchange endpoint + as an OAuth client (`client_id`/`client_secret`); the inbound token is sent only to that + endpoint, never to the upstream. + """ + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.token_exchange] = AuthSpecKind.token_exchange + subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token" + token_exchange_endpoint: str | None = None + client_id: str | None = None + client_secret: SecretStr | None = None + scopes: tuple[str, ...] = () + + +class SharedKey(BaseModel): + """A fixed key configured on the server, identical for every caller.""" + + model_config = ConfigDict(frozen=True) + source: Literal["shared"] = "shared" + value: SecretStr + + +class Byok(BaseModel): + """A key the user brings via the entry flow, stored per-user and pulled from the credential + store at resolve time. Missing means the user must provide it, a 401 + WWW-Authenticate + challenge.""" + + model_config = ConfigDict(frozen=True) + source: Literal["byok"] = "byok" + + +ApiKeySource = Annotated[SharedKey | Byok, Field(discriminator="source")] + + +class ApiKeyConfig(BaseModel): + """A fixed credential injected as a header. The value is shared (in config) or seeded + per-user (pulled from the store); `header_name` and `value_prefix` say where and how it is + written, modeled like OpenAPI's apiKey scheme so any upstream convention is expressible + (Authorization + Bearer, a raw value on X-API-Key, Ocp-Apim-Subscription-Key, etc.). + """ + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.api_key] = AuthSpecKind.api_key + header_name: str = "Authorization" + value_prefix: str = "Bearer" + key_source: ApiKeySource + + def header(self, value: str) -> tuple[str, str]: + formatted = f"{self.value_prefix} {value}" if self.value_prefix else value + return self.header_name, formatted + + +class PassthroughConfig(BaseModel): + """Client-driven upstream OAuth; the gateway forwards the client's upstream token.""" + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.passthrough] = AuthSpecKind.passthrough + + +class NoneConfig(BaseModel): + """No upstream credential; the request is sent unauthenticated.""" + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.none] = AuthSpecKind.none + + +class StaticKeys(BaseModel): + """Long-lived AWS access keys configured on the server.""" + + model_config = ConfigDict(frozen=True) + source: Literal["static_keys"] = "static_keys" + access_key_id: str + secret_access_key: SecretStr + session_token: SecretStr | None = None + + +class AssumeRole(BaseModel): + """An IAM role the gateway assumes via STS for short-lived, auto-refreshed credentials.""" + + model_config = ConfigDict(frozen=True) + source: Literal["assume_role"] = "assume_role" + role_arn: str + session_name: str | None = None + external_id: str | None = None + + +class Ambient(BaseModel): + """The environment's default AWS credential chain (instance profile, IRSA, env vars).""" + + model_config = ConfigDict(frozen=True) + source: Literal["ambient"] = "ambient" + + +AwsCredentialSource = Annotated[ + StaticKeys | AssumeRole | Ambient, Field(discriminator="source") +] + + +class AwsSigV4Config(BaseModel): + """AWS SigV4 per-request signing for an AWS-hosted upstream (e.g. Bedrock AgentCore). The + gateway signs with its own AWS identity, never the caller's; `credentials` selects how that + identity is obtained, defaulting to the ambient credential chain.""" + + model_config = ConfigDict(frozen=True) + kind: Literal[AuthSpecKind.aws_sigv4] = AuthSpecKind.aws_sigv4 + region: str + service: str = "bedrock-agentcore" + credentials: AwsCredentialSource = Ambient() + + +AuthConfig = Annotated[ + AuthorizationCodeConfig + | ClientCredentialsConfig + | TokenExchangeConfig + | ApiKeyConfig + | PassthroughConfig + | NoneConfig + | AwsSigV4Config, + Field(discriminator="kind"), +] + + +class Subject(BaseModel): + """The validated inbound principal. NOT the v1 request object and NOT the LiteLLM key.""" + + model_config = ConfigDict(frozen=True) + + tenant_id: str + subject_id: str + # Opaque, already-validated inbound identity. Only `token_exchange` / `passthrough` read it. + inbound_token: SecretStr | None = None + + +class ServerSpec(BaseModel): + """The declared upstream. A v2-native type; the v1 -> v2 adapter maps onto this.""" + + model_config = ConfigDict(frozen=True) + + server_id: str + resource: str # RFC 8707 audience URI this upstream's tokens are bound to + config: AuthConfig + + @property + def auth_spec_kind(self) -> AuthSpecKind: + return self.config.kind + + +def parse_auth_spec_kind(raw: str) -> Result[AuthSpecKind, CredError]: + """Boundary parser — the *only* place an unknown mode is handled, and it fails closed. + + Inside the core the mode is always a valid `AuthSpecKind`, so the resolver never needs a + wildcard arm and basedpyright can prove its `match` exhaustive. + """ + try: + return Ok(AuthSpecKind(raw)) + except ValueError: + return Error(CredError.of_unsupported_mode(f"unknown auth_spec_kind: {raw!r}")) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 08e42e918e9..e891425274f 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -63,7 +63,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ( + ProxyException, + SpecialMCPServerNames, + UserAPIKeyAuth, +) from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, @@ -229,6 +233,28 @@ def _jsonrpc_text_has_top_level_method(text: str) -> bool: return False +def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException: + """Map a ``ProxyException`` to an ``HTTPException`` that preserves its real + status code and headers. + + ``user_api_key_auth`` raises ``ProxyException`` (not ``HTTPException``) on + auth failures. The MCP ASGI handlers re-raise ``HTTPException`` to keep the + status and any ``WWW-Authenticate`` challenge, but a ``ProxyException`` would + otherwise fall through to their generic handler and be flattened to a 500 — + dropping the 401 + challenge an OAuth client needs to re-authenticate, so the + tool call surfaces as a cancelled/terminated session instead. + """ + try: + status_code = int(exc.code) + except (TypeError, ValueError): + status_code = 500 + return HTTPException( + status_code=status_code, + detail=exc.message, + headers=exc.headers or None, + ) + + if MCP_AVAILABLE: from mcp.server import Server from mcp.server.lowlevel.server import NotificationOptions @@ -3352,6 +3378,19 @@ if MCP_AVAILABLE: from litellm.proxy._types import LiteLLM_ObjectPermissionTable from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view + # A key scoped to no MCP servers opts out of every MCP path. Enforce it + # here too, since toolset scoping replaces mcp_servers and would otherwise + # drop the sentinel. Checked before the admin branch, mirroring + # get_allowed_mcp_servers. + original_op = user_api_key_auth.object_permission + if original_op is not None and SpecialMCPServerNames.no_mcp_servers.value in ( + original_op.mcp_servers or [] + ): + raise HTTPException( + status_code=403, + detail="API key is scoped to no MCP servers; toolset access is denied.", + ) + # Access control: non-admin keys must have this toolset in their grant list. # Use _user_has_admin_view so that PROXY_ADMIN_VIEW_ONLY is also treated as admin. is_admin = _user_has_admin_view(user_api_key_auth) @@ -4006,6 +4045,12 @@ if MCP_AVAILABLE: except HTTPException: # Re-raise HTTP exceptions to preserve status codes and details raise + except ProxyException as e: + # Auth failures from user_api_key_auth arrive as ProxyException, not + # HTTPException. Preserve the real status (e.g. 401 + WWW-Authenticate) + # so OAuth clients can re-authenticate instead of receiving a generic + # 500 that surfaces as a cancelled tool call. + raise _proxy_exception_to_http_exception(e) except Exception as e: verbose_logger.exception(f"Error handling MCP request: {e}") # Try to send a graceful error response for non-HTTP exceptions @@ -4123,6 +4168,12 @@ if MCP_AVAILABLE: # Re-raise HTTP exceptions to preserve status codes and details # (e.g. 401 + WWW-Authenticate challenges from OAuth pass-through). raise + except ProxyException as e: + # Auth failures from user_api_key_auth arrive as ProxyException, not + # HTTPException. Preserve the real status (e.g. 401 + WWW-Authenticate) + # so OAuth clients can re-authenticate instead of receiving a generic + # 500 that surfaces as a cancelled tool call. + raise _proxy_exception_to_http_exception(e) except Exception as e: verbose_logger.exception(f"Error handling MCP request: {e}") # Try to send a graceful error response for non-HTTP exceptions diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b9697ee8300..5bba842c7eb 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -461,6 +461,7 @@ class LiteLLMRoutes(enum.Enum): "/mcp/tools/call", "/mcp-rest/tools/list", "/mcp-rest/tools/call", + "/v1/mcp/tools", ] # MCP server CRUD routes — control-plane. Gated by DISABLE_ADMIN_ENDPOINTS. @@ -2977,6 +2978,10 @@ class SpecialModelNames(enum.Enum): no_default_models = "no-default-models" +class SpecialMCPServerNames(enum.Enum): + no_mcp_servers = "no-mcp-servers" + + class SpecialProxyStrings(enum.Enum): default_user_id = "default_user_id" # global proxy admin diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 570b6742f57..88db2a2b7ea 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -699,6 +699,11 @@ async def common_checks( if valid_token is not None: from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route( + request_data=request_body, + route=route, + ) + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( request_data=request_body, user_api_key_dict=valid_token, @@ -3684,9 +3689,18 @@ async def _virtual_key_max_budget_check( # so a NaN max_budget would silently disable enforcement. Treat a # non-finite max_budget as "no configured limit" rather than as a bypass. if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget: + # name the key in the error so operators don't have to reverse-map + # spend back to a key; key_name is the masked form (last 4 chars) + key_label = valid_token.key_alias or "key" + key_descriptor = ( + f"{key_label} ({valid_token.key_name})" + if valid_token.key_name + else key_label + ) raise litellm.BudgetExceededError( current_cost=spend, max_budget=valid_token.max_budget, + message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}", ) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 94b2ed84f20..3a2f2221ee3 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -285,6 +285,8 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = ( "s3_endpoint_url", "sagemaker_base_url", "deployment_url", + # SDK-only field; also rejected outright in is_request_body_safe. + "model_list", # Observability credentials, hosts, and project identifiers: derived # from the canonical ``_supported_callback_params`` allowlist so new # integrations are covered automatically. Sorted for stable iteration @@ -365,6 +367,10 @@ def is_request_body_safe( ``litellm_embedding_config.api_base`` (VERIA-6) without exposing a recursion-depth DoS surface. """ + if "model_list" in request_body: + raise ValueError( + "Rejected Request: model_list is not allowed in the request body." + ) _check_banned_params(request_body, general_settings, llm_router, model) for nested_key in _NESTED_CONFIG_KEYS: nested = _coerce_metadata_to_dict(request_body.get(nested_key)) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 4a2df18b93b..e439f6a5998 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2396,6 +2396,17 @@ async def _run_centralized_common_checks( llm_router=llm_router, ) + # Pin the metadata variable name (litellm_metadata vs metadata) before + # any tag merge runs. Without this, header tags from + # apply_client_tag_policy_pre_auth would land in `metadata` while the + # later seed in common_checks pushes key tags and the + # _tag_max_budget_check read into `litellm_metadata`, hiding header + # tags from per-tag budget enforcement on LITELLM_METADATA_ROUTES. + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route( + request_data=request_data, + route=route, + ) + # Merge x-litellm-tags into request_data BEFORE common_checks runs. # _tag_max_budget_check inside common_checks only inspects request_data; # without this pre-merge, header-supplied tags bypass tag-budget diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 177fced5cd4..c0cdf84dfb6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -108,6 +108,7 @@ def parse_cache_control(cache_control): LITELLM_METADATA_ROUTES = ( "batches", + "bedrock", "/v1/messages", "responses", "files", @@ -1237,6 +1238,27 @@ class LiteLLMProxyRequestSetup: return tags + @staticmethod + def pre_seed_litellm_metadata_for_route( + request_data: dict, + route: str, + ) -> None: + """Pre-seed ``litellm_metadata`` for routes that track tags there. + + Routes in ``LITELLM_METADATA_ROUTES`` (e.g. Bedrock, ``/v1/messages``, + responses, batches, files) store request-scoped tag metadata in + ``litellm_metadata`` rather than the provider-facing ``metadata`` + field. ``get_metadata_variable_name_from_kwargs`` picks the target + based on whether ``litellm_metadata`` is present, so it must be + seeded BEFORE any tag merge runs; otherwise header tags from + ``apply_client_tag_policy_pre_auth`` land in ``metadata`` while + key tags from ``apply_key_tags_pre_auth`` and the read in + ``_tag_max_budget_check`` resolve to ``litellm_metadata``, leaving + header tags invisible to per-tag budget enforcement. + """ + if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES): + request_data.setdefault("litellm_metadata", {}) + @staticmethod def apply_key_tags_pre_auth( request_data: dict, @@ -1468,8 +1490,7 @@ async def add_litellm_data_to_request( _metadata_variable_name=_metadata_variable_name, ) - # Add headers to metadata for guardrails to access (fixes #17477) - # Guardrails use metadata["headers"] to access request headers (e.g., User-Agent) + # Expose request headers under the metadata field for guardrails (fixes #17477) if _metadata_variable_name in data and isinstance( data[_metadata_variable_name], dict ): diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 341a8767db0..79882909c23 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1,7 +1,7 @@ import asyncio from datetime import datetime from types import SimpleNamespace -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union from fastapi import HTTPException, status @@ -887,8 +887,17 @@ async def get_daily_activity( exclude_entity_ids: Optional[List[str]] = None, metadata_metrics_func: Optional[Callable[[List[Any]], SpendMetrics]] = None, timezone_offset_minutes: Optional[int] = None, + resolve_entity_metadata: Optional[ + Callable[[list[Any]], Awaitable[dict[str, dict]]] + ] = None, ) -> SpendAnalyticsPaginatedResponse: - """Common function to get daily activity for any entity type.""" + """Common function to get daily activity for any entity type. + + ``resolve_entity_metadata`` lets a caller resolve entity metadata from the + rows actually on the page (e.g. user_id -> user_email) instead of fetching + the whole entity table upfront, which matters when the entity set is + unbounded. + """ if prisma_client is None: raise HTTPException( @@ -939,11 +948,18 @@ async def get_daily_activity( take=page_size, ) + resolved_entity_metadata = entity_metadata_field + if resolve_entity_metadata is not None: + resolved_entity_metadata = { + **(entity_metadata_field or {}), + **(await resolve_entity_metadata(daily_spend_data)), + } + aggregated = await _aggregate_spend_records( prisma_client=prisma_client, records=daily_spend_data, entity_id_field=entity_id_field, - entity_metadata_field=entity_metadata_field, + entity_metadata_field=resolved_entity_metadata, ) metadata_metrics = aggregated["totals"] diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index ba7013570fe..6d7f565fb85 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -57,6 +57,9 @@ from litellm.repositories.verification_token_repository import ( from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) +from litellm.types.proxy.management_endpoints.scim_v2 import ( + SCIM_ENTERPRISE_METADATA_KEY, +) from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( BulkUpdateUserRequest, BulkUpdateUserResponse, @@ -719,6 +722,17 @@ async def _get_user_info_teams( return team_list, teams_1 +def _redact_scim_enterprise_metadata( + metadata: Optional[Dict[str, Any]], +) -> Optional[Dict[str, Any]]: + """SCIM enterprise attributes are persisted in user metadata so reporting can + group on them, but they are directory-only fields that generic user-info + endpoints must not surface; SCIM clients read them through the SCIM endpoints.""" + if not isinstance(metadata, dict) or SCIM_ENTERPRISE_METADATA_KEY not in metadata: + return metadata + return {k: v for k, v in metadata.items() if k != SCIM_ENTERPRISE_METADATA_KEY} + + def _build_user_info_response( user_id: Optional[str], user_info: Optional[Any], @@ -739,6 +753,9 @@ def _build_user_info_response( ) if isinstance(_user_info, dict): _user_info.pop("password", None) + _user_info["metadata"] = _redact_scim_enterprise_metadata( + _user_info.get("metadata") + ) return UserInfoResponse( user_id=user_id, @@ -983,7 +1000,7 @@ async def user_info_v2( models=user_data.get("models") or [], budget_duration=user_data.get("budget_duration"), budget_reset_at=user_data.get("budget_reset_at"), - metadata=user_data.get("metadata"), + metadata=_redact_scim_enterprise_metadata(user_data.get("metadata")), created_at=user_data.get("created_at"), updated_at=user_data.get("updated_at"), sso_user_id=user_data.get("sso_user_id"), @@ -2098,9 +2115,13 @@ async def get_users( user_list: List[LiteLLM_UserTableWithKeyCount] = [] if users is not None: for user in users: + user_dump = user.model_dump() + user_dump["metadata"] = _redact_scim_enterprise_metadata( + user_dump.get("metadata") + ) user_list.append( LiteLLM_UserTableWithKeyCount( - **user.model_dump(), key_count=user_key_counts.get(user.user_id, 0) + **user_dump, key_count=user_key_counts.get(user.user_id, 0) ) ) else: @@ -2596,6 +2617,25 @@ async def ui_view_users( # Using shared metric helper implementations from common_daily_activity +async def _resolve_user_email_metadata( + prisma_client: "PrismaClient", records: list[Any] +) -> dict[str, dict]: + """Map each user_id on the page to its email/alias so the Usage dashboard can + label the 'Spend Per User' chart with the email instead of the raw UUID.""" + user_ids = { + record.user_id for record in records if getattr(record, "user_id", None) + } + if not user_ids: + return {} + users = await UserRepository(prisma_client).table.find_many( + where={"user_id": {"in": list(user_ids)}} + ) + return { + user.user_id: {"user_email": user.user_email, "user_alias": user.user_alias} + for user in users + } + + @router.get( "/user/daily/activity", tags=["Budget & Spend Tracking", "Internal User management"], @@ -2698,6 +2738,9 @@ async def get_user_daily_activity( page=page, page_size=page_size, timezone_offset_minutes=timezone, + resolve_entity_metadata=lambda records: _resolve_user_email_metadata( + prisma_client, records + ), ) except HTTPException: diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index e86982307e7..f896047a219 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -1016,15 +1016,10 @@ if MCP_AVAILABLE: if is_restricted_virtual_key: return _sanitize_mcp_server_list_for_virtual_key(redacted_mcp_servers) - # Non-admin authenticated users may see the server inventory but - # not credential-bearing fields like `url` (often contains bearer - # tokens) or headers/env (often contain Authorization). - if not _user_has_admin_view(user_api_key_dict): - return _sanitize_mcp_server_list_for_non_admin(redacted_mcp_servers) - + # only a full PROXY_ADMIN sees credential-bearing fields; everyone else + # goes through the non-admin sanitizer if not _user_is_full_admin(user_api_key_dict): - for server in redacted_mcp_servers: - _redact_global_env_var_values(server) + return _sanitize_mcp_server_list_for_non_admin(redacted_mcp_servers) return redacted_mcp_servers @@ -1415,10 +1410,10 @@ if MCP_AVAILABLE: redacted = _redact_mcp_credentials(mcp_server) if is_restricted_virtual_key: return _sanitize_mcp_server_for_virtual_key(redacted) - if not _user_has_admin_view(user_api_key_dict): - return _sanitize_mcp_server_for_non_admin(redacted) + # only a full PROXY_ADMIN sees credential-bearing fields; everyone else + # goes through the non-admin sanitizer if not _user_is_full_admin(user_api_key_dict): - _redact_global_env_var_values(redacted) + return _sanitize_mcp_server_for_non_admin(redacted) return redacted @router.post( @@ -1935,12 +1930,9 @@ if MCP_AVAILABLE: prisma_client = get_prisma_client_or_throw( "Database not connected. Connect a database to your proxy" ) - mcp_server = await get_mcp_server(prisma_client, server_id) - if mcp_server is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={"error": f"MCP Server {server_id} not found"}, - ) + mcp_server = await _authorize_and_fetch_mcp_server( + prisma_client, user_api_key_dict, server_id + ) if not getattr(mcp_server, "is_byok", False): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -2015,12 +2007,9 @@ if MCP_AVAILABLE: prisma_client = get_prisma_client_or_throw( "Database not connected. Connect a database to your proxy" ) - mcp_server = await get_mcp_server(prisma_client, server_id) - if mcp_server is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={"error": f"MCP Server {server_id} not found"}, - ) + await _authorize_and_fetch_mcp_server( + prisma_client, user_api_key_dict, server_id + ) user_id = user_api_key_dict.user_id or "" if not user_id: raise HTTPException( @@ -2182,37 +2171,47 @@ if MCP_AVAILABLE: user_api_key_dict: UserAPIKeyAuth, server_id: str, ) -> LiteLLM_MCPServerTable: - """Return the MCP server the caller may manage env vars for. + """Resolve the MCP server a caller may manage their own per-user state for. - Admins look the server up directly. Non-admins reuse the access-scoped - listing that already loads every server they can see, so we don't issue - a second per-server query just to re-fetch a record the authorization - check produced. A non-admin who can't see the server gets 403 (never - 404) so server ids can't be enumerated. + Looks the server up in the DB, then the in-memory registry, so a + config-defined server (which never gets a DB row) resolves too. Admins + may reach any server and get a 404 for an unknown id. A non-admin may + only reach a server in their allowed set and otherwise gets 403 (never + 404, so server ids can't be enumerated), using the same allowed-server + resolution the MCP gateway enforces on tool calls. """ + server = await get_mcp_server(prisma_client, server_id) + if server is None: + registry_server = global_mcp_server_manager.get_mcp_server_by_id(server_id) + if registry_server is not None: + server = global_mcp_server_manager._build_mcp_server_table( + registry_server + ) + if _user_has_admin_view(user_api_key_dict): - server = await get_mcp_server(prisma_client, server_id) if server is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail={"error": f"MCP Server {server_id} not found"}, ) return server - accessible = await get_all_mcp_servers_for_user( - prisma_client, user_api_key_dict - ) - for server in accessible: - if server.server_id == server_id: - return server - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail={ - "error": ( - f"User does not have permission to access mcp server with id {server_id}. " - "You can only manage env vars for mcp servers that you have access to." - ) - }, - ) + + allowed_server_ids: set[str] = set() + for auth_context in await build_effective_auth_contexts(user_api_key_dict): + allowed_server_ids.update( + await global_mcp_server_manager.get_allowed_mcp_servers(auth_context) + ) + if server is None or server.server_id not in allowed_server_ids: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": ( + f"User does not have permission to access mcp server with id {server_id}. " + "You can only manage mcp servers that you have access to." + ) + }, + ) + return server def _compute_user_env_var_status( *, diff --git a/litellm/proxy/management_endpoints/scim/scim_transformations.py b/litellm/proxy/management_endpoints/scim/scim_transformations.py index d1e00f87b69..866f5baf3a4 100644 --- a/litellm/proxy/management_endpoints/scim/scim_transformations.py +++ b/litellm/proxy/management_endpoints/scim/scim_transformations.py @@ -50,8 +50,16 @@ class ScimTransformations: scim_active = metadata.get("scim_active") active = True if scim_active is None else bool(scim_active) + schemas = ["urn:ietf:params:scim:schemas:core:2.0:User"] + enterprise_user = None + if metadata.get(SCIM_ENTERPRISE_METADATA_KEY): + enterprise_user = SCIMEnterpriseUser.model_validate( + metadata[SCIM_ENTERPRISE_METADATA_KEY] + ) + schemas.append(SCIM_ENTERPRISE_USER_SCHEMA) + return SCIMUser( - schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + schemas=schemas, id=user.user_id, userName=ScimTransformations._get_scim_user_name(user), displayName=ScimTransformations._get_scim_user_name(user), @@ -62,6 +70,7 @@ class ScimTransformations: emails=emails, groups=groups, active=active, + enterprise_user=enterprise_user, meta={ "resourceType": "User", "created": user_created_at, diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 0798d1a510d..d5da0372a8f 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -5,7 +5,7 @@ This is an enterprise feature and requires a premium license. """ import re -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from fastapi import ( APIRouter, @@ -69,14 +69,21 @@ class UserProvisionerHelpers: @staticmethod async def handle_existing_user_by_email( - prisma_client, new_user_request: NewUserRequest + prisma_client, + new_user_request: NewUserRequest, + admin_group: Optional[str] = None, ) -> Optional[SCIMUser]: """ Check if a user with the given email already exists and update them if found. + When admin_group is configured the resolved global role on new_user_request + is persisted too, so re-upserting an existing email demotes a user who is no + longer in the admin group instead of leaving the stale role. + Args: prisma_client: Database client new_user_request: New user request data + admin_group: Configured SCIM admin group, or None to leave role untouched Returns: SCIMUser if user was updated, None if no existing user found @@ -100,6 +107,11 @@ class UserProvisionerHelpers: "user_alias": new_user_request.user_alias, "teams": new_user_request.teams, "metadata": safe_dumps(new_user_request.metadata), + **( + {"user_role": new_user_request.user_role} + if admin_group is not None + else {} + ), }, ) @@ -118,6 +130,7 @@ class ScimUserData(TypedDict): given_name: Optional[str] family_name: Optional[str] active: Optional[bool] + enterprise: Optional[SCIMEnterpriseUser] class GroupMemberExtractionResult(BaseModel): @@ -199,11 +212,15 @@ def _extract_scim_user_data(user: SCIMUser) -> ScimUserData: "given_name": user.name.givenName if user.name else None, "family_name": user.name.familyName if user.name else None, "active": user.active, + "enterprise": user.enterprise_user, } def _build_scim_metadata( - given_name: Optional[str], family_name: Optional[str], active: Optional[bool] = None + given_name: Optional[str], + family_name: Optional[str], + active: Optional[bool] = None, + enterprise: Optional[SCIMEnterpriseUser] = None, ) -> Dict[str, Any]: """Build metadata dictionary with SCIM data.""" metadata: Dict[str, Any] = { @@ -216,6 +233,11 @@ def _build_scim_metadata( if active is not None: metadata["scim_active"] = active + if enterprise is not None: + metadata[SCIM_ENTERPRISE_METADATA_KEY] = enterprise.model_dump( + by_alias=True, exclude_none=True + ) + return metadata @@ -244,6 +266,117 @@ async def _get_scim_upsert_user_setting() -> bool: return True +ScimUserRole = Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, +] + + +def _default_scim_user_role() -> ScimUserRole: + """Non-admin default role for SCIM-provisioned users.""" + if litellm.default_internal_user_params: + configured_role = litellm.default_internal_user_params.get("user_role") + if configured_role is not None: + return configured_role + return LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +async def _get_scim_admin_group() -> Optional[str]: + """ + Get the scim_admin_group setting from litellm_settings. + + Returns the configured admin group identifier, or None when unset so callers + leave a user's global role untouched (default-safe). + """ + try: + from litellm.proxy.proxy_server import proxy_config + + config = await proxy_config.get_config() + litellm_settings = config.get("litellm_settings", {}) or {} + return litellm_settings.get("scim_admin_group") or None + except Exception as e: + verbose_proxy_logger.warning( + f"Error reading scim_admin_group setting, defaulting to None: {e}" + ) + return None + + +def _resolve_scim_user_role( + groups: list[SCIMUserGroup], + admin_group: Optional[str], + default_role: ScimUserRole, +) -> Optional[LitellmUserRoles]: + """ + Resolve a user's global proxy role from their SCIM groups. + + Returns None when no admin group is configured, signalling callers to leave + the role unchanged. Otherwise grants PROXY_ADMIN when any group matches the + admin group by value or display, and falls back to the non-admin default. + """ + if admin_group is None: + return None + for group in groups: + if group.value == admin_group or group.display == admin_group: + return LitellmUserRoles.PROXY_ADMIN + return default_role + + +async def _scim_groups_from_team_ids( + prisma_client: Any, team_ids: list[str] +) -> list[SCIMUserGroup]: + """ + Build SCIMUserGroup objects from team ids, populating display from each + team's alias so admin-group matching by display name works the same way it + does on PUT (where SCIM groups carry display names natively). + """ + teams = [ + await TeamRepository(prisma_client).table.find_unique( + where={"team_id": team_id} + ) + for team_id in team_ids + ] + return [ + SCIMUserGroup( + value=team_id, + display=team.team_alias if team is not None else None, + ) + for team_id, team in zip(team_ids, teams) + ] + + +async def _recompute_scim_member_roles( + prisma_client: Any, user_ids: Iterable[str] +) -> None: + """ + Recompute and persist each user's global proxy role from their resulting team + membership. No-op unless scim_admin_group is configured, so a SCIM group write + that drops a member from the admin group demotes them just like the user + endpoints do, and the role is left untouched when the feature is off. + """ + admin_group = await _get_scim_admin_group() + if admin_group is None: + return + + default_role = _default_scim_user_role() + for user_id in user_ids: + user = await UserRepository(prisma_client).table.find_unique( + where={"user_id": user_id} + ) + if user is None: + continue + resolved_role = _resolve_scim_user_role( + await _scim_groups_from_team_ids(prisma_client, user.teams or []), + admin_group, + default_role, + ) + await UserRepository(prisma_client).table.update( + where={"user_id": user_id}, + data={"user_role": resolved_role}, + ) + + async def _extract_group_member_ids(group: SCIMGroup) -> GroupMemberExtractionResult: """ Extract member IDs from SCIMGroup, validating that all users exist. @@ -999,19 +1132,16 @@ async def create_user( # Create user in database user_id = user.userName or str(uuid.uuid4()) metadata = _build_scim_metadata( - user_data["given_name"], user_data["family_name"] + user_data["given_name"], + user_data["family_name"], + enterprise=user_data["enterprise"], ) - default_role: Optional[ - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY - if litellm.default_internal_user_params: - default_role = litellm.default_internal_user_params.get("user_role") + default_role = _default_scim_user_role() + admin_group = await _get_scim_admin_group() + resolved_role = _resolve_scim_user_role( + user.groups or [], admin_group, default_role + ) new_user_request = NewUserRequest( user_id=user_id, @@ -1020,12 +1150,14 @@ async def create_user( teams=user_data["teams"], metadata=metadata, auto_create_key=False, - user_role=default_role, + user_role=resolved_role if admin_group is not None else default_role, ) # Check if user with email already exists and update if found existing_user_scim = await UserProvisionerHelpers.handle_existing_user_by_email( - prisma_client=prisma_client, new_user_request=new_user_request + prisma_client=prisma_client, + new_user_request=new_user_request, + admin_group=admin_group, ) if existing_user_scim: @@ -1088,6 +1220,7 @@ async def update_user( user_data["given_name"], user_data["family_name"], scim_active_for_metadata, + enterprise=user_data["enterprise"], ) await _handle_team_membership_changes( @@ -1104,6 +1237,12 @@ async def update_user( "metadata": safe_dumps(metadata), } + admin_group = await _get_scim_admin_group() + if admin_group is not None: + update_data["user_role"] = _resolve_scim_user_role( + user.groups or [], admin_group, _default_scim_user_role() + ) + updated_user = await UserRepository(prisma_client).table.update( where={"user_id": user_id}, data=update_data, @@ -1417,6 +1556,14 @@ async def patch_user( update_data["teams"] = list(final_team_set) + admin_group = await _get_scim_admin_group() + if admin_group is not None: + update_data["user_role"] = _resolve_scim_user_role( + await _scim_groups_from_team_ids(prisma_client, list(final_team_set)), + admin_group, + _default_scim_user_role(), + ) + # Serialize metadata to JSON string for Prisma to avoid GraphQL parsing issues if "metadata" in update_data and isinstance(update_data["metadata"], dict): from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -1599,6 +1746,8 @@ async def create_group( user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), ) + await _recompute_scim_member_roles(prisma_client, member_result.all_member_ids) + scim_group = await ScimTransformations.transform_litellm_team_to_scim_group( created_team ) @@ -1665,6 +1814,19 @@ async def update_group( final_members=final_members, ) + # A rename can flip whether this group matches scim_admin_group by display + # name, so retained members must be re-resolved too, not just the ones whose + # membership changed. + alias_changed = existing_team.team_alias != group.displayName + await _recompute_scim_member_roles( + prisma_client, + ( + current_members | final_members + if alias_changed + else current_members ^ final_members + ), + ) + # Convert to SCIM format and return scim_group = await ScimTransformations.transform_litellm_team_to_scim_group( updated_team @@ -1691,8 +1853,10 @@ async def delete_group( prisma_client = await _get_prisma_client_or_raise_exception() existing_team = await _check_team_exists(group_id) + member_ids = await _get_team_member_user_ids_from_team(existing_team) + # For each member, remove this team from their teams list - for member_id in existing_team.members or []: + for member_id in member_ids: user = await UserRepository(prisma_client).table.find_unique( where={"user_id": member_id} ) @@ -1704,6 +1868,8 @@ async def delete_group( where={"user_id": member_id}, data={"teams": new_teams} ) + await _recompute_scim_member_roles(prisma_client, member_ids) + # Delete team await TeamRepository(prisma_client).table.delete(where={"team_id": group_id}) @@ -1903,6 +2069,20 @@ async def patch_group( # Handle user-team relationship changes await _handle_group_membership_changes(group_id, current_members, final_members) + # A rename can flip whether this group matches scim_admin_group by display + # name, so retained members must be re-resolved too, not just the ones whose + # membership changed. + new_alias = update_data.get("team_alias", existing_team.team_alias) + alias_changed = new_alias != existing_team.team_alias + await _recompute_scim_member_roles( + prisma_client, + ( + current_members | final_members + if alias_changed + else current_members ^ final_members + ), + ) + # Refresh team one more time to get final state after membership changes final_team = await TeamRepository(prisma_client).table.find_unique( where={"team_id": group_id} diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index f2ddae40d8c..07c355f2cd9 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -11,6 +11,7 @@ from fastapi import HTTPException, status from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.proxy._types import SpecialMCPServerNames from litellm.proxy.utils import PrismaClient from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.table_repositories import MCPServerRepository @@ -287,6 +288,9 @@ def _rewrite_object_permission_mcp_servers( normalized_servers: List[str] = [] for identifier in mcp_servers: + if identifier == SpecialMCPServerNames.no_mcp_servers.value: + normalized_servers.append(SpecialMCPServerNames.no_mcp_servers.value) + continue normalized_servers.extend(sorted(identifier_to_server_ids.get(identifier, []))) object_permission["mcp_servers"] = _dedupe_preserving_order(normalized_servers) @@ -426,6 +430,7 @@ def _extract_requested_mcp_server_ids( mcp_servers = object_permission.get("mcp_servers") if isinstance(mcp_servers, list): server_ids.update(mcp_servers) + server_ids.discard(SpecialMCPServerNames.no_mcp_servers.value) mcp_tool_permissions = object_permission.get("mcp_tool_permissions") if isinstance(mcp_tool_permissions, dict): diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c8f6749a196..8986166ba92 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -15,12 +16,19 @@ from litellm.llms.anthropic import get_anthropic_config from litellm.llms.anthropic.chat.handler import ( ModelResponseIterator as AnthropicModelResponseIterator, ) +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) -from litellm.types.utils import LiteLLMBatch, ModelResponse, TextCompletionResponse +from litellm.types.utils import ( + Choices, + LiteLLMBatch, + Message, + ModelResponse, + TextCompletionResponse, +) if TYPE_CHECKING: from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType @@ -272,6 +280,9 @@ class AnthropicPassthroughLoggingHandler: kwargs["response_cost"] = response_cost kwargs["model"] = model + # the pass-through success path reads spend from + # model_call_details["response_cost"], not from kwargs + logging_obj.model_call_details["response_cost"] = response_cost passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = ( # type: ignore kwargs.get("passthrough_logging_payload") ) @@ -343,13 +354,42 @@ class AnthropicPassthroughLoggingHandler: if chunk_model: model = chunk_model - complete_streaming_response = ( - AnthropicPassthroughLoggingHandler._build_complete_streaming_response( - all_chunks=all_chunks, - litellm_logging_obj=litellm_logging_obj, - model=model, + try: + complete_streaming_response = ( + AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=all_chunks, + litellm_logging_obj=litellm_logging_obj, + model=model, + ) ) - ) + except Exception as e: + # stream_chunk_builder re-raises assembly failures (as litellm.APIError) + # on large agentic tool-use / thinking streams; treat that the same as a + # None result so the usage-only fallback below still recovers cost + verbose_proxy_logger.warning( + "Anthropic passthrough: stream assembly raised (model=%s): %s; falling " + "back to usage-only cost from raw SSE events.", + model, + e, + ) + complete_streaming_response = None + if complete_streaming_response is None: + # stream_chunk_builder cannot always reassemble large agentic streams, but + # Anthropic still emits token usage in the message_start / message_delta SSE + # events regardless of content shape; recover usage-only so cost is tracked. + # Guard it too: a raise here would defeat the point and drop the request + try: + complete_streaming_response = AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=all_chunks, + model=model, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Anthropic passthrough: usage-only fallback failed (model=%s): %s", + model, + e, + ) + complete_streaming_response = None if complete_streaming_response is None: verbose_proxy_logger.error( "Unable to build complete streaming response for Anthropic passthrough endpoint, not logging..." @@ -636,6 +676,141 @@ class AnthropicPassthroughLoggingHandler: ) return complete_streaming_response + @staticmethod + def _extract_sse_data(event_str: str) -> Optional[dict]: + """Parse the JSON object from the ``data:`` line of an Anthropic SSE event.""" + for line in event_str.splitlines(): + stripped = line.strip() + if stripped.startswith("data:"): + payload = stripped[len("data:") :].strip() + if not payload or payload == "[DONE]": + return None + try: + return cast(dict, json.loads(payload)) + except (ValueError, TypeError): + return None + return None + + @staticmethod + def _build_usage_only_response_from_chunks( + all_chunks: Sequence[Union[str, bytes]], + model: str, + ) -> Optional[ModelResponse]: + """ + Build a usage-bearing ModelResponse from Anthropic SSE token-usage events, for + cost tracking when stream_chunk_builder cannot reassemble the stream. + + Anthropic emits usage in ``message_start`` (uncached input + cache tokens, and an + initial output_tokens) and the final ``message_delta`` (cumulative output_tokens) + regardless of the content/tool shape, so cost is recoverable even when full + content assembly fails. Returns ``None`` if no usage event is found. + """ + input_tokens = 0 + cache_read = 0 + cache_creation = 0 + cache_creation_5m: Optional[int] = None + cache_creation_1h: Optional[int] = None + output_tokens = 0 + web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None + inference_geo: Optional[str] = None + stop_reason: Optional[str] = None + found_usage = False + resolved_model = model + for _chunk_str in all_chunks: + for ( + event_str + ) in AnthropicPassthroughLoggingHandler._split_sse_chunk_into_events( + _chunk_str + ): + data = AnthropicPassthroughLoggingHandler._extract_sse_data(event_str) + if not data: + continue + event_type = data.get("type") + if event_type == "message_start": + message = data.get("message") or {} + if not resolved_model or resolved_model == "unknown": + resolved_model = message.get("model") or resolved_model + usage = message.get("usage") or {} + input_tokens = usage.get("input_tokens") or input_tokens + cache_read = usage.get("cache_read_input_tokens") or cache_read + cache_creation = ( + usage.get("cache_creation_input_tokens") or cache_creation + ) + _cc = usage.get("cache_creation") + if isinstance(_cc, dict): + cache_creation_5m = _cc.get("ephemeral_5m_input_tokens") + cache_creation_1h = _cc.get("ephemeral_1h_input_tokens") + if usage.get("inference_geo") is not None: + inference_geo = usage.get("inference_geo") + if usage.get("output_tokens") is not None: + output_tokens = usage.get("output_tokens") + found_usage = True + elif event_type == "message_delta": + _delta_stop = (data.get("delta") or {}).get("stop_reason") + if _delta_stop: + stop_reason = _delta_stop + usage = data.get("usage") or {} + if usage.get("output_tokens") is not None: + output_tokens = usage.get("output_tokens") + _stu = usage.get("server_tool_use") + if isinstance(_stu, dict): + if _stu.get("web_search_requests") is not None: + web_search_requests = _stu.get("web_search_requests") + if _stu.get("tool_search_requests") is not None: + tool_search_requests = _stu.get("tool_search_requests") + if usage.get("cache_read_input_tokens") is not None: + cache_read = usage.get("cache_read_input_tokens") + if usage.get("inference_geo") is not None: + inference_geo = usage.get("inference_geo") + found_usage = True + if not found_usage: + return None + # If only the 5m/1h split was provided, derive the cache_creation total from it. + if not cache_creation and (cache_creation_5m or cache_creation_1h): + cache_creation = (cache_creation_5m or 0) + (cache_creation_1h or 0) + # build usage via the same AnthropicConfig.calculate_usage path the success + # cases use, so prompt_tokens are cache-inclusive and cache / server_tool_use / + # inference_geo tokens are priced instead of left at $0 + usage_object: dict = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + if cache_read: + usage_object["cache_read_input_tokens"] = cache_read + if cache_creation: + usage_object["cache_creation_input_tokens"] = cache_creation + if cache_creation_5m is not None or cache_creation_1h is not None: + usage_object["cache_creation"] = { + "ephemeral_5m_input_tokens": cache_creation_5m or 0, + "ephemeral_1h_input_tokens": cache_creation_1h or 0, + } + if web_search_requests is not None or tool_search_requests is not None: + _server_tool_use: dict = {} + if web_search_requests is not None: + _server_tool_use["web_search_requests"] = web_search_requests + if tool_search_requests is not None: + _server_tool_use["tool_search_requests"] = tool_search_requests + usage_object["server_tool_use"] = _server_tool_use + if inference_geo is not None: + usage_object["inference_geo"] = inference_geo + usage_obj = AnthropicConfig().calculate_usage( + usage_object=usage_object, reasoning_content=None + ) + return ModelResponse( + model=resolved_model, + choices=[ + Choices( + finish_reason=( + map_finish_reason(stop_reason) if stop_reason else "stop" + ), + index=0, + message=Message(role="assistant", content=""), + ) + ], + usage=usage_obj, + ) + @staticmethod def batch_creation_handler( httpx_response: httpx.Response, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py index b9df8ecede3..a7ec2f0d368 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py @@ -116,6 +116,9 @@ class BasePassthroughLoggingHandler(ABC): kwargs["response_cost"] = response_cost kwargs["model"] = model + # the pass-through success path reads spend from + # model_call_details["response_cost"], not from kwargs + logging_obj.model_call_details["response_cost"] = response_cost passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = ( # type: ignore kwargs.get("passthrough_logging_payload") ) diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 33a6b719280..7a725472dd7 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -285,8 +285,10 @@ class PassThroughStreamingHandler: Returns: List of string lines, with each line being a complete data: {} chunk """ - # Combine all bytes and decode to string - combined_str = b"".join(raw_bytes).decode("utf-8") + # errors="replace" so a stream cut mid-multibyte-sequence (client disconnect) + # still decodes and logs the usage events already received, instead of raising + # and dropping the whole request from SpendLogs + combined_str = b"".join(raw_bytes).decode("utf-8", errors="replace") # Split by newlines and filter out empty lines lines = [line.strip() for line in combined_str.split("\n") if line.strip()] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7dd7e6b0a93..4c36b42615e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -38,7 +38,7 @@ from typing import ( import anyio import websockets import websockets.exceptions -from pydantic import BaseModel, Json +from pydantic import BaseModel, Json, JsonValue from litellm._uuid import uuid from litellm.constants import ( @@ -4463,6 +4463,8 @@ class ProxyConfig: f"{blue_color_code} setting litellm.{key}={value}{reset_color_code}" ) setattr(litellm, key, value) + if key == "request_timeout": + litellm.request_timeout_explicitly_set = True if key in {"s3_audit_callback_params", "s3_callback_params"}: from litellm.integrations.s3_v2 import S3Logger as S3V2Logger from litellm.litellm_core_utils.litellm_logging import ( @@ -11498,18 +11500,20 @@ def get_direct_access_models( llm_router: Router, ) -> List[str]: """ - Get all models that user has direct access to - """ + Get all models that user has direct access to. - direct_access_models: List[str] = [] - for model in user_db_object.models: - deployments = llm_router.get_model_list(model_name=model) - if deployments is not None: - for deployment in deployments: - model_id = deployment.get("model_info", {}).get("id", None) - if model_id is not None: - direct_access_models.append(model_id) - return direct_access_models + The 'all-proxy-models' sentinel grants direct access to every non-team + deployment, mirroring how get_key_models expands it for the key/team path. + """ + if SpecialModelNames.all_proxy_models.value in user_db_object.models: + return llm_router.get_model_ids(exclude_team_models=True) + + return [ + model_id + for model in user_db_object.models + for deployment in (llm_router.get_model_list(model_name=model) or []) + if (model_id := deployment.get("model_info", {}).get("id", None)) is not None + ] def _filter_models_to_user_accessible(all_models: List[Dict]) -> List[Dict]: @@ -11689,9 +11693,12 @@ async def _get_caller_byok_team_scope( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ): return None + key_team_scope: set[str] = ( + {user_api_key_dict.team_id} if user_api_key_dict.team_id else set() + ) user_id = user_api_key_dict.user_id if user_id is None: - return set() + return key_team_scope try: user_row = await UserRepository(prisma_client).table.find_unique( where={"user_id": user_id} @@ -11699,12 +11706,12 @@ async def _get_caller_byok_team_scope( except Exception: verbose_proxy_logger.exception( "Failed to look up caller teams while scoping BYOK search; " - "defaulting to no team access." + "defaulting to key team scope only." ) - return set() + return key_team_scope if user_row is None: - return set() - return set(user_row.teams or []) + return key_team_scope + return key_team_scope | set(user_row.teams or []) def _byok_row_outside_caller_teams( @@ -15178,6 +15185,68 @@ async def update_config_general_settings( return response +# Secret-bearing general_settings fields the segment masker does not match by +# name: database_url and database_extra_connection_params embed DB credentials, +# pass_through_endpoints carry upstream Authorization headers, and +# alert_to_webhook_url is itself a webhook secret +_EXTRA_SECRET_GENERAL_SETTINGS_FIELDS = frozenset( + { + "database_url", + "database_extra_connection_params", + "pass_through_endpoints", + "alert_to_webhook_url", + } +) + + +def _is_secret_general_setting_field(field_name: str) -> bool: + return ( + field_name in _EXTRA_SECRET_GENERAL_SETTINGS_FIELDS + or SENSITIVE_DATA_MASKER.is_sensitive_key(field_name) + ) + + +# Matches the cap on _redact_sensitive_litellm_params (the closest analog in the +# proxy). Past this depth we fail closed by returning "REDACTED" for the whole +# subtree rather than recursing further — better to over-redact a pathological +# config than to silently return a deeply-nested credential verbatim +_REDACT_SECRET_MAX_DEPTH = 10 + + +def _redact_secret_values_in_obj(value: JsonValue, depth: int = 0) -> JsonValue: + """Recursively redact secret leaves inside a structured field so a nested + credential (e.g. aws_web_identity_token under database_args) is never + returned to a non-admin, while non-secret siblings stay visible. At + _REDACT_SECRET_MAX_DEPTH the whole subtree is replaced with "REDACTED" + so depth-overrun fails closed.""" + if depth >= _REDACT_SECRET_MAX_DEPTH: + return "REDACTED" + if isinstance(value, dict): + return { + key: ( + "REDACTED" + if _is_secret_general_setting_field(key) + else _redact_secret_values_in_obj(sub, depth + 1) + ) + for key, sub in value.items() + } + if isinstance(value, list): + return [_redact_secret_values_in_obj(item, depth + 1) for item in value] + return value + + +def _redact_general_setting_value( + field_name: str, value: JsonValue, is_full_admin: bool +) -> JsonValue: + if is_full_admin: + return value + if _is_secret_general_setting_field(field_name): + return "REDACTED" + if isinstance(value, (dict, list)): + return _redact_secret_values_in_obj(value) + return value + + @router.get( "/config/field/info", tags=["config.yaml"], @@ -15230,9 +15299,11 @@ async def get_config_general_settings( general_settings = dict(db_general_settings.param_value) if field_name in general_settings: - field_value = general_settings[field_name] - # Redact plugin_key from plugin configs so the shared credential - # is never returned even to admin-viewer callers. + field_value = _redact_general_setting_value( + field_name, + general_settings[field_name], + user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN, + ) if field_name == "plugins" and isinstance(field_value, list): field_value = [ ( @@ -15288,6 +15359,8 @@ async def get_config_list( }, ) + is_full_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + ## get general settings from db db_general_settings = await ConfigRepository(prisma_client).table.find_first( where={"param_name": "general_settings"} @@ -15336,7 +15409,11 @@ async def get_config_list( field_name=sub_field, field_type=sub_field_type.__name__, field_description="", # Add custom logic if descriptions are available - field_default_value=general_settings.get(sub_field, None), + field_default_value=_redact_general_setting_value( + sub_field, + general_settings.get(sub_field, None), + is_full_admin, + ), stored_in_db=None, ) for sub_field, sub_field_type in pydantic_class.__annotations__.items() @@ -15366,7 +15443,11 @@ async def get_config_list( field_name=field_name, field_type=allowed_args[field_name]["type"], field_description=field_info.description or "", - field_value=general_settings.get(field_name, None), + field_value=_redact_general_setting_value( + field_name, + general_settings.get(field_name, None), + is_full_admin, + ), stored_in_db=_stored_in_db, field_default_value=field_info.default, nested_fields=nested_fields, @@ -15390,7 +15471,9 @@ async def get_config_list( field_name=field_name, field_type=allowed_args[field_name]["type"], field_description=field_info.description or "", - field_value=_field_value, + field_value=_redact_general_setting_value( + field_name, _field_value, is_full_admin + ), stored_in_db=_stored_in_db, field_default_value=field_info.default, nested_fields=nested_fields, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index fac732bac68..fcc6aac1c14 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -209,6 +209,114 @@ ], "default_model_placeholder": "claude-3-opus" }, + { + "provider": "BedrockMantle", + "provider_display_name": "Amazon Bedrock Mantle", + "litellm_provider": "bedrock_mantle", + "credential_fields": [ + { + "key": "api_key", + "label": "Bedrock Mantle API Key", + "placeholder": null, + "tooltip": "Bearer token for the Bedrock Mantle OpenAI-compatible endpoint. You can provide the raw token or the environment variable (e.g. `os.environ/BEDROCK_MANTLE_API_KEY`). Leave blank to authenticate with AWS SigV4 credentials instead.", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "aws_access_key_id", + "label": "AWS Access Key ID", + "placeholder": null, + "tooltip": "Used for AWS SigV4 auth when no API key is set. You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "aws_secret_access_key", + "label": "AWS Secret Access Key", + "placeholder": null, + "tooltip": "Used for AWS SigV4 auth when no API key is set. You can provide the raw key or the environment variable (e.g. `os.environ/MY_SECRET_KEY`).", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "aws_session_token", + "label": "AWS Session Token", + "placeholder": null, + "tooltip": "Temporary credentials session token. You can provide the raw token or the environment variable (e.g. `os.environ/MY_SESSION_TOKEN`).", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "aws_region_name", + "label": "AWS Region Name", + "placeholder": "us-east-1", + "tooltip": "Region of the Bedrock Mantle endpoint. Defaults to us-east-1. You can provide the raw value or the environment variable (e.g. `os.environ/AWS_REGION_NAME`).", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "aws_session_name", + "label": "AWS Session Name", + "placeholder": "my-session", + "tooltip": "Name for the AWS session. You can provide the raw value or the environment variable (e.g. `os.environ/MY_SESSION_NAME`).", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "aws_profile_name", + "label": "AWS Profile Name", + "placeholder": "default", + "tooltip": "AWS profile name to use for authentication. You can provide the raw value or the environment variable (e.g. `os.environ/MY_PROFILE_NAME`).", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "aws_role_name", + "label": "AWS Role Name", + "placeholder": "MyRole", + "tooltip": "AWS IAM role name to assume. You can provide the raw value or the environment variable (e.g. `os.environ/MY_ROLE_NAME`).", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "aws_web_identity_token", + "label": "AWS Web Identity Token", + "placeholder": null, + "tooltip": "Web identity token for OIDC authentication. You can provide the raw token or the environment variable (e.g. `os.environ/MY_WEB_IDENTITY_TOKEN`).", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://bedrock-mantle.us-east-1.api.aws", + "tooltip": "Optional. Custom Bedrock Mantle endpoint. Defaults to https://bedrock-mantle..api.aws. You can provide the raw value or the environment variable (e.g. `os.environ/BEDROCK_MANTLE_API_BASE`).", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "bedrock_mantle/openai.gpt-oss-120b" + }, { "provider": "Anthropic", "provider_display_name": "Anthropic", diff --git a/litellm/proxy/read_model_list.py b/litellm/proxy/read_model_list.py new file mode 100644 index 00000000000..2dff8eaf698 --- /dev/null +++ b/litellm/proxy/read_model_list.py @@ -0,0 +1,28 @@ +"""Resolve a proxy config's ``model_list`` for the Rust AI gateway. + +The Rust gateway calls this once at load time (via an embedded interpreter) and +builds its own (Rust) router from the returned ``model_list``. We do NOT call +``ProxyConfig.load_config`` here: that returns a *Python* ``litellm.Router`` (not +usable from Rust) and boots the whole proxy (callbacks, cache, DB, auth) as side +effects. + +Instead we reuse ``ProxyConfig.get_config`` — the actual config reader — so the +gateway inherits the same heavy lifting the proxy does: ``include:`` merging, +``os.environ/`` + secret-manager resolution, and DB-stored models (when a DB is +configured). It has no proxy-setup side effects. Returns the resolved +``model_list``; the Rust side deserializes each entry into its ``Deployment``. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + + +def read_model_list(config_path: str) -> list[dict[str, Any]]: + """Load ``config_path`` via the proxy's own reader and return its + resolved ``model_list``.""" + from litellm.proxy.proxy_server import ProxyConfig + + config = asyncio.run(ProxyConfig().get_config(config_file_path=config_path)) + return config.get("model_list") or [] diff --git a/litellm/proxy/spend_tracking/cold_storage_handler.py b/litellm/proxy/spend_tracking/cold_storage_handler.py index 57c41bafccd..3974d4df618 100644 --- a/litellm/proxy/spend_tracking/cold_storage_handler.py +++ b/litellm/proxy/spend_tracking/cold_storage_handler.py @@ -16,8 +16,14 @@ class ColdStorageHandler: This class is responsible for handling Getting/Setting the proxy server request from cold storage. It allows fetching a dict of the proxy server request from s3 or GCS bucket. + + The cold storage logger can be injected for testing; when omitted it is + resolved from the configured ``litellm.cold_storage_custom_logger``. """ + def __init__(self, cold_storage_logger: Optional[CustomLogger] = None): + self._injected_cold_storage_logger = cold_storage_logger + async def get_proxy_server_request_from_cold_storage_with_object_key( self, object_key: str, @@ -31,33 +37,26 @@ class ColdStorageHandler: Returns: Optional[dict]: The proxy server request dict or None if not found """ - - # select the custom logger to use for cold storage - custom_logger_name: Optional[_custom_logger_compatible_callbacks_literal] = ( - self._select_custom_logger_for_cold_storage() + custom_logger = ( + self._injected_cold_storage_logger or self._resolve_cold_storage_logger() ) - - # if no custom logger name is configured, return None - if custom_logger_name is None: + if custom_logger is None: return None - # get the active/initialized custom logger - custom_logger: Optional[CustomLogger] = ( + return await custom_logger.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=object_key, + ) + + def _resolve_cold_storage_logger(self) -> Optional[CustomLogger]: + custom_logger_name = self._select_custom_logger_for_cold_storage() + if custom_logger_name is None: + return None + return ( litellm.logging_callback_manager.get_active_custom_logger_for_callback_name( custom_logger_name ) ) - # if no custom logger is found, return None - if custom_logger is None: - return None - - proxy_server_request = await custom_logger.get_proxy_server_request_from_cold_storage_with_object_key( - object_key=object_key, - ) - - return proxy_server_request - def _select_custom_logger_for_cold_storage( self, ) -> Optional[_custom_logger_compatible_callbacks_literal]: diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 0ba77dcd2f0..48f12d44370 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -3,7 +3,17 @@ import collections import json import os from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Mapping, + NamedTuple, + Optional, + Union, +) import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, status @@ -29,6 +39,7 @@ from litellm.repositories.verification_token_repository import ( if TYPE_CHECKING: from litellm.proxy.proxy_server import PrismaClient + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler else: PrismaClient = Any @@ -2175,6 +2186,89 @@ async def ui_view_spend_logs( raise handle_exception_on_proxy(e) +class RequestResponsePayload(NamedTuple): + messages: Optional[Union[str, list, dict]] + response: Optional[Union[str, list, dict]] + proxy_server_request: Optional[Union[str, dict]] + + +_EMPTY_SPEND_LOG_VALUES = frozenset({"", "{}", "[]", "null"}) + + +def _spend_log_field_has_content(value: Optional[Union[str, list, dict]]) -> bool: + if value is None: + return False + if isinstance(value, str): + return value.strip() not in _EMPTY_SPEND_LOG_VALUES + if isinstance(value, (list, dict)): + return len(value) > 0 + return True + + +def _cold_storage_object_key_from_metadata( + metadata: Optional[Union[str, dict]], +) -> Optional[str]: + if isinstance(metadata, str): + try: + metadata = json.loads(metadata) + except (json.JSONDecodeError, TypeError): + return None + if not isinstance(metadata, dict): + return None + object_key = metadata.get("cold_storage_object_key") + return object_key if isinstance(object_key, str) and object_key else None + + +async def _resolve_request_response_payload( + row: Mapping[str, Any], + cold_storage_handler: "ColdStorageHandler", +) -> RequestResponsePayload: + """ + Decide where the prompt/response come from for a single spend-log row. + + PG holds the content when ``store_prompts_in_spend_logs`` is on; otherwise it + holds ``"{}"`` placeholders and the real payload lives in cold storage keyed + by ``metadata.cold_storage_object_key``. The choice is made on actual row + content, not config flags, so historical and mixed-storage rows both resolve + correctly. + """ + messages = row.get("messages") + response = row.get("response") + proxy_server_request = row.get("proxy_server_request") + + pg_payload = RequestResponsePayload(messages, response, proxy_server_request) + if ( + _spend_log_field_has_content(messages) + or _spend_log_field_has_content(response) + or _spend_log_field_has_content(proxy_server_request) + ): + return pg_payload + + object_key = _cold_storage_object_key_from_metadata(row.get("metadata")) + if object_key is None: + return pg_payload + + try: + payload = await cold_storage_handler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=object_key + ) + except Exception: + verbose_proxy_logger.warning( + "Failed to fetch cold storage payload for key %s; falling back to DB values", + object_key, + exc_info=True, + ) + return pg_payload + if payload is None: + return pg_payload + + return RequestResponsePayload( + messages=payload.get("messages"), + response=payload.get("response"), + proxy_server_request=payload.get("proxy_server_request"), + ) + + @router.get( "/spend/logs/ui/{request_id}", tags=["Budget & Spend Tracking"], @@ -2241,26 +2335,27 @@ async def ui_view_request_response_for_request_id( if payload is not None: return payload - # Fallback: fetch heavy columns directly from the database. - # The list endpoint (/spend/logs/ui) intentionally excludes messages, - # response, and proxy_server_request for performance. When no custom - # logger (S3, GCS, etc.) is configured, we still need to serve these - # fields from the DB for the detail/drawer view. + # Fallback: the list endpoint omits the heavy columns for performance, so + # serve them here. When prompts were offloaded to cold storage the DB holds + # only placeholders, so _resolve_request_response_payload fetches the real + # payload from the configured cold storage backend by object key. if prisma_client is not None: + from litellm.proxy.spend_tracking.cold_storage_handler import ( + ColdStorageHandler, + ) + sql_query = """ - SELECT messages, response, proxy_server_request + SELECT messages, response, proxy_server_request, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = $1 LIMIT 1 """ db_result = await prisma_client.db.query_raw(sql_query, request_id) if db_result and len(db_result) > 0: - row = db_result[0] - return { - "messages": row.get("messages"), - "response": row.get("response"), - "proxy_server_request": row.get("proxy_server_request"), - } + resolved = await _resolve_request_response_payload( + db_result[0], cold_storage_handler=ColdStorageHandler() + ) + return resolved._asdict() return None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ea8ab2f9b8e..755602cbcc0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3792,6 +3792,10 @@ class PrismaClient: db_data["members_with_roles"], list ): db_data["members_with_roles"] = json.dumps(db_data["members_with_roles"]) + if db_data.get("budget_limits", None) is not None and isinstance( + db_data["budget_limits"], list + ): + db_data["budget_limits"] = json.dumps(db_data["budget_limits"]) return db_data # Define a retrying strategy with exponential backoff diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index e40e12e9197..3ef74d596ad 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -103,6 +103,11 @@ def rerank( """ Reranks a list of documents based on their relevance to the query """ + # `instruction` is read from kwargs rather than declared as a named param. + # The router forwards rerank calls via an untyped `**kwargs` unpack, and a + # typed named param there would trip the basedpyright budget gate without + # adding real safety; it stays typed downstream via get_optional_rerank_params. + instruction: Optional[str] = kwargs.get("instruction", None) headers: Optional[dict] = kwargs.get("headers") # type: ignore litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) @@ -155,6 +160,7 @@ def rerank( return_documents=return_documents, max_chunks_per_doc=max_chunks_per_doc, max_tokens_per_doc=max_tokens_per_doc, + instruction=instruction, non_default_params=kwargs, ) verbose_logger.info(f"optional_rerank_params: {optional_rerank_params}") diff --git a/litellm/rerank_api/rerank_utils.py b/litellm/rerank_api/rerank_utils.py index 38e599ef824..a8a665496fc 100644 --- a/litellm/rerank_api/rerank_utils.py +++ b/litellm/rerank_api/rerank_utils.py @@ -15,6 +15,7 @@ def get_optional_rerank_params( return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, non_default_params: Optional[dict] = None, ) -> Dict: all_non_default_params = non_default_params or {} @@ -30,6 +31,11 @@ def get_optional_rerank_params( all_non_default_params["max_chunks_per_doc"] = max_chunks_per_doc if max_tokens_per_doc is not None: all_non_default_params["max_tokens_per_doc"] = max_tokens_per_doc + if instruction is not None: + # Also surfaced in non_default_params so providers that read it from + # there (e.g. DeepInfra) keep working now that `rerank()` consumes + # `instruction` as a named param instead of leaving it in **kwargs. + all_non_default_params["instruction"] = instruction return rerank_provider_config.map_cohere_rerank_params( model=model, drop_params=drop_params, @@ -41,5 +47,6 @@ def get_optional_rerank_params( return_documents=return_documents, max_chunks_per_doc=max_chunks_per_doc, max_tokens_per_doc=max_tokens_per_doc, + instruction=instruction, non_default_params=all_non_default_params, ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 34c9cdd3d1c..2c46baaada5 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -58,7 +58,10 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.secret_managers.main import get_secret_str from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams -from litellm.utils import ProviderConfigManager, client +from litellm.utils import ( + ProviderConfigManager, + client, +) if TYPE_CHECKING: from mcp.types import Tool as MCPTool diff --git a/litellm/router.py b/litellm/router.py index acce1c58a8b..6e7b9689415 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -61,6 +61,9 @@ from litellm.constants import ( ) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, +) from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -565,6 +568,12 @@ class Router: self._explicit_timeout = timeout # None when user did not pass timeout self.timeout = timeout or litellm.request_timeout + # Per-attempt request_timeout, independent of router_settings.timeout. + # Only stored when a router timeout is also set, since otherwise + # request_timeout already flows through self.timeout above. + self.request_timeout = ( + get_configured_request_timeout() if timeout is not None else None + ) self.stream_timeout = stream_timeout self.retry_after = retry_after @@ -3134,7 +3143,18 @@ class Router: - litellm_trace_id - metadata """ - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) + # Normalise an explicit num_retries=None to the router default here (dict.get() + # only falls back when the key is absent, not when its value is None), then to 0 + # if the router default is itself None - mirroring the guard in + # async_function_with_retries, which remains the safety net for paths that bypass + # this setter. + _req_num_retries = kwargs.get("num_retries") + if _req_num_retries is not None: + kwargs["num_retries"] = _req_num_retries + else: + kwargs["num_retries"] = ( + self.num_retries if self.num_retries is not None else 0 + ) kwargs.setdefault("litellm_trace_id", str(uuid.uuid4())) model_group_alias: Optional[str] = None if self._get_model_from_alias(model=model): @@ -3380,6 +3400,7 @@ class Router: "stream_timeout", None ) # timeout set on litellm_params for this deployment or self.stream_timeout # timeout set on router + or self.request_timeout # litellm_settings.request_timeout (per-attempt) or self.default_litellm_params.get("stream_timeout", None) ) @@ -3396,7 +3417,8 @@ class Router: or data.get( "request_timeout", None ) # timeout set on litellm_params for this deployment - or self.timeout # timeout set on router + or self.request_timeout # litellm_settings.request_timeout (per-attempt) + or self.timeout # timeout set on router (router_settings.timeout) or self.default_litellm_params.get("timeout", None) ) return timeout @@ -6931,7 +6953,11 @@ class Router: "model_group_retry_policy", self.model_group_retry_policy ) model_group: Optional[str] = kwargs.get("model") - num_retries = kwargs.pop("num_retries") + num_retries = kwargs.pop("num_retries", None) + if num_retries is None: + # Fall back to the router setting (then 0) so the comparisons below never + # hit `None > int`, which would mask the real upstream error with a TypeError. + num_retries = self.num_retries if self.num_retries is not None else 0 ## ADD MODEL GROUP SIZE TO METADATA - used for model_group_rate_limit_error tracking _metadata: dict = kwargs.get("litellm_metadata", kwargs.get("metadata")) or {} diff --git a/litellm/sandbox/main.py b/litellm/sandbox/main.py index 45d3bffb4f9..76d9994c683 100644 --- a/litellm/sandbox/main.py +++ b/litellm/sandbox/main.py @@ -68,7 +68,7 @@ async def acreate_sandbox( provider: str, template: str | None = None, timeout: int | None = None, - allow_internet_access: bool = True, + allow_internet_access: bool | None = None, api_key: str | None = None, api_base: str | None = None, **kwargs, diff --git a/litellm/types/completion.py b/litellm/types/completion.py index cb263914be8..a91f6234fad 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -1,8 +1,28 @@ -from typing import Iterable, List, Optional, Union +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Coroutine, + Iterable, + List, + Optional, + Union, +) from pydantic import BaseModel, ConfigDict from typing_extensions import Literal, Required, TypedDict +if TYPE_CHECKING: + import httpx + from aiohttp import ClientSession + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm import BaseConfig + from litellm.utils import CustomStreamWrapper, ModelResponse + class ChatCompletionSystemMessageParam(TypedDict, total=False): content: Required[str] @@ -191,3 +211,44 @@ class CompletionRequest(BaseModel): model_list: Optional[List[str]] = None model_config = ConfigDict(protected_namespaces=(), extra="allow") + + +@dataclass(frozen=True, slots=True) +class _CompletionDispatchContext: + _azure_detection_model: str + acompletion: bool + api_base: Optional[str] + api_key: Optional[str] + api_version: Optional[str] + client: Any + custom_llm_provider: str + custom_prompt_dict: dict + extra_headers: Optional[dict] + headers: dict + hf_model_name: Optional[str] + kwargs: dict + litellm_params: dict + logger_fn: Optional[Callable] + logging: LiteLLMLoggingObj + max_retries: Optional[int] + max_tokens: Optional[int] + messages: list + metadata: Optional[dict] + model: str + model_response: ModelResponse + optional_params: dict + organization: Optional[str] + provider_config: Optional[BaseConfig] + shared_session: Optional[ClientSession] + stream: Optional[bool] + temperature: Optional[float] + text_completion: bool + timeout: Optional[Union[float, str, httpx.Timeout]] + top_p: Optional[float] + + +_CompletionDispatchResult = Union[ + Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]], + "ModelResponse", + "CustomStreamWrapper", +] diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index b5726a11ca0..26a0be36ef4 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -2,6 +2,25 @@ from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field +CHAT_COMPLETION_AGENTIC_SURFACE = "chat_completions" +CODE_INTERPRETER_INTERCEPTION_PREFIX = "_code_interpreter_interception" +NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES = frozenset( + ("_websearch_interception", "_compression_interception") +) +INTERCEPTION_INTERNAL_PREFIXES = frozenset( + ( + *NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + CODE_INTERPRETER_INTERCEPTION_PREFIX, + ) +) + + +def is_interception_internal_key( + key: str, + prefixes: frozenset[str] = INTERCEPTION_INTERNAL_PREFIXES, +) -> bool: + return any(key.startswith(prefix) for prefix in prefixes) + class StandardCustomLoggerInitParams(BaseModel): """ diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index 21d4da82041..94e4c68f5e2 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -69,7 +69,6 @@ class MCPPublicServer(BaseModel): name: str alias: Optional[str] = None server_name: Optional[str] = None - url: Optional[str] = None transport: MCPTransportType spec_path: Optional[str] = None auth_type: Optional[MCPAuthType] = None diff --git a/litellm/types/proxy/management_endpoints/scim_v2.py b/litellm/types/proxy/management_endpoints/scim_v2.py index c5fdc66154f..6270d2d0925 100644 --- a/litellm/types/proxy/management_endpoints/scim_v2.py +++ b/litellm/types/proxy/management_endpoints/scim_v2.py @@ -1,7 +1,20 @@ from typing import Any, Dict, List, Literal, Optional, Union from fastapi import HTTPException -from pydantic import BaseModel, ConfigDict, EmailStr, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + EmailStr, + Field, + field_validator, + model_serializer, +) +from pydantic_core.core_schema import SerializerFunctionWrapHandler + +SCIM_ENTERPRISE_USER_SCHEMA = ( + "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User" +) +SCIM_ENTERPRISE_METADATA_KEY = "scim_enterprise" class LiteLLM_UserScimMetadata(BaseModel): @@ -42,13 +55,49 @@ class SCIMUserGroup(BaseModel): type: Optional[str] = "direct" # direct or indirect +class SCIMUserManager(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + value: Optional[str] = None + displayName: Optional[str] = None + ref: Optional[str] = Field(default=None, alias="$ref") + + +class SCIMEnterpriseUser(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + employeeNumber: Optional[str] = None + costCenter: Optional[str] = None + organization: Optional[str] = None + division: Optional[str] = None + department: Optional[str] = None + manager: Optional[SCIMUserManager] = None + + class SCIMUser(SCIMResource): + model_config = ConfigDict(populate_by_name=True) + userName: Optional[str] = None name: Optional[SCIMUserName] = None displayName: Optional[str] = None active: bool = True emails: Optional[List[SCIMUserEmail]] = None groups: Optional[List[SCIMUserGroup]] = None + enterprise_user: Optional[SCIMEnterpriseUser] = Field( + default=None, + alias=SCIM_ENTERPRISE_USER_SCHEMA, + serialization_alias=SCIM_ENTERPRISE_USER_SCHEMA, + ) + + @model_serializer(mode="wrap") + def _omit_absent_enterprise( + self, handler: SerializerFunctionWrapHandler + ) -> Dict[str, Any]: + dumped = handler(self) + if self.enterprise_user is None: + dumped.pop(SCIM_ENTERPRISE_USER_SCHEMA, None) + dumped.pop("enterprise_user", None) + return dumped class SCIMMember(BaseModel): diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index d2c252a1e92..376d6f66603 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -19,6 +19,10 @@ class RerankRequest(BaseModel): return_documents: Optional[bool] = None max_chunks_per_doc: Optional[int] = None max_tokens_per_doc: Optional[int] = None + # Optional task/query instruction passed through to providers that support it + # (e.g. hosted vLLM / Qwen3-Reranker, DeepInfra). Omitted from the outgoing + # request when None, so this is fully backward-compatible. + instruction: Optional[str] = None class OptionalRerankParams(TypedDict, total=False): @@ -29,6 +33,7 @@ class OptionalRerankParams(TypedDict, total=False): return_documents: Optional[bool] max_chunks_per_doc: Optional[int] max_tokens_per_doc: Optional[int] + instruction: Optional[str] class RerankBilledUnits(TypedDict, total=False): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index acfee30ef54..d28a2680b73 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -37,6 +37,8 @@ from pydantic import ( ConfigDict, Field, PrivateAttr, + SkipValidation, + field_serializer, field_validator, ) from typing_extensions import Required, TypedDict @@ -2439,7 +2441,7 @@ class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject): class TranscriptionUsageDurationObject(BaseModel): type: Literal["duration"] - seconds: int + seconds: float class TranscriptionUsageInputTokenDetailsObject(BaseModel): @@ -3146,10 +3148,41 @@ class CustomPricingLiteLLMParams(BaseModel): search_context_cost_per_query: Optional[Dict[str, Any]] = None citation_cost_per_token: Optional[float] = None tiered_pricing: Optional[List[Dict[str, Any]]] = None + cache_read_input_token_cost_above_272k_tokens: Optional[float] = None + cache_read_input_token_cost_above_512k_tokens: Optional[float] = None + input_cost_per_image_token: Optional[float] = None + input_cost_per_token_above_272k_tokens: Optional[float] = None + input_cost_per_token_above_512k_tokens: Optional[float] = None + output_cost_per_token_above_272k_tokens: Optional[float] = None + output_cost_per_token_above_512k_tokens: Optional[float] = None + output_vector_size: Optional[int] = None + ocr_cost_per_page: Optional[float] = None + ocr_cost_per_credit: Optional[float] = None + annotation_cost_per_page: Optional[float] = None + regional_processing_uplift_multiplier_eu: Optional[float] = None + regional_processing_uplift_multiplier_us: Optional[float] = None +# Server-controlled fields that bound or drive an interceptor's agentic loop +# (depth, cycle fingerprints, ceiling, code-interpreter sandbox state). Listed +# in all_litellm_params so they are treated as LiteLLM-level and excluded from +# get_non_default_completion_params; otherwise the OpenAI param builder sweeps +# any unrecognized top-level key into extra_body and leaks them to the provider. +# This is what lets the loop carry state across rerun calls without a provider +# scrubber. +agentic_loop_internal_litellm_params = [ + "_agentic_loop_depth", + "_agentic_loop_fingerprints", + "_agentic_loop_api_surface", + "max_agentic_loops", + "_code_interpreter_interception_active", + "_code_interpreter_interception_sandbox_key", + "_code_interpreter_interception_converted_stream", +] + all_litellm_params = ( - [ + agentic_loop_internal_litellm_params + + [ "metadata", "litellm_metadata", "litellm_trace_id", @@ -3506,6 +3539,7 @@ class SandboxProviders(str, Enum): """ E2B = "e2b" + OPENSANDBOX = "opensandbox" class LiteLLMLoggingBaseClass: @@ -3611,10 +3645,20 @@ class LiteLLMBatch(Batch): class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase): - results: OpenAIRealtimeStreamList + # Events are already well-formed provider dicts. Validating them against the + # OpenAIRealtimeEvents union makes Pydantic try every member per event, which + # floods thousands of ValidationErrors for events outside the union (e.g. + # rate_limits.updated), blocks the event loop, and discards the session usage. + results: SkipValidation[OpenAIRealtimeStreamList] usage: Usage _hidden_params: dict = {} + @field_serializer("results") + def _serialize_results( + self, results: OpenAIRealtimeStreamList + ) -> List[Dict[str, Any]]: + return [dict(event) for event in results] + def __contains__(self, key): # Define custom behavior for the 'in' operator return hasattr(self, key) diff --git a/litellm/utils.py b/litellm/utils.py index a842e9e058d..5c3ab3e1490 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9731,9 +9731,14 @@ class ProviderConfigManager: Get sandbox (code execution) configuration for a given provider. """ from litellm.llms.e2b.sandbox.transformation import E2BSandboxConfig + from litellm.llms.opensandbox.sandbox.transformation import ( + OpenSandboxSandboxConfig, + ) if provider == SandboxProviders.E2B: return E2BSandboxConfig() + if provider == SandboxProviders.OPENSANDBOX: + return OpenSandboxSandboxConfig() return None @staticmethod diff --git a/migrations/Dockerfile b/migrations/Dockerfile index a78a4e2225a..caca280cbfc 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 56baa5c573f..d7017a40993 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10443,7 +10443,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "supports_speed": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -10476,7 +10477,8 @@ "fast": 6.0 }, "supports_max_reasoning_effort": true, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -10511,7 +10513,8 @@ "us": 1.1, "fast": 6.0 }, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -10546,7 +10549,8 @@ "us": 1.1, "fast": 6.0 }, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -10615,7 +10619,8 @@ "us": 1.1, "fast": 2.0 }, - "supports_output_config": true + "supports_output_config": true, + "supports_speed": true }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -10684,6 +10689,268 @@ "mode": "chat", "output_cost_per_token": 1.923e-06 }, + "cloudflare/@cf/openai/gpt-oss-120b": { + "input_cost_per_token": 3.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/google/gemma-2b-it-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/meta/llama-3.2-3b-instruct": { + "input_cost_per_token": 5.09e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 80000, + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 3.35e-07 + }, + "cloudflare/@cf/meta/llama-guard-3-8b": { + "input_cost_per_token": 4.84e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 3e-08 + }, + "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 15000, + "max_output_tokens": 15000, + "max_tokens": 15000, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": { + "input_cost_per_token": 4.97e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 80000, + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 4.881e-06, + "supports_reasoning": true + }, + "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": { + "input_cost_per_token": 1.52e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.87e-07 + }, + "cloudflare/@cf/meta/llama-3.2-1b-instruct": { + "input_cost_per_token": 2.7e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 60000, + "max_output_tokens": 60000, + "max_tokens": 60000, + "mode": "chat", + "output_cost_per_token": 2.01e-07 + }, + "cloudflare/@cf/moonshotai/kimi-k2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/zai-org/glm-4.7-flash": { + "input_cost_per_token": 6.05e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/meta-llama/llama-2-7b-chat-hf-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { + "input_cost_per_token": 2.93e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 24000, + "max_output_tokens": 24000, + "max_tokens": 24000, + "mode": "chat", + "output_cost_per_token": 2.253e-06, + "supports_function_calling": true + }, + "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": { + "input_cost_per_token": 1.7e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 131000, + "max_output_tokens": 131000, + "max_tokens": 131000, + "mode": "chat", + "output_cost_per_token": 1.12e-07, + "supports_function_calling": true + }, + "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": { + "input_cost_per_token": 6.6e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1e-06 + }, + "cloudflare/@cf/zai-org/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "cloudflare", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/nvidia/nemotron-3-120b-a12b": { + "input_cost_per_token": 5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/aisingapore/gemma-sea-lion-v4-27b-it": { + "input_cost_per_token": 3.51e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.55e-07 + }, + "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": { + "input_cost_per_token": 5.09e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3.35e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/google/gemma-7b-it-lora": { + "input_cost_per_token": 0.0, + "litellm_provider": "cloudflare", + "max_input_tokens": 3500, + "max_output_tokens": 3500, + "max_tokens": 3500, + "mode": "chat", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/google/gemma-4-26b-a4b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 3e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/mistralai/mistral-small-3.1-24b-instruct": { + "input_cost_per_token": 3.51e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.55e-07, + "supports_function_calling": true + }, + "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": { + "input_cost_per_token": 4.85e-08, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6.76e-07, + "supports_vision": true + }, + "cloudflare/@cf/openai/gpt-oss-20b": { + "input_cost_per_token": 2e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-07, + "supports_function_calling": true, + "supports_reasoning": true + }, + "cloudflare/@cf/meta/llama-4-scout-17b-16e-instruct": { + "input_cost_per_token": 2.7e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 131000, + "max_output_tokens": 131000, + "max_tokens": 131000, + "mode": "chat", + "output_cost_per_token": 8.5e-07, + "supports_function_calling": true + }, + "cloudflare/@cf/qwen/qwq-32b": { + "input_cost_per_token": 6.6e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 24000, + "max_output_tokens": 24000, + "max_tokens": 24000, + "mode": "chat", + "output_cost_per_token": 1e-06, + "supports_reasoning": true + }, "codestral/codestral-2405": { "input_cost_per_token": 0.0, "litellm_provider": "codestral", @@ -20096,8 +20363,6 @@ "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, "output_cost_per_token_priority": 1.4e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -20171,8 +20436,6 @@ "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, "output_cost_per_token_priority": 2.8e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -20246,8 +20509,6 @@ "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 8e-07, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -20319,8 +20580,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.7e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20362,8 +20621,6 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20385,8 +20642,6 @@ "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -20675,8 +20930,6 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -21380,8 +21633,6 @@ "output_cost_per_token": 1e-05, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21775,6 +22026,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21823,6 +22076,8 @@ "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_priority": 6e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21867,6 +22122,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21911,6 +22168,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -21959,6 +22218,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22006,6 +22267,8 @@ "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 3e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22046,6 +22309,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22089,6 +22354,8 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -22134,6 +22401,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22180,6 +22449,8 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_priority": 9e-06, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22223,6 +22494,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22266,6 +22539,8 @@ "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 6.25e-07, + "regional_processing_uplift_multiplier_eu": 1.10, + "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22304,8 +22579,6 @@ "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/batch", "/v1/responses" @@ -22712,8 +22985,6 @@ "output_cost_per_token": 2e-06, "output_cost_per_token_flex": 1e-06, "output_cost_per_token_priority": 3.6e-06, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -22795,8 +23066,6 @@ "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, - "regional_processing_uplift_multiplier_eu": 1.10, - "regional_processing_uplift_multiplier_us": 1.10, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_flex": 2e-07, @@ -29597,6 +29866,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/z-ai/glm-5.1": { + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 3.5e-06, + "cache_read_input_token_cost": 5.25e-07, + "cache_creation_input_token_cost": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 202752, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "chat", + "source": "https://openrouter.ai/z-ai/glm-5.1", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/minimax/minimax-m2.1": { "input_cost_per_token": 2.7e-07, "output_cost_per_token": 1.2e-06, @@ -37603,6 +37888,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-5.1": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-5-code": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-07, @@ -37633,6 +37933,21 @@ "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing" }, + "zai/glm-4.7-flash": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 0, + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-4.6": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, diff --git a/osv-scanner.toml b/osv-scanner.toml index f0f5f045f1a..7ab450945f5 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -2,13 +2,3 @@ id = "GHSA-w8v5-vhqr-4h9v" ignoreUntil = 2026-09-09 reason = "diskcache has no fixed release published; remove this entry once one exists" - -[[IgnoredVulns]] -id = "GHSA-hg6j-4rv6-33pg" -ignoreUntil = 2026-08-15 -reason = "aiohttp held at 3.13.5: vcrpy releases <= 8.1.1 cannot import aiohttp >= 3.14 and the merged upstream fix (vcrpy PR 996) is unreleased; bump aiohttp and drop this entry when a newer vcrpy ships" - -[[IgnoredVulns]] -id = "GHSA-jg22-mg44-37j8" -ignoreUntil = 2026-08-15 -reason = "aiohttp held at 3.13.5: vcrpy releases <= 8.1.1 cannot import aiohttp >= 3.14 and the merged upstream fix (vcrpy PR 996) is unreleased; bump aiohttp and drop this entry when a newer vcrpy ships" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 9991ff9e01e..b137ec59a1f 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1833,6 +1833,23 @@ "text_completion": true } }, + "opensandbox": { + "display_name": "OpenSandbox (`opensandbox`)", + "url": "https://open-sandbox.ai/api/", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "sandbox": true + } + }, "openai_like": { "display_name": "OpenAI-like (`openai_like`)", "url": "https://docs.litellm.ai/docs/providers/openai_compatible", diff --git a/pyproject.toml b/pyproject.toml index 91de8683968..1cb39153e83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.90.0" +version = "1.91.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.14" @@ -55,7 +55,7 @@ proxy = [ "fastapi-sso>=0.19.0,<1.0", "PyJWT>=2.13.0,<3.0", "python-multipart>=0.0.27,<1.0", - "cryptography>=46.0.7,<47.0", + "cryptography>=48.0.1,<49.0", "pynacl>=1.6.2,<2.0", "websockets>=15.0.1,<16.0", "boto3>=1.43.1,<2.0", @@ -70,6 +70,7 @@ proxy = [ "soundfile>=0.12.1,<1.0", "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "pydantic-settings>=2.14.1,<3.0", + "expression>=5.6.0,<6.0", ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy # imports (fastapi, cryptography, ...) are all guarded, so it runs on the base @@ -181,7 +182,7 @@ dev = [ "parameterized==0.9.0", "openapi-core==0.22.0; python_version < '3.14'", "pytest-timeout==2.4.0", - "vcrpy==8.1.1", + "vcrpy==8.2.1", "pytest-recording==0.13.4", ] proxy-dev = [ @@ -207,7 +208,7 @@ ci = [ "pytest-codspeed==4.3.0", "pytest-retry==1.7.0", "pyarrow==23.0.1", - "langchain==1.2.10", + "langchain==1.3.9", "lunary==1.4.36; python_version == '3.10'", "lunary==1.4.37; python_version >= '3.11'", "logfire==4.6.0", @@ -224,11 +225,8 @@ ci = [ "pylint==4.0.5", "langchain-mcp-adapters==0.2.1", "langchain-openai==1.1.14", - "langgraph==1.0.10", - # langgraph-prebuilt 1.0.9 imports ExecutionInfo/ServerInfo from - # langgraph.runtime, which is not exported until langgraph 1.1.0. - # Pin to 1.0.8 so it pairs correctly with langgraph==1.0.10. - "langgraph-prebuilt==1.0.8", + "langgraph>=1.2.4,<1.3.0", + "langgraph-prebuilt>=1.1.0,<1.3.0", "claude-agent-sdk==0.1.44", ] healthcheck = [ @@ -243,7 +241,7 @@ build-backend = "uv_build" [tool.uv] constraint-dependencies = [ "tornado>=6.5.6", - "aiohttp>=3.13.5,<3.14", + "aiohttp>=3.14.1,<4.0", ] default-groups = ["dev"] required-version = ">=0.10.9" @@ -272,7 +270,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.90.0" +version = "1.91.0" version_files = [ "pyproject.toml:^version", ] diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 62ebdb559fc..ae46f020de1 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -300,7 +300,7 @@ "slack": 3 }, "RET504": { - "baseline": 709, + "baseline": 702, "slack": 20 }, "RUF010": { diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index 0f9a44703f9..2ef332d91ea 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -1,21 +1,22 @@ #!/usr/bin/env python3 -"""Per-rule count gate for basedpyright. +"""Delta-vs-base per-rule gate for basedpyright. basedpyright's ``--outputjson`` is reduced to a count of errors per *rule* (``reportAny``, ``reportArgumentType``, ...) and checked against a committed budget of the form ``{rule: {baseline, slack}}``, the same shape as -``ruff-strict-budget.json``. A rule fails when its codebase-wide total exceeds -``baseline + slack``. Counts ignore file, line, and column, so a violation -moving anywhere in the tree is invisible; only the per-rule total moves the -needle. +``ruff-strict-budget.json``. A rule fails only when its codebase-wide total is +both over its ceiling (``baseline + slack``) *and* higher than the count on the +base it merges into, so a change is blamed for the errors it adds, never for +drift that already sits in the base. That ``> base`` guard is what stops an +unrelated PR from inheriting a red once two PRs each land near the ceiling and +their sum crosses it: the bystander's count equals its base, so it is spared, +while any PR that actually grows the rule past the cap still fails. -Unlike ``ruff_strict_gate.py`` this does *not* re-run the tool on the merge base -to compute a delta: a second basedpyright pass is minutes and gigabytes, whereas -ruff is milliseconds. The committed budget is the baseline instead -- exactly -how the previous per-file gate worked -- so keep it fresh with ``--update`` -(ratchet), which re-captures every rule's count from the current tree while -preserving each rule's slack. Tool output is read from stdin, so the caller -decides how to invoke basedpyright (and from which cwd). +Head counts are read from stdin (the caller runs basedpyright once and pipes +``--outputjson`` in); the base count is a second basedpyright pass over a +detached worktree at the merge-base, run under the same environment so import +resolution matches. ``--update`` re-captures the absolute per-rule baselines for +the ratchet, preserving each rule's slack. ``--outputjson`` is used rather than text diagnostics because the latter wrap across lines, leaving the ``(reportRule)`` on a continuation line away from the @@ -24,13 +25,21 @@ carries an unambiguous ``rule`` field. """ import argparse +import contextlib import json +import shutil +import subprocess import sys +import tempfile from collections import Counter +from collections.abc import Iterator, Mapping from pathlib import Path -from typing import Mapping, NamedTuple +from typing import NamedTuple REPO_ROOT = Path(__file__).resolve().parent.parent +BUDGET_PATH = REPO_ROOT / "basedpyright-code-budget.json" +PYRIGHT_CONFIG = REPO_ROOT / "pyrightconfig.json" +DEFAULT_BASE = "origin/litellm_internal_staging" # Bucket for a basedpyright diagnostic with no `rule`. Counted so it's gated. UNCODED = "" @@ -45,6 +54,7 @@ class Breach(NamedTuple): code: str total: int cap: int + added: int def _seed_slack(baseline: int) -> int: @@ -54,18 +64,19 @@ def _seed_slack(baseline: int) -> int: return 10 if baseline >= 50 else 3 -def _to_repo_relative(raw: str) -> str | None: +def _to_relative(raw: str, root: Path) -> str | None: path = Path(raw) - absolute = path if path.is_absolute() else Path.cwd() / path + absolute = path if path.is_absolute() else root / path try: - return absolute.resolve().relative_to(REPO_ROOT).as_posix() + return absolute.resolve().relative_to(root).as_posix() except ValueError: return None -def count_basedpyright(payload: str) -> dict[str, int]: - """Count in-repo basedpyright errors per rule from `--outputjson`. Warnings - and information are ignored; only `severity == "error"` is gated.""" +def count_basedpyright(payload: str, root: Path = REPO_ROOT) -> dict[str, int]: + """Count in-tree basedpyright errors per rule from `--outputjson`. Warnings + and information are ignored; only `severity == "error"` is gated. Files + outside `root` (the venv's site-packages, say) are dropped.""" try: data = json.loads(payload or "{}") except json.JSONDecodeError as exc: @@ -79,21 +90,62 @@ def count_basedpyright(payload: str) -> dict[str, int]: for diag in data.get("generalDiagnostics", []): if diag.get("severity") != "error": continue - if _to_repo_relative(diag.get("file", "")) is None: + if _to_relative(diag.get("file", ""), root) is None: continue counts[diag.get("rule") or UNCODED] += 1 return dict(counts) +def _run(cmd: list[str], cwd: Path = REPO_ROOT) -> str: + proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if proc.returncode not in (0, 1): + sys.stderr.write(proc.stderr) + raise SystemExit(f"{cmd[0]} exited {proc.returncode}") + return proc.stdout + + +@contextlib.contextmanager +def _temp_worktree(ref: str) -> Iterator[Path]: + parent = Path(tempfile.mkdtemp(prefix="bpr_base_")) + worktree = parent / "wt" + try: + _run(["git", "worktree", "add", "--detach", str(worktree), ref]) + yield worktree + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + shutil.rmtree(parent, ignore_errors=True) + + +def base_counts(ref: str) -> dict[str, int]: + """basedpyright error counts per rule for the merge-base tree. The head + config is copied in so the base is judged by today's rules, and the run uses + the head environment's basedpyright (on PATH) so imports resolve the same.""" + exe = shutil.which("basedpyright") or "basedpyright" + with _temp_worktree(ref) as worktree: + shutil.copy(PYRIGHT_CONFIG, worktree / "pyrightconfig.json") + proc = subprocess.run( + [exe, "--outputjson"], cwd=worktree, capture_output=True, text=True + ) + return count_basedpyright(proc.stdout, root=worktree) + + def evaluate( - counts: Mapping[str, int], budget: Mapping[str, Mapping[str, int]] + head: Mapping[str, int], + base: Mapping[str, int], + budget: Mapping[str, Mapping[str, int]], ) -> list[Breach]: breaches = [] - for code, total in counts.items(): + for code, total in head.items(): spec = budget.get(code) cap = spec["baseline"] + spec["slack"] if spec else DEFAULT_SLACK - if total > cap: - breaches.append(Breach(code, total, cap)) + prior = base.get(code, 0) + if total > cap and total > prior: + breaches.append(Breach(code, total, cap, total - prior)) return sorted(breaches) @@ -107,9 +159,6 @@ def is_vacuous_run( return not counts and any(spec["baseline"] for spec in budget.values()) -BUDGET_PATH = REPO_ROOT / "basedpyright-code-budget.json" - - def cmd_update(counts: Mapping[str, int]) -> None: existing = json.loads(BUDGET_PATH.read_text()) if BUDGET_PATH.exists() else {} budget = { @@ -127,9 +176,10 @@ def cmd_update(counts: Mapping[str, int]) -> None: ) -def cmd_check(counts: Mapping[str, int]) -> None: +def cmd_check(base_ref: str) -> None: budget = json.loads(BUDGET_PATH.read_text()) - if is_vacuous_run(counts, budget): + head = count_basedpyright(sys.stdin.read()) + if is_vacuous_run(head, budget): expected = sum(spec["baseline"] for spec in budget.values()) print( f"FAIL: basedpyright produced no errors, but {BUDGET_PATH.name} expects " @@ -137,27 +187,44 @@ def cmd_check(counts: Mapping[str, int]) -> None: f"nothing; refusing to certify a vacuous run." ) raise SystemExit(1) - breaches = evaluate(counts, budget) + base_point = _run(["git", "merge-base", base_ref, "HEAD"]).strip() or base_ref + base = base_counts(base_point) + if is_vacuous_run(base, budget): + print( + f"FAIL: basedpyright produced no errors for the base tree at " + f"{base_point[:12]}, so every rule would look freshly added. The base " + f"pass almost certainly crashed; refusing to blame this change for it." + ) + raise SystemExit(1) + breaches = evaluate(head, base, budget) if not breaches: print( - f"OK: every rule is within its basedpyright ceiling ({sum(counts.values())} errors total)" + f"OK: every rule is within its basedpyright ceiling or no higher than base ({sum(head.values())} errors total)" ) return print("FAIL: basedpyright errors exceed the per-rule ceiling:") for breach in breaches: - print(f" {breach.code}: {breach.total} errors over cap {breach.cap}") + print( + f" {breach.code}: total {breach.total} over cap {breach.cap} (this change added {breach.added})" + ) print( - "Resolve the new errors, or run 'make lint-basedpyright-budget-update' if the ceiling should move." + "Reduce the new errors or remove an equal number elsewhere; the ceiling is " + "baseline + slack in basedpyright-code-budget.json." ) + summary = "; ".join(f"{b.code} {b.total}/{b.cap} (+{b.added})" for b in breaches) + print(f"BREACHED RULES: {summary}") raise SystemExit(1) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", default=DEFAULT_BASE) parser.add_argument("--update", action="store_true") args = parser.parse_args() - counts = count_basedpyright(sys.stdin.read()) - cmd_update(counts) if args.update else cmd_check(counts) + if args.update: + cmd_update(count_basedpyright(sys.stdin.read())) + else: + cmd_check(args.base) if __name__ == "__main__": diff --git a/tests/batches_tests/test_batch_custom_pricing.py b/tests/batches_tests/test_batch_custom_pricing.py index cb2ca385ffc..3dc1d116e8d 100644 --- a/tests/batches_tests/test_batch_custom_pricing.py +++ b/tests/batches_tests/test_batch_custom_pricing.py @@ -159,12 +159,12 @@ def test_batch_cost_calculator_applies_data_residency_uplift( base_prompt, base_completion = batch_cost_calculator( usage=usage, - model="gpt-5", + model="gpt-5.4", custom_llm_provider="openai", ) regional_prompt, regional_completion = batch_cost_calculator( usage=usage, - model="gpt-5", + model="gpt-5.4", custom_llm_provider="openai", data_residency=data_residency, ) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 254d700ee5a..1d11d676207 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -47,6 +47,7 @@ IGNORE_FUNCTIONS = [ "_read_image_bytes", # max depth set. "_get_masked_values", # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts. "_redact_sensitive_litellm_params", # max depth set (default 10). + "_redact_secret_values_in_obj", # max depth set (default 10, _REDACT_SECRET_MAX_DEPTH); fails closed by returning "REDACTED" at the cap. "_resolve", # OCI: $ref resolver bounded by `resolving_stack` cycle guard. "resolve_oci_schema_anyof", # OCI: bounded by JSON-schema tree depth (no cycles possible in well-formed input). "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. diff --git a/tests/llm_translation/test_cloudflare.py b/tests/llm_translation/test_cloudflare.py index 5a6a0008398..54c5d9e4e07 100644 --- a/tests/llm_translation/test_cloudflare.py +++ b/tests/llm_translation/test_cloudflare.py @@ -9,9 +9,7 @@ import pytest from litellm import acompletion, completion from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -FAKE_API_BASE = ( - "https://fake-cloudflare.example.com/client/v4/accounts/fake-acct/ai/run/" -) +FAKE_API_BASE = "https://fake-cloudflare.example.com/client/v4/accounts/fake-acct/ai/v1" FAKE_API_KEY = "fake-cf-api-key" @@ -26,28 +24,78 @@ def _make_mock_response(json_data: Dict[str, Any]) -> MagicMock: def _chat_response() -> Dict[str, Any]: return { - "result": { - "response": "I am a large language model created to assist you.", - }, - "success": True, - "errors": [], - "messages": [], + "id": "chatcmpl-cf", + "object": "chat.completion", + "created": 1234567890, + "model": "@cf/meta/llama-2-7b-chat-int8", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I am a large language model created to assist you.", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 8, "completion_tokens": 11, "total_tokens": 19}, + } + + +def _tool_call_response() -> Dict[str, Any]: + return { + "id": "chatcmpl-cf-tools", + "object": "chat.completion", + "created": 1234567890, + "model": "@cf/meta/llama-2-7b-chat-int8", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "New York"}', + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 9, "total_tokens": 29}, } def _streaming_chunks() -> list[str]: + base = { + "id": "chatcmpl-cf", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "@cf/meta/llama-2-7b-chat-int8", + } return [ - json.dumps({"response": "I am"}), - json.dumps({"response": " a language"}), - json.dumps({"response": " model."}), - ] - - -def _streaming_chunks_response_text() -> list[str]: - return [ - json.dumps({"response_text": "I am"}), - json.dumps({"response_text": " a language"}), - json.dumps({"response_text": " model."}), + json.dumps({**base, "choices": [{"index": 0, "delta": {"content": "I am"}}]}), + json.dumps( + {**base, "choices": [{"index": 0, "delta": {"content": " a language"}}]} + ), + json.dumps( + { + **base, + "choices": [ + { + "index": 0, + "delta": {"content": " model."}, + "finish_reason": "stop", + } + ], + } + ), ] @@ -85,6 +133,48 @@ def test_completion_cloudflare(sync_mode): assert response.choices[0].message.content is not None assert "language model" in response.choices[0].message.content.lower() + called_url = mock_post.call_args.kwargs.get("url") or mock_post.call_args.args[0] + assert called_url.endswith("/ai/v1/chat/completions") + assert "/ai/run/" not in called_url + + +def test_completion_cloudflare_tool_calls_sent_to_openai_endpoint(): + messages = [{"role": "user", "content": "weather in New York?"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + mock_resp = _make_mock_response(_tool_call_response()) + + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="cloudflare/@cf/meta/llama-2-7b-chat-int8", + messages=messages, + tools=tools, + tool_choice="auto", + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + mock_post.assert_called_once() + + sent_body = json.loads(mock_post.call_args.kwargs["data"]) + assert sent_body["tools"] == tools + assert sent_body["tool_choice"] == "auto" + + assert response.choices[0].finish_reason == "tool_calls" + tool_calls = response.choices[0].message.tool_calls + assert tool_calls is not None and len(tool_calls) == 1 + assert tool_calls[0].function.name == "get_weather" + @pytest.mark.parametrize("sync_mode", [True, False]) def test_completion_cloudflare_stream(sync_mode): @@ -153,76 +243,3 @@ def test_completion_cloudflare_stream(sync_mode): if c.choices[0].delta.content ) assert "language" in content.lower() - - -@pytest.mark.parametrize("sync_mode", [True, False]) -def test_completion_cloudflare_stream_response_text(sync_mode): - """Newer Cloudflare Workers AI models (e.g. Nemotron) emit `response_text` - instead of `response` in streamed chunks. The iterator must surface that - text so streaming output is not silently empty. - """ - messages = [{"role": "user", "content": "what llm are you"}] - raw_chunks = _streaming_chunks_response_text() - - if sync_mode: - - def _iter_lines(): - for chunk in raw_chunks: - yield f"data: {chunk}" - yield "data: [DONE]" - - mock_resp = MagicMock() - mock_resp.iter_lines.return_value = _iter_lines() - mock_resp.status_code = 200 - mock_resp.headers = {"content-type": "text/event-stream"} - - with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: - response = completion( - model="cloudflare/@cf/nvidia/nemotron-mini-4b-instruct", - messages=messages, - max_tokens=15, - stream=True, - api_base=FAKE_API_BASE, - api_key=FAKE_API_KEY, - ) - chunks_received = list(response) - mock_post.assert_called_once() - else: - - async def _aiter_lines(): - for chunk in raw_chunks: - yield f"data: {chunk}" - yield "data: [DONE]" - - mock_resp = MagicMock() - mock_resp.aiter_lines.return_value = _aiter_lines() - mock_resp.status_code = 200 - mock_resp.headers = {"content-type": "text/event-stream"} - - async def _run(): - with patch.object( - AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp - ) as mock_post: - resp = await acompletion( - model="cloudflare/@cf/nvidia/nemotron-mini-4b-instruct", - messages=messages, - max_tokens=15, - stream=True, - api_base=FAKE_API_BASE, - api_key=FAKE_API_KEY, - ) - received = [] - async for chunk in resp: - received.append(chunk) - mock_post.assert_called_once() - return received - - chunks_received = asyncio.run(_run()) - - assert len(chunks_received) > 0 - content = "".join( - c.choices[0].delta.content - for c in chunks_received - if c.choices[0].delta.content - ) - assert "language" in content.lower() diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index 9a69f513069..d46436f209b 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1627,7 +1627,12 @@ class TestMissingChoicesGuard: assert "no 'choices'" in exc_info.value.message def test_convert_to_model_response_object_empty_choices_raises_api_error(self): - """Empty choices list raises APIError.""" + """Empty choices list raises APIError, same as missing/null choices. + + Provider-specific repair (e.g. github_copilot synthesizing choices for + Anthropic-native responses) happens before this guard, in the provider + config; the core utility keeps treating empty choices as an error. + """ from litellm.exceptions import APIError response_object = { @@ -1683,7 +1688,9 @@ class TestMissingChoicesGuard: assert "no 'choices'" in exc_info.value.message - def test_convert_to_model_response_object_stream_true_no_choices_raises_api_error(self): + def test_convert_to_model_response_object_stream_true_no_choices_raises_api_error( + self, + ): """Missing choices via stream=True path raises APIError when generator is consumed.""" from litellm.exceptions import APIError @@ -2471,6 +2478,13 @@ class TestConvertToModelResponseObjectCompletion: def test_model_response_none_raises(self): with pytest.raises(Exception): convert_to_model_response_object( - response_object={"choices": [{"message": {"content": "hi", "role": "assistant"}, "finish_reason": "stop"}]}, + response_object={ + "choices": [ + { + "message": {"content": "hi", "role": "assistant"}, + "finish_reason": "stop", + } + ] + }, model_response_object=None, ) diff --git a/tests/search_tests/test_searchapi_search.py b/tests/search_tests/test_searchapi_search.py index 5ef9d922b89..d16868502a4 100644 --- a/tests/search_tests/test_searchapi_search.py +++ b/tests/search_tests/test_searchapi_search.py @@ -46,10 +46,9 @@ class TestSearchAPIConfig: assert result["Content-Type"] == "application/json" - @patch("litellm.llms.searchapi.search.transformation.get_secret_str") - def test_validate_environment_without_api_key(self, mock_get_secret): + def test_validate_environment_without_api_key(self, monkeypatch): """Test environment validation without API key raises error.""" - mock_get_secret.return_value = None + monkeypatch.delenv("SEARCHAPI_API_KEY", raising=False) config = SearchAPIConfig() headers = {} diff --git a/tests/search_tests/test_searxng_search.py b/tests/search_tests/test_searxng_search.py index 45b0f3214d9..c12d44183b0 100644 --- a/tests/search_tests/test_searxng_search.py +++ b/tests/search_tests/test_searxng_search.py @@ -318,13 +318,11 @@ class TestSearXNGSearchHeaders: assert headers["Content-Type"] == "application/json" assert headers["Authorization"] == "Bearer test-key-123" - def test_headers_with_env_api_key(self): + def test_headers_with_env_api_key(self, monkeypatch): """Test that headers use SEARXNG_API_KEY from env.""" - with patch( - "litellm.llms.searxng.search.transformation.get_secret_str", - return_value="env-key-456", - ): - headers = self.config.validate_environment(headers={}) + monkeypatch.setenv("SEARXNG_API_KEY", "env-key-456") + + headers = self.config.validate_environment(headers={}) assert headers["Authorization"] == "Bearer env-key-456" diff --git a/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py b/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py index f33814b86df..7ff58ba6324 100644 --- a/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py +++ b/tests/test_litellm/integrations/code_interpreter_interception/test_handler.py @@ -1,8 +1,8 @@ """ Unit tests for CodeInterpreterInterceptionLogger. -All sandbox dependencies are injected (dependency injection, no monkeypatch): -a FakeSandbox stands in for the real e2b config and records how it is called. +All sandbox dependencies are injected: a FakeSandbox stands in for the real e2b +config and records how it is called. """ import time @@ -12,13 +12,17 @@ import pytest from litellm.integrations.code_interpreter_interception.handler import ( CodeInterpreterInterceptionLogger, LITELLM_CODE_EXECUTION_TOOL_NAME, + _INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY, + _SANDBOX_KEY, +) +from litellm.types.integrations.custom_logger import ( + CHAT_COMPLETION_AGENTIC_SURFACE, + NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + is_interception_internal_key, ) from litellm.llms.base_llm.sandbox.transformation import CodeExecutionResult from litellm.types.utils import CallTypes -_ACTIVE_KEY = "_code_interpreter_interception_active" -_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key" - class FakeHandle: def __init__(self, sandbox_id="sbx_fake"): @@ -51,6 +55,13 @@ class FakeLogging: def __init__(self, litellm_call_id="k1"): self.litellm_call_id = litellm_call_id self.model_call_details = {} + self.dynamic_success_callbacks = [] + + def pre_call(self, *args, **kwargs): + return None + + def post_call(self, *args, **kwargs): + return None def _function_call_item(call_id="c1", name=LITELLM_CODE_EXECUTION_TOOL_NAME): @@ -62,6 +73,17 @@ def _function_call_item(call_id="c1", name=LITELLM_CODE_EXECUTION_TOOL_NAME): } +def _chat_function_call_item(call_id="call_1", name=LITELLM_CODE_EXECUTION_TOOL_NAME): + return { + "id": call_id, + "type": "function", + "function": { + "name": name, + "arguments": '{"code":"print(40 + 2)"}', + }, + } + + class FakeResponse: def __init__(self, output): self.output = output @@ -74,6 +96,18 @@ def _iter_messages(plan): return patch.messages +def test_interception_internal_key_prefix_sets_preserve_code_interpreter_state(): + assert is_interception_internal_key("_code_interpreter_interception_active") + assert not is_interception_internal_key( + "_code_interpreter_interception_active", + prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + ) + assert is_interception_internal_key( + "_websearch_interception_converted_stream", + prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + ) + + @pytest.mark.asyncio async def test_build_plan_runs_code_and_feeds_output_back(): sandbox = FakeSandbox(stdout="42") @@ -133,6 +167,30 @@ async def test_pre_call_converts_code_interpreter_tool(): assert LITELLM_CODE_EXECUTION_TOOL_NAME in names +@pytest.mark.asyncio +async def test_pre_call_converts_code_interpreter_tool_for_chat_completions(): + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + kwargs = { + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "tool_choice": {"type": "code_interpreter"}, + "custom_llm_provider": "openai", + } + + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion) + + assert result is not None + tool = result["tools"][0] + assert tool["type"] == "function" + assert tool["function"]["name"] == LITELLM_CODE_EXECUTION_TOOL_NAME + assert tool["function"]["parameters"]["required"] == ["code"] + assert result["tool_choice"] == { + "type": "function", + "function": {"name": LITELLM_CODE_EXECUTION_TOOL_NAME}, + } + assert result["litellm_metadata"][_ACTIVE_KEY] is True + assert result["litellm_metadata"][_SANDBOX_KEY] == result[_SANDBOX_KEY] + + @pytest.mark.asyncio @pytest.mark.parametrize( "tool_choice", @@ -184,9 +242,24 @@ async def test_pre_call_noop_on_non_responses(): "custom_llm_provider": "openai", } + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.aembedding) + + assert result is None + + +@pytest.mark.asyncio +async def test_pre_call_noop_on_chat_completion_without_code_interpreter_tool(): + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + kwargs = { + "tools": [{"type": "web_search"}], + "custom_llm_provider": "openai", + } + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion) assert result is None + assert _ACTIVE_KEY not in kwargs + assert _SANDBOX_KEY not in kwargs @pytest.mark.asyncio @@ -524,14 +597,142 @@ async def test_gate_rechecks_provider_scope(): assert should_run is False +@pytest.mark.asyncio +async def test_chat_completion_gate_detects_code_execution_tool_call(): + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + response = { + "choices": [ + {"message": {"tool_calls": [_chat_function_call_item(call_id="call_123")]}} + ] + } + + should_run, payload = await logger.async_should_run_agentic_loop( + response=response, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + tools=[], + stream=False, + custom_llm_provider="openai", + kwargs={ + _ACTIVE_KEY: True, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + }, + ) + + assert should_run is True + assert payload["tool_calls"][0]["id"] == "call_123" + assert payload["tool_calls"][0]["arguments"] == '{"code":"print(40 + 2)"}' + + +@pytest.mark.asyncio +async def test_chat_completion_gate_refuses_without_server_active_marker(): + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + response = {"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]} + + should_run, payload = await logger.async_should_run_agentic_loop( + response=response, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + tools=[], + stream=False, + custom_llm_provider="openai", + kwargs={"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE}, + ) + + assert should_run is False + assert payload == {} + + +@pytest.mark.asyncio +async def test_chat_completion_build_plan_runs_code_and_appends_tool_message(): + sandbox = FakeSandbox(stdout="42") + logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + native_chat_tool = {"type": "code_interpreter", "container": {"type": "auto"}} + + plan = await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "call_1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": '{"code":"print(40 + 2)"}', + } + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response={ + "choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}] + }, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={ + "tools": [native_chat_tool], + "tool_choice": {"type": "code_interpreter", "container": {"type": "auto"}}, + "temperature": 0, + }, + logging_obj=FakeLogging(litellm_call_id="k1"), + stream=False, + kwargs={ + "acompletion": True, + "litellm_call_id": "k1", + _ACTIVE_KEY: True, + _SANDBOX_KEY: "sbxkey1", + "_code_interpreter_interception_converted_stream": True, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + }, + ) + + assert sandbox.run_calls[0]["code"] == "print(40 + 2)" + patch = plan.request_patch + assert patch is not None + assert patch.tools == [ + { + "type": "function", + "function": { + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "description": "Execute python code in a sandbox and return stdout.", + "parameters": { + "type": "object", + "properties": {"code": {"type": "string"}}, + "required": ["code"], + }, + }, + } + ] + assert patch.optional_params == {"temperature": 0} + assert patch.kwargs == { + "litellm_call_id": "k1", + _ACTIVE_KEY: True, + _SANDBOX_KEY: "sbxkey1", + "_code_interpreter_interception_converted_stream": True, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + } + assert patch.messages is not None + assert patch.messages[-2]["role"] == "assistant" + assert patch.messages[-2]["tool_calls"][0]["id"] == "call_1" + assert patch.messages[-1] == { + "role": "tool", + "tool_call_id": "call_1", + "content": "42", + } + assert plan.metadata["code_interpreter_calls"][0]["code"] == "print(40 + 2)" + + @pytest.mark.asyncio async def test_pre_call_strips_client_forged_marker_on_initial_request(): - """A client cannot pre-set the active marker on the original request.""" + """A client cannot pre-set the active marker on the original request: with no + native code_interpreter tool, any client-supplied interception markers in + litellm_metadata are scrubbed and the active flag in kwargs is cleared.""" logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) kwargs = { "tools": [{"type": "web_search"}], "custom_llm_provider": "openai", _ACTIVE_KEY: True, + "litellm_metadata": { + _ACTIVE_KEY: True, + _SANDBOX_KEY: "client-forged", + "safe_user_value": "kept", + }, } await logger.async_pre_call_deployment_hook(kwargs, CallTypes.aresponses) @@ -540,6 +741,42 @@ async def test_pre_call_strips_client_forged_marker_on_initial_request(): "no native code_interpreter tool was present, so a client-supplied " "active marker must be cleared" ) + assert kwargs["litellm_metadata"] == {"safe_user_value": "kept"} + + +@pytest.mark.asyncio +async def test_pre_call_strips_forged_loop_controls_then_mints_own_markers(): + """On an INITIAL request (no server-set _agentic_loop_depth) a client cannot + smuggle loop-control state: forged _agentic_loop_depth / max_agentic_loops and + interception markers in litellm_metadata are stripped before the interceptor + activates, so the only interception markers that survive are the ones the + server mints for the converted code_interpreter tool.""" + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandbox()) + kwargs = { + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}], + "custom_llm_provider": "openai", + "litellm_metadata": { + _ACTIVE_KEY: True, + _SANDBOX_KEY: "client-forged", + "_agentic_loop_depth": 99, + "max_agentic_loops": 999, + "safe_user_value": "kept", + }, + } + + result = await logger.async_pre_call_deployment_hook(kwargs, CallTypes.acompletion) + + assert result is not None + metadata = result["litellm_metadata"] + assert metadata["safe_user_value"] == "kept" + assert "_agentic_loop_depth" not in metadata, "forged loop depth must be stripped" + assert "max_agentic_loops" not in metadata, "forged loop cap must be stripped" + assert metadata[_ACTIVE_KEY] is True + assert metadata[_SANDBOX_KEY] == result[_SANDBOX_KEY] + assert metadata[_SANDBOX_KEY] != "client-forged", ( + "the surviving sandbox key must be the server-minted one, not the forged " + "value the client supplied" + ) @pytest.mark.asyncio diff --git a/tests/test_litellm/integrations/focus/test_mavvrik_destination.py b/tests/test_litellm/integrations/focus/test_mavvrik_destination.py index 797238ae238..3a23dc4ffb2 100644 --- a/tests/test_litellm/integrations/focus/test_mavvrik_destination.py +++ b/tests/test_litellm/integrations/focus/test_mavvrik_destination.py @@ -34,6 +34,12 @@ def _dest(**overrides) -> FocusMavvrikDestination: return FocusMavvrikDestination(prefix="mavvrik_focus_exports", config=config) +def _patch_resp(status: int = 204) -> MagicMock: + r = MagicMock() + r.status_code = status + return r + + def test_missing_api_key_raises(): with pytest.raises(ValueError, match="MAVVRIK_API_KEY"): FocusMavvrikDestination( @@ -127,6 +133,8 @@ async def test_large_content_uploads_in_multiple_chunks(): chunk2_resp = MagicMock() chunk2_resp.status_code = 200 + patch_resp = _patch_resp(204) + mock_http = MagicMock() mock_http.client = MagicMock() mock_http.client.request = AsyncMock( @@ -136,6 +144,7 @@ async def test_large_content_uploads_in_multiple_chunks(): init_resp, chunk1_resp, chunk2_resp, + patch_resp, ] ) dest._http = mock_http @@ -152,15 +161,20 @@ async def test_large_content_uploads_in_multiple_chunks(): filename="usage.csv", ) - # register + get_signed_url + init + 2 chunk PUTs = 5 calls - assert mock_http.client.request.call_count == 5 + # register + get_signed_url + init + 2 chunk PUTs + PATCH = 6 calls + assert mock_http.client.request.call_count == 6 - # Check Content-Range headers - put_calls = mock_http.client.request.call_args_list[3:] + # Check Content-Range headers on the chunk PUTs (calls 3 and 4) + put_calls = mock_http.client.request.call_args_list[3:5] assert "bytes" in put_calls[0].kwargs["headers"]["Content-Range"] assert "/*" in put_calls[0].kwargs["headers"]["Content-Range"] # intermediate assert "/*" not in put_calls[1].kwargs["headers"]["Content-Range"] # final + # Verify the PATCH call advanced metricsMarker + patch_call = mock_http.client.request.call_args_list[5] + assert patch_call.kwargs["method"] == "PATCH" + assert "metricsMarker" in patch_call.kwargs["json"] + @pytest.mark.asyncio async def test_deliver_calls_register_get_url_and_upload(): @@ -180,12 +194,14 @@ async def test_deliver_calls_register_get_url_and_upload(): upload_resp = MagicMock() upload_resp.status_code = 200 + patch_resp = _patch_resp(204) + mock_http = MagicMock() mock_http.client = MagicMock() - # All 4 calls go through self._http.client.request: - # 1. register, 2. get_signed_url, 3. GCS session init POST, 4. GCS PUT + # All 5 calls go through self._http.client.request: + # 1. register, 2. get_signed_url, 3. GCS session init POST, 4. GCS PUT, 5. PATCH marker mock_http.client.request = AsyncMock( - side_effect=[register_resp, signed_url_resp, init_resp, upload_resp] + side_effect=[register_resp, signed_url_resp, init_resp, upload_resp, patch_resp] ) dest._http = mock_http @@ -196,10 +212,14 @@ async def test_deliver_calls_register_get_url_and_upload(): ) assert dest._registered is True - assert mock_http.client.request.call_count == 4 + assert mock_http.client.request.call_count == 5 # Verify Content-Range header was set on the PUT put_call = mock_http.client.request.call_args_list[3] assert "Content-Range" in put_call.kwargs["headers"] + # Verify PATCH was called last with metricsMarker + patch_call = mock_http.client.request.call_args_list[4] + assert patch_call.kwargs["method"] == "PATCH" + assert "metricsMarker" in patch_call.kwargs["json"] @pytest.mark.asyncio @@ -224,17 +244,19 @@ async def test_register_called_only_once_across_multiple_deliveries(): mock_http = MagicMock() mock_http.client = MagicMock() - # First delivery: register, get_signed_url, GCS init, GCS PUT - # Second delivery: get_signed_url, GCS init, GCS PUT (register skipped) + # First delivery: register, get_signed_url, GCS init, GCS PUT, PATCH + # Second delivery: get_signed_url, GCS init, GCS PUT, PATCH (register skipped) mock_http.client.request = AsyncMock( side_effect=[ register_resp, _signed_url_resp(), init_resp, upload_resp, + _patch_resp(204), _signed_url_resp(), init_resp, upload_resp, + _patch_resp(204), ] ) dest._http = mock_http @@ -243,8 +265,8 @@ async def test_register_called_only_once_across_multiple_deliveries(): await dest.deliver(content=b"header\nrow1\n", time_window=window, filename="1.csv") await dest.deliver(content=b"header\nrow2\n", time_window=window, filename="2.csv") - # 7 total: register(1) + [get_url+init+put](2) × 2 deliveries - assert mock_http.client.request.call_count == 7 + # 9 total: register(1) + [get_url+init+put+patch](4) × 2 deliveries + assert mock_http.client.request.call_count == 9 # First call was register first_call = mock_http.client.request.call_args_list[0] assert first_call.kwargs["method"] == "POST" @@ -749,3 +771,43 @@ async def test_gcs_session_cancelled_on_chunk_failure(): delete_call = calls[4] assert delete_call.kwargs["method"] == "DELETE" assert "storage.googleapis.com/session" in delete_call.kwargs["url"] + + +@pytest.mark.asyncio +async def test_update_metrics_marker_warns_on_non_410_error(): + """_update_metrics_marker must log a warning on any >=400 (non-410) status but not raise.""" + dest = _dest() + + fail_resp = MagicMock() + fail_resp.status_code = 500 + fail_resp.text = "Internal Server Error" + + mock_http = MagicMock() + mock_http.client = MagicMock() + mock_http.client.request = AsyncMock(return_value=fail_resp) + dest._http = mock_http + + # Must not raise — warning only + await dest._update_metrics_marker(1234567890) + assert mock_http.client.request.call_count == 1 + + +@pytest.mark.asyncio +async def test_update_metrics_marker_raises_on_410(): + """_update_metrics_marker must raise RuntimeError and reset _registered on 410.""" + dest = _dest() + dest._registered = True + + resp_410 = MagicMock() + resp_410.status_code = 410 + resp_410.text = "Gone" + + mock_http = MagicMock() + mock_http.client = MagicMock() + mock_http.client.request = AsyncMock(return_value=resp_410) + dest._http = mock_http + + with pytest.raises(RuntimeError, match="disconnected"): + await dest._update_metrics_marker(1234567890) + + assert dest._registered is False diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py index 956d8c53cee..7240d49d022 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py @@ -35,6 +35,13 @@ from litellm.integrations.otel.mount import ( # noqa: E402 ) +@pytest.fixture(autouse=True) +def _clear_otel_v2_flag_cache(): + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + class _FakeSpan: """Minimal recording span capturing what the hook writes.""" @@ -70,8 +77,10 @@ def _instrumented_app(): def test_gate_toggles_with_env(monkeypatch): """The startup mount is guarded by this flag.""" monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() assert is_otel_v2_enabled() is False monkeypatch.setenv("LITELLM_OTEL_V2", "1") + is_otel_v2_enabled.cache_clear() assert is_otel_v2_enabled() is True diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 3447f5bdb7e..4bb26a70b02 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -1,6 +1,8 @@ """Tests for the OTel v2 sources of truth: span registry, semconv keys, config, and the typed StandardLoggingPayload adapter. These need no OTel SDK.""" +import pytest + from litellm.integrations.otel import ( BAGGAGE_PROMOTED_KEYS, DB, @@ -28,6 +30,13 @@ from litellm.integrations.otel.model.spans import ( ) +@pytest.fixture(autouse=True) +def _clear_otel_v2_flag_cache(): + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + def _sample_payload(**overrides): payload = { "call_type": "acompletion", @@ -561,11 +570,37 @@ def test_capture_message_content_normalizer_only_touches_strings(): def test_v2_flag_is_off_by_default(monkeypatch): monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() assert is_otel_v2_enabled() is False monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() assert is_otel_v2_enabled() is True +def test_v2_flag_resolved_once_not_per_call(monkeypatch): + """Regression for LIT-3895: ``is_otel_v2_enabled`` sits on the proxy hot path + (auth, logging-callback setup). Building the pydantic-settings model on every + call re-scanned the environment at ~28us a pop and dropped throughput, so the + flag must be resolved once and cached rather than reconstructed per call.""" + from litellm.integrations.otel.model import config as config_mod + + constructions = 0 + real_flag = config_mod._OTelV2Flag + + def _counting_flag(*args, **kwargs): + nonlocal constructions + constructions += 1 + return real_flag(*args, **kwargs) + + monkeypatch.setattr(config_mod, "_OTelV2Flag", _counting_flag) + config_mod.is_otel_v2_enabled.cache_clear() + + for _ in range(50): + config_mod.is_otel_v2_enabled() + + assert constructions == 1 + + def test_config_from_env(monkeypatch): for var in ( "OTEL_EXPORTER", diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 810c6a0ca08..209e99895db 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -156,7 +156,10 @@ class TestResponseCompliance: # The response is the dedicated `Interaction` schema. Google moved the # output-only fields (notably the `steps` array, formerly `outputs`) # off `CreateModelInteractionParams` and onto `Interaction`; the request - # schema no longer carries `steps`. Keep this aligned with the live spec. + # schema no longer carries `steps`. Google later moved `role` off + # `Interaction` onto the per-turn `Turn` schema (asserted in + # test_turn_schema), so it is no longer a top-level output field here. + # Keep this aligned with the live spec. schema = spec_dict["components"]["schemas"]["Interaction"] # Output fields (readOnly). `role` was removed from the `Interaction` diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9b3152fae07..7f3d5a959a1 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1499,19 +1499,20 @@ def _local_model_cost_map(): @pytest.mark.parametrize("data_residency", ["eu", "us"]) def test_data_residency_applies_uplift(data_residency, _local_model_cost_map): - """gpt-5 should apply the regional processing uplift multiplier when - data_residency is set.""" + """gpt-5.4 should apply the regional processing uplift multiplier when + data_residency is set. gpt-5.4+ (released 2026-03-05) carry the 10% uplift; + gpt-5 and older models do not.""" from litellm.types.utils import Usage usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) base = generic_cost_per_token( - model="gpt-5", + model="gpt-5.4", usage=usage, custom_llm_provider="openai", ) regional = generic_cost_per_token( - model="gpt-5", + model="gpt-5.4", usage=usage, custom_llm_provider="openai", data_residency=data_residency, @@ -1526,6 +1527,23 @@ def test_data_residency_applies_uplift(data_residency, _local_model_cost_map): assert regional[1] == pytest.approx(base[1] * 1.10, rel=1e-9) +@pytest.mark.parametrize("model", ["gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5-pro", "gpt-4o", "gpt-4.1"]) +def test_data_residency_no_uplift_for_pre_march_2026_models(model, _local_model_cost_map): + """Models released before 2026-03-05 must not have the regional uplift.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + regional = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai", data_residency="eu" + ) + + assert base == regional, ( + f"{model} should not have a regional uplift, but cost changed with data_residency" + ) + + def test_data_residency_no_uplift_for_unmarked_model(_local_model_cost_map): """A model without a regional_processing_uplift_multiplier_* entry should fall back to base pricing, not error.""" @@ -1555,12 +1573,12 @@ def test_data_residency_none_no_uplift(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) base = generic_cost_per_token( - model="gpt-5", + model="gpt-5.4", usage=usage, custom_llm_provider="openai", ) explicit_none = generic_cost_per_token( - model="gpt-5", + model="gpt-5.4", usage=usage, custom_llm_provider="openai", data_residency=None, @@ -1576,13 +1594,13 @@ def test_data_residency_composes_with_service_tier(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) priority_base = generic_cost_per_token( - model="gpt-5", + model="gpt-5.4", usage=usage, custom_llm_provider="openai", service_tier="priority", ) priority_eu = generic_cost_per_token( - model="gpt-5", + model="gpt-5.4", usage=usage, custom_llm_provider="openai", service_tier="priority", diff --git a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py new file mode 100644 index 00000000000..f1196ab4692 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -0,0 +1,422 @@ +""" +Tests for the provider-agnostic chat completion agentic loop dispatcher +(`litellm/litellm_core_utils/chat_completion_agentic_loop.py`) and the +code-interpreter interception integration that drives it. + +The load-bearing regression here protects a reviewer requirement: the internal +agentic/interception control fields must NEVER reach the outbound provider HTTP +request body. The relevant fields are: + + _agentic_loop_depth + _agentic_loop_fingerprints + _agentic_loop_api_surface + max_agentic_loops + _code_interpreter_interception_active + _code_interpreter_interception_sandbox_key + _code_interpreter_interception_converted_stream + +A scrubber in gpt_transformation.py used to strip these. That scrubber was +removed, so `test_internal_control_fields_never_leak_into_provider_body` proves +they stay out of the body even without it. +""" + +import os +import sys +from typing import Any, Dict, List, Optional, Tuple +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.code_interpreter_interception.handler import ( + CodeInterpreterInterceptionLogger, +) +from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + maybe_run_chat_completion_agentic_loop, +) +from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, +) +from litellm.types.utils import ( + Choices, + Function, + ChatCompletionMessageToolCall, + Message, + ModelResponse, +) + +# The internal control fields that must never reach a provider request body. +_INTERNAL_CONTROL_FIELDS = ( + "_agentic_loop_depth", + "_agentic_loop_fingerprints", + "_agentic_loop_api_surface", + "max_agentic_loops", + "_code_interpreter_interception_active", + "_code_interpreter_interception_sandbox_key", + "_code_interpreter_interception_converted_stream", + "litellm_metadata", +) + + +@pytest.fixture +def restore_callbacks(): + """Save/restore litellm.callbacks so a registered fake logger never pollutes + other tests in the suite.""" + saved = list(litellm.callbacks) + try: + yield + finally: + litellm.callbacks = saved + + +class _SandboxResult: + def __init__(self, stdout: str) -> None: + self.stdout = stdout + self.error = None + + +class FakeSandboxConfig: + """Injected sandbox so the interception loop runs no real network / E2B.""" + + def __init__(self) -> None: + self.created = 0 + self.deleted = 0 + self.run_codes: List[str] = [] + + async def acreate_sandbox(self) -> Any: + self.created += 1 + return MagicMock(id="sandbox-123") + + async def arun_code(self, container: Any, code: str) -> _SandboxResult: + self.run_codes.append(code) + return _SandboxResult(stdout="42\n") + + async def adelete_sandbox(self, container: Any) -> None: + self.deleted += 1 + + +def _tool_call_model_response() -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id="call_abc", + type="function", + function=Function( + name="litellm_code_execution", + arguments='{"code": "print(6*7)"}', + ), + ) + ], + ), + ) + ] + ) + + +def _plain_model_response(content: str = "The answer is 42") -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="stop", + message=Message(role="assistant", content=content), + ) + ] + ) + + +def _raw_response_for(model_response: ModelResponse) -> MagicMock: + """Wrap a ModelResponse as the OpenAI `with_raw_response.create` return value + (an object exposing `.headers` and `.parse()` -> something with model_dump).""" + parsed = MagicMock() + parsed.model_dump.return_value = model_response.model_dump() + raw = MagicMock() + raw.headers = {} + raw.parse.return_value = parsed + return raw + + +# --------------------------------------------------------------------------- +# A) PROVIDER-PAYLOAD REGRESSION +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_internal_control_fields_never_leak_into_provider_body(restore_callbacks): + """Drive a real acompletion with a native code_interpreter tool through the + interception logger + agentic loop, capturing every outbound OpenAI request + body. None of the internal control fields may appear at top-level or inside + extra_body on ANY of the captured calls.""" + logger = CodeInterpreterInterceptionLogger(sandbox_config=FakeSandboxConfig()) + litellm.callbacks = [logger] + + # First create -> model emits a code_execution tool call (triggers the loop). + # Second create -> model returns a plain answer (loop terminates). + create = AsyncMock( + side_effect=[ + _raw_response_for(_tool_call_model_response()), + _raw_response_for(_plain_model_response()), + ] + ) + mock_client = MagicMock() + mock_client.chat.completions.with_raw_response.create = create + + response = await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + tools=[{"type": "code_interpreter"}], + tool_choice={"type": "code_interpreter"}, + api_key="sk-test", + client=mock_client, + ) + + # The loop must have actually fired (sanity: two provider calls). + assert create.await_count == 2, ( + "expected the agentic loop to issue a follow-up provider call; " + f"got {create.await_count} call(s)" + ) + + for idx, call in enumerate(create.await_args_list): + body = call.kwargs + extra_body = body.get("extra_body") or {} + for field in _INTERNAL_CONTROL_FIELDS: + assert field not in body, ( + f"provider call #{idx}: internal field {field!r} leaked into " + f"top-level request body: {sorted(body.keys())}" + ) + assert field not in extra_body, ( + f"provider call #{idx}: internal field {field!r} leaked into " + f"extra_body: {sorted(extra_body.keys())}" + ) + # The native code_interpreter tool must have been swapped for the + # function tool, never sent raw to OpenAI as a chat-completions request. + for tool in body.get("tools") or []: + assert tool.get("type") != "code_interpreter" + + # The final response is the post-loop answer, not the tool-call turn. + assert response.choices[0].message.content == "The answer is 42" + + +# --------------------------------------------------------------------------- +# B) DISPATCHER UNIT TESTS +# --------------------------------------------------------------------------- + + +class _LoggingStub: + """Minimal logging_obj: dispatcher only reads dynamic_success_callbacks and + litellm_call_id off it.""" + + litellm_call_id = "call-test" + dynamic_success_callbacks: List[Any] = [] + + +class _GateOnlyLogger(CustomLogger): + """Overrides the gate to fire, but builds a plan from request_patch.""" + + def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None: + super().__init__() + self._plan = plan + self._tool_calls = tool_calls + self.cleanup_calls = 0 + + async def async_should_run_agentic_loop( + self, + response: Any, + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + stream: bool, + custom_llm_provider: str, + kwargs: Dict[str, Any], + ) -> Tuple[bool, Dict[str, Any]]: + return True, self._tool_calls + + async def async_build_agentic_loop_plan( + self, + tools: Dict[str, Any], + model: str, + messages: List[Dict[str, Any]], + response: Any, + anthropic_messages_provider_config: Any, + anthropic_messages_optional_request_params: Dict[str, Any], + logging_obj: Any, + stream: bool, + kwargs: Dict[str, Any], + ) -> AgenticLoopPlan: + return self._plan + + async def async_agentic_loop_cleanup_hook( + self, plan: AgenticLoopPlan, kwargs: Dict[str, Any] + ) -> None: + self.cleanup_calls += 1 + + +def _patched_messages() -> List[Dict[str, Any]]: + return [ + {"role": "user", "content": "what is 6*7?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "litellm_code_execution", + "arguments": '{"code": "print(6*7)"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_abc", "content": "42\n"}, + ] + + +@pytest.mark.asyncio +async def test_dispatcher_returns_none_when_no_callback_gates(restore_callbacks): + """No callback overrides the gate -> dispatcher returns None so the caller + keeps the original response untouched.""" + litellm.callbacks = [] + + result = await maybe_run_chat_completion_agentic_loop( + response=_plain_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs={}, + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=False, + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messages( + restore_callbacks, +): + """A gating logger with a request_patch -> the dispatcher calls + litellm.acompletion exactly once with _agentic_loop_depth == 1 and the + patched messages. Loop-control state rides as litellm-level kwargs and is + mirrored into litellm_metadata; the provider-surface transient + _agentic_loop_api_surface is never forwarded. (Provider-body stripping of + these litellm-level kwargs is asserted separately in test A.)""" + followup = _plain_model_response("done") + plan = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages()), + ) + logger = _GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]}) + litellm.callbacks = [logger] + + acompletion_mock = AsyncMock(return_value=followup) + with patch.object(litellm, "acompletion", acompletion_mock): + result = await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={"temperature": 0.1}, + kwargs={"_code_interpreter_interception_active": True}, + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=False, + ) + + assert result is followup + acompletion_mock.assert_awaited_once() + call_kwargs = acompletion_mock.await_args.kwargs + + assert call_kwargs["_agentic_loop_depth"] == 1 + assert call_kwargs["messages"] == _patched_messages() + # Preserved non-internal optional param survives the rerun. + assert call_kwargs["temperature"] == 0.1 + # Loop-control state is carried at the litellm level for the follow-up. + assert call_kwargs["max_agentic_loops"] >= 1 + assert "_agentic_loop_fingerprints" in call_kwargs + # Interception markers are mirrored into litellm_metadata for the follow-up. + assert ( + call_kwargs["litellm_metadata"]["_code_interpreter_interception_active"] is True + ) + # The transient surface marker is NOT forwarded to the follow-up call. + assert "_agentic_loop_api_surface" not in call_kwargs + # Cleanup hook always runs. + assert logger.cleanup_calls == 1 + + +@pytest.mark.asyncio +async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops( + restore_callbacks, +): + """depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops, + before any follow-up call is attempted.""" + logger = _GateOnlyLogger( + plan=AgenticLoopPlan(run_agentic_loop=True), + tool_calls={"tool_calls": [{"id": "call_abc"}]}, + ) + litellm.callbacks = [logger] + + acompletion_mock = AsyncMock() + with patch.object(litellm, "acompletion", acompletion_mock): + with pytest.raises(ValueError, match="max_agentic_loops"): + await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=False, + ) + + acompletion_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks): + """A tool_calls fingerprint already present in _agentic_loop_fingerprints -> + ValueError about the repeated fingerprint (cycle guard), with no follow-up + call.""" + import json + + # The dispatcher fingerprints the whole value the gate returns as its second + # tuple element, so the seeded fingerprint must mirror that dict exactly. + gate_tool_calls = { + "tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}] + } + fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str) + + logger = _GateOnlyLogger( + plan=AgenticLoopPlan(run_agentic_loop=True), + tool_calls=gate_tool_calls, + ) + litellm.callbacks = [logger] + + acompletion_mock = AsyncMock() + with patch.object(litellm, "acompletion", acompletion_mock): + with pytest.raises(ValueError, match="fingerprint"): + await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs={ + "_agentic_loop_depth": 0, + "max_agentic_loops": 3, + "_agentic_loop_fingerprints": [fingerprint], + }, + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=False, + ) + + acompletion_mock.assert_not_awaited() diff --git a/tests/test_litellm/litellm_core_utils/test_request_timeout_resolver.py b/tests/test_litellm/litellm_core_utils/test_request_timeout_resolver.py new file mode 100644 index 00000000000..4e016622f1c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_request_timeout_resolver.py @@ -0,0 +1,58 @@ +"""Unit tests for litellm.litellm_core_utils.request_timeout_resolver. + +The resolver decides whether ``litellm.request_timeout`` was *explicitly configured* +(env REQUEST_TIMEOUT / litellm_settings, or a non-default runtime value) versus left +at the package default. This is what lets request_timeout act as an independent +per-attempt timeout instead of being indistinguishable from "nobody set it". +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))) + +import litellm +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, +) + + +@pytest.fixture +def restore_request_timeout(): + original_value = litellm.request_timeout + original_flag = litellm.request_timeout_explicitly_set + try: + yield + finally: + litellm.request_timeout = original_value + litellm.request_timeout_explicitly_set = original_flag + + +def test_default_value_without_flag_is_unset(restore_request_timeout): + litellm.request_timeout = DEFAULT_REQUEST_TIMEOUT_SECONDS + litellm.request_timeout_explicitly_set = False + assert get_configured_request_timeout() is None + + +def test_explicit_flag_returns_value(restore_request_timeout): + litellm.request_timeout = 300 + litellm.request_timeout_explicitly_set = True + assert get_configured_request_timeout() == 300.0 + + +def test_explicit_flag_preserves_value_equal_to_default(restore_request_timeout): + # The case the bare ``!= default`` heuristic gets wrong: a user who explicitly + # configures the default value still means it explicitly. + litellm.request_timeout = DEFAULT_REQUEST_TIMEOUT_SECONDS + litellm.request_timeout_explicitly_set = True + assert get_configured_request_timeout() == float(DEFAULT_REQUEST_TIMEOUT_SECONDS) + + +def test_non_default_runtime_value_treated_as_explicit(restore_request_timeout): + # SDK users assigning litellm.request_timeout directly (no flag) must keep working. + litellm.request_timeout = 300 + litellm.request_timeout_explicitly_set = False + assert get_configured_request_timeout() == 300.0 diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 860d235acd2..81af0ad3e6f 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2754,7 +2754,9 @@ def test_chunk_creator_tool_calls_not_dropped_on_finish( tool_calls=[ ChatCompletionDeltaToolCall( id="call_abc", - function=Function(name="get_weather", arguments='{"city":"NYC"}'), + function=Function( + name="get_weather", arguments='{"city":"NYC"}' + ), type="function", index=0, ) @@ -2849,3 +2851,131 @@ def test_record_partial_usage_for_failure_noop_without_chunks(): wrapper._record_partial_usage_for_failure() assert "combined_usage_object" not in logging_obj.model_call_details + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_stream_chunk_builder_raise_at_end_of_stream_still_recovers_usage( + sync_mode, +): + """stream_chunk_builder re-raises (as APIError) on large agentic tool-use + streams. That raise originates inside the except-StopIteration handler, so + before the fix it escaped __next__/__anext__ and the request was dropped from + SpendLogs while the provider billed the tokens. The wrapper must catch it and + recover usage from the raw chunks so cost is still tracked.""" + final_usage_block = Usage( + completion_tokens=392, prompt_tokens=1799, total_tokens=2191 + ) + final_chunk = ModelResponseStream( + id="chatcmpl-raise-test", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="", role="assistant"), + ) + ], + usage=final_usage_block, + ) + test_chunks = bedrock_chunks + [final_chunk] + + logging_obj = Logging( + model="bedrock/claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="raise-test", + function_id="1245", + ) + + response = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=test_chunks), + model="bedrock/claude-haiku-4-5-20251001-v1:0", + custom_llm_provider="bedrock", + logging_obj=logging_obj, + stream_options={"include_usage": True}, + ) + + seen_usage = [] + with patch.object( + litellm, + "stream_chunk_builder", + side_effect=Exception("simulated assembly failure"), + ): + # before the fix this raised and dropped the request; it must not raise now + if sync_mode: + for chunk in response: + if getattr(chunk, "usage", None) is not None: + seen_usage.append(chunk.usage) + else: + async for chunk in response: + if getattr(chunk, "usage", None) is not None: + seen_usage.append(chunk.usage) + + assert any( + u.total_tokens == final_usage_block.total_tokens for u in seen_usage + ), "usage recovered from raw chunks was not emitted after stream_chunk_builder raised" + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_stream_chunk_builder_raise_and_usage_recovery_failure_does_not_crash( + sync_mode, +): + """If end-of-stream assembly raises AND best-effort usage recovery from the raw + chunks also fails, the stream must still complete cleanly rather than propagate + the exception to the consumer.""" + from litellm.litellm_core_utils import streaming_handler as sh_module + + final_chunk = ModelResponseStream( + id="chatcmpl-raise-recover-fail", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="", role="assistant"), + ) + ], + usage=Usage(completion_tokens=1, prompt_tokens=1, total_tokens=2), + ) + + response = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=bedrock_chunks + [final_chunk] + ), + model="bedrock/claude-haiku-4-5-20251001-v1:0", + custom_llm_provider="bedrock", + logging_obj=Logging( + model="bedrock/claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="raise-recover-fail", + function_id="1245", + ), + stream_options={"include_usage": True}, + ) + + with ( + patch.object( + litellm, "stream_chunk_builder", side_effect=Exception("assembly failed") + ), + patch.object( + sh_module, "calculate_total_usage", side_effect=Exception("recovery failed") + ), + ): + # must not raise even though both assembly and recovery fail + if sync_mode: + chunks = [c for c in response] + else: + chunks = [c async for c in response] + + assert len(chunks) > 0 diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 2fdd639e74d..0b1aaf87516 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -74,6 +74,137 @@ def test_redacted_thinking_content_block_delta(): assert "thinking_blocks" in model_response.choices[0].delta.provider_specific_fields +def test_streaming_thinking_blocks_are_replayable_after_signature_delta(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=False + ) + chunks = [ + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "Step 1. "}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "Step 2."}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "sig-final"}, + }, + ] + + parsed_chunks = [ + model_response_iterator.chunk_parser(chunk=chunk) for chunk in chunks + ] + reasoning_content = "".join( + getattr(chunk.choices[0].delta, "reasoning_content", None) or "" + for chunk in parsed_chunks + ) + thinking_blocks = tuple( + block + for chunk in parsed_chunks + for block in (getattr(chunk.choices[0].delta, "thinking_blocks", None) or []) + ) + expected_thinking_block = { + "type": "thinking", + "thinking": "Step 1. Step 2.", + "signature": "sig-final", + } + + assert reasoning_content == "Step 1. Step 2." + assert thinking_blocks == (expected_thinking_block,) + assert parsed_chunks[-1].choices[0].delta.provider_specific_fields == { + "thinking_blocks": [expected_thinking_block] + } + + +def test_streaming_unsigned_thinking_deltas_keep_reasoning_content(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=False + ) + chunks = [ + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "Step 1. "}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "Step 2."}, + }, + {"type": "content_block_stop", "index": 0}, + ] + + parsed_chunks = [ + model_response_iterator.chunk_parser(chunk=chunk) for chunk in chunks + ] + reasoning_content = "".join( + getattr(chunk.choices[0].delta, "reasoning_content", None) or "" + for chunk in parsed_chunks + ) + thinking_blocks = tuple( + block + for chunk in parsed_chunks + for block in (getattr(chunk.choices[0].delta, "thinking_blocks", None) or []) + ) + + assert reasoning_content == "Step 1. Step 2." + assert thinking_blocks == () + + +def test_streaming_truncated_thinking_deltas_keep_reasoning_content(): + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=True, json_mode=False + ) + chunks = [ + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "Step 1. "}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "Step 2."}, + }, + ] + + parsed_chunks = [ + model_response_iterator.chunk_parser(chunk=chunk) for chunk in chunks + ] + reasoning_content = "".join( + getattr(chunk.choices[0].delta, "reasoning_content", None) or "" + for chunk in parsed_chunks + ) + thinking_blocks = tuple( + block + for chunk in parsed_chunks + for block in (getattr(chunk.choices[0].delta, "thinking_blocks", None) or []) + ) + + assert reasoning_content == "Step 1. Step 2." + assert thinking_blocks == () + + def test_handle_json_mode_chunk_response_format_tool(): model_response_iterator = ModelResponseIterator( streaming_response=MagicMock(), sync_stream=True, json_mode=True diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 2876b56f516..b111b65e3af 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1886,6 +1886,89 @@ def test_anthropic_model_supports_effort_param_rejects_non_supporting_models(mod assert AnthropicConfig._model_supports_effort_param(model) is False +@pytest.mark.parametrize( + "model", + [ + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-4-6-20260205", + "claude-opus-4-7-20260416", + ], +) +def test_anthropic_model_supports_speed_param_recognizes_supporting_models(model): + assert AnthropicConfig._model_supports_speed_param(model) is True + + +@pytest.mark.parametrize( + "model", + [ + "claude-sonnet-4-6", + "claude-fable-5", + "claude-3-haiku-20240307", + "vertex_ai/claude-opus-4-8", + "azure_ai/claude-opus-4-8", + "anthropic.claude-opus-4-8", + ], +) +def test_anthropic_model_supports_speed_param_rejects_non_supporting_models(model): + assert AnthropicConfig._model_supports_speed_param(model) is False + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "azure_ai", "bedrock"]) +def test_anthropic_model_supports_speed_param_rejects_non_anthropic_providers( + custom_llm_provider, +): + """Fast mode is direct-Anthropic-only. Vertex/Azure/Bedrock strip their prefix + before the shared transform runs, so the bare Opus id must still be rejected.""" + assert ( + AnthropicConfig._model_supports_speed_param( + "claude-opus-4-8", custom_llm_provider + ) + is False + ) + assert ( + AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic") + is True + ) + + +def test_vertex_anthropic_drops_speed_for_opus_with_drop_params(monkeypatch): + """Regression: vertex_ai Opus must drop ``speed`` even though the prefix-stripped + ``claude-opus-4-8`` maps to a fast-mode-capable direct-Anthropic entry.""" + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( + VertexAIAnthropicConfig, + ) + + monkeypatch.setattr(litellm, "drop_params", True) + result = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"speed": "fast", "max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + + assert "speed" not in result + + +def test_vertex_anthropic_raises_on_speed_without_drop_params(monkeypatch): + """Regression: vertex_ai Opus raises rather than forwarding an unsupported + ``speed`` when neither global nor per-request drop_params is set.""" + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( + VertexAIAnthropicConfig, + ) + + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + VertexAIAnthropicConfig().map_openai_params( + non_default_params={"speed": "fast"}, + optional_params={}, + model="claude-opus-4-8", + drop_params=False, + ) + + def test_translate_system_message_skips_empty_string_content(): """ Test that translate_system_message skips system messages with empty string content. @@ -3766,6 +3849,61 @@ def test_fast_mode_parameter_mapping(): assert result["speed"] == "fast" +def test_anthropic_drop_params_strips_speed_for_unsupported_models(): + """``drop_params=True`` strips unsupported ``speed`` for non-Opus models.""" + config = AnthropicConfig() + messages = [{"role": "user", "content": "Hello"}] + + original = litellm.drop_params + litellm.drop_params = True + try: + result = config.transform_request( + model="claude-sonnet-4-6", + messages=messages, + optional_params={"speed": "fast", "max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + finally: + litellm.drop_params = original + + assert "speed" not in result + + +def test_anthropic_drop_params_keeps_speed_for_supporting_models(): + """``drop_params=True`` must not strip ``speed`` on Opus fast-mode models.""" + config = AnthropicConfig() + messages = [{"role": "user", "content": "Hello"}] + + original = litellm.drop_params + litellm.drop_params = True + try: + result = config.transform_request( + model="claude-opus-4-6", + messages=messages, + optional_params={"speed": "fast", "max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + finally: + litellm.drop_params = original + + assert result.get("speed") == "fast" + + +def test_speed_raises_clean_error_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + config = AnthropicConfig() + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + config.map_openai_params( + non_default_params={"speed": "fast"}, + optional_params={}, + model="claude-sonnet-4-6", + drop_params=False, + ) + + def test_map_openai_params_max_tokens_normalized_to_int(): """ Test that map_openai_params normalizes max_tokens to an integer (e.g. 0.7 -> 1). diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py new file mode 100644 index 00000000000..6900f1062bf --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py @@ -0,0 +1,117 @@ +import litellm +import pytest +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + AnthropicMessagesRequestUtils, +) + + +def test_messages_drop_params_strips_speed_for_unsupported_models(): + original = litellm.drop_params + litellm.drop_params = True + try: + optional_params = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={ + "max_tokens": 1024, + "speed": "fast", + "messages": [{"role": "user", "content": "Hello"}], + }, + model="claude-sonnet-4-6", + drop_params=False, + ) + config = AnthropicMessagesConfig() + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params=dict(optional_params), + litellm_params={}, + ) + result = config.transform_anthropic_messages_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=dict(optional_params), + litellm_params={}, + headers=headers, + ) + finally: + litellm.drop_params = original + + assert "speed" not in optional_params + assert "speed" not in result + assert "fast-mode-2026-02-01" not in headers.get("anthropic-beta", "") + + +def test_messages_drop_params_keeps_speed_for_supporting_models(): + original = litellm.drop_params + litellm.drop_params = True + try: + optional_params = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"max_tokens": 1024, "speed": "fast"}, + model="claude-opus-4-6", + drop_params=False, + ) + config = AnthropicMessagesConfig() + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params=dict(optional_params), + litellm_params={}, + ) + result = config.transform_anthropic_messages_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params=dict(optional_params), + litellm_params={}, + headers=headers, + ) + finally: + litellm.drop_params = original + + assert optional_params.get("speed") == "fast" + assert result.get("speed") == "fast" + assert "fast-mode-2026-02-01" in headers.get("anthropic-beta", "") + + +def test_messages_raises_when_speed_unsupported_and_drop_params_false(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"max_tokens": 1024, "speed": "fast"}, + model="claude-sonnet-4-6", + drop_params=False, + ) + + +def test_messages_drops_speed_for_vertex_opus_with_drop_params(monkeypatch): + """Regression: a vertex_ai Opus passthrough must drop ``speed`` even though the + prefix-stripped model id maps to a fast-mode-capable direct-Anthropic entry.""" + monkeypatch.setattr(litellm, "drop_params", True) + optional_params = ( + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"max_tokens": 1024, "speed": "fast"}, + model="claude-opus-4-8", + drop_params=False, + custom_llm_provider="vertex_ai", + ) + ) + + assert "speed" not in optional_params + + +def test_messages_raises_for_vertex_opus_without_drop_params(monkeypatch): + """Regression: vertex_ai Opus passthrough raises rather than forwarding an + unsupported ``speed`` when drop_params is unset.""" + monkeypatch.setattr(litellm, "drop_params", False) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"max_tokens": 1024, "speed": "fast"}, + model="claude-opus-4-8", + drop_params=False, + custom_llm_provider="vertex_ai", + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py index 3ce076640e8..f0252e13336 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py @@ -6,6 +6,7 @@ Regression tests for the /v1/messages request-parse fast paths: while resolving the (static) type hints only once per process. """ +import litellm from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( AnthropicMessagesRequestUtils, _anthropic_messages_optional_param_keys, @@ -54,3 +55,36 @@ def test_empty_params(): ) == {} ) + + +def test_drop_params_strips_speed_for_unsupported_model(): + original = litellm.drop_params + litellm.drop_params = True + try: + result = ( + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"speed": "fast", "temperature": 0.5}, + model="claude-sonnet-4-6", + ) + ) + finally: + litellm.drop_params = original + + assert result == {"temperature": 0.5} + assert "speed" not in result + + +def test_drop_params_keeps_speed_for_supporting_model(): + original = litellm.drop_params + litellm.drop_params = True + try: + result = ( + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"speed": "fast"}, + model="claude-opus-4-6", + ) + ) + finally: + litellm.drop_params = original + + assert result == {"speed": "fast"} diff --git a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py index 2cb7b4db3d4..31047d30970 100644 --- a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py @@ -516,3 +516,217 @@ async def test_max_uses_none_falls_back_to_default(): ) assert str(_c.ADVISOR_MAX_USES) in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# 12. Defense-in-depth: client-supplied advisor api_base/api_key are dropped +# unless the proxy admin opted into clientside credentials +# --------------------------------------------------------------------------- + + +ADVISOR_TOOL_WITH_CREDS = { + "type": "advisor_20260301", + "name": "advisor", + "model": "claude-opus-4-6", + "api_base": "https://other.example", + "api_key": "sk-other", +} + + +async def _run_advisor_and_capture_subcall_kwargs(): + """Run one advisor turn and return the kwargs of the advisor sub-call.""" + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + advisor_tool_use_resp = _make_advisor_tool_use_response(tool_id="toolu_01") + advisor_advice_resp = _make_text_response("advice", model="claude-opus-4-6") + final_resp = _make_text_response("final answer") + + captured = {} + call_count = 0 + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return advisor_tool_use_resp + if call_count == 2: + # The advisor sub-call — capture its routing kwargs. + captured["api_key"] = kwargs.get("api_key") + captured["api_base"] = kwargs.get("api_base") + return advisor_advice_resp + return final_resp + + with patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ): + h = AdvisorOrchestrationHandler() + await h.handle( + model="openai/gpt-4o-mini", + messages=MESSAGES, + tools=[ADVISOR_TOOL_WITH_CREDS], + stream=False, + max_tokens=512, + custom_llm_provider="openai", + ) + return captured + + +@pytest.mark.asyncio +async def test_advisor_creds_dropped_when_proxy_opt_in_disabled(): + """On the proxy without opt-in, the caller's advisor api_base/api_key must + NOT reach the sub-call (would redirect it / leak the server key).""" + with patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + return_value=False, + ): + captured = await _run_advisor_and_capture_subcall_kwargs() + assert captured["api_key"] is None + assert captured["api_base"] is None + + +@pytest.mark.asyncio +async def test_advisor_creds_honored_when_proxy_opt_in_enabled(): + """With the admin opt-in, the documented clientside routing still works.""" + with patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + return_value=True, + ): + captured = await _run_advisor_and_capture_subcall_kwargs() + assert captured["api_key"] == "sk-other" + assert captured["api_base"] == "https://other.example" + + +# --------------------------------------------------------------------------- +# 13. The proxy gate itself: _allow_client_side_advisor_credentials() and the +# full handle() driven by the real proxy general_settings flag. +# --------------------------------------------------------------------------- + + +def _fake_proxy_server(general_settings: Dict): + """A stand-in litellm.proxy.proxy_server module exposing general_settings. + + The real proxy_server pulls in heavy optional deps that may be absent in a + unit-test environment, so the gate's + ``from litellm.proxy.proxy_server import general_settings`` is satisfied by + injecting this lightweight module into sys.modules. + """ + import types + + module = types.ModuleType("litellm.proxy.proxy_server") + module.general_settings = general_settings # type: ignore[attr-defined] + return module + + +def test_allow_client_side_advisor_credentials_reads_proxy_flag(): + """The gate mirrors the proxy's allow_client_side_credentials opt-in.""" + import sys + + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + _allow_client_side_advisor_credentials, + ) + + cases = ( + ({"allow_client_side_credentials": True}, True), + ({"allow_client_side_credentials": False}, False), + # Flag absent entirely -> default deny on the proxy. + ({}, False), + ) + for settings, expected in cases: + with patch.dict( + sys.modules, + {"litellm.proxy.proxy_server": _fake_proxy_server(settings)}, + ): + assert _allow_client_side_advisor_credentials() is expected + + +def test_allow_client_side_advisor_credentials_defaults_true_outside_proxy(): + """Outside the proxy (proxy_server import unavailable), there is no admin + boundary, so the gate permits client-supplied routing.""" + import builtins + import sys + + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + _allow_client_side_advisor_credentials, + ) + + real_import = builtins.__import__ + + def _blocked_import(name, *args, **kwargs): + if name == "litellm.proxy.proxy_server": + raise ImportError("proxy server unavailable") + return real_import(name, *args, **kwargs) + + with patch.dict(sys.modules): + sys.modules.pop("litellm.proxy.proxy_server", None) + with patch.object(builtins, "__import__", _blocked_import): + assert _allow_client_side_advisor_credentials() is True + + +def test_advisor_gate_propagates_non_import_errors(): + """Non-ImportError failures during the proxy module probe must not + default permissive. If the proxy is partially loaded and raises + RuntimeError, the gate should surface that rather than silently + returning True.""" + import sys + + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors import ( + advisor, + ) + + original = sys.modules.get("litellm.proxy.proxy_server") + + class _Broken: + def __getattr__(self, _name): + raise RuntimeError("partial proxy boot") + + sys.modules["litellm.proxy.proxy_server"] = _Broken() + try: + with pytest.raises(RuntimeError, match="partial proxy boot"): + advisor._allow_client_side_advisor_credentials() + finally: + if original is None: + sys.modules.pop("litellm.proxy.proxy_server", None) + else: + sys.modules["litellm.proxy.proxy_server"] = original + + +@pytest.mark.asyncio +async def test_advisor_ignores_tool_credentials_when_clientside_disabled(): + """Driven by the real proxy flag (not a patched gate): with + allow_client_side_credentials False, the tool-supplied api_base/api_key must + not reach the advisor sub-call.""" + import sys + + with patch.dict( + sys.modules, + { + "litellm.proxy.proxy_server": _fake_proxy_server( + {"allow_client_side_credentials": False} + ) + }, + ): + captured = await _run_advisor_and_capture_subcall_kwargs() + assert captured["api_key"] is None + assert captured["api_base"] is None + + +@pytest.mark.asyncio +async def test_advisor_uses_tool_credentials_when_clientside_enabled(): + """Driven by the real proxy flag: with allow_client_side_credentials True, + the tool-supplied api_base/api_key flow through to the advisor sub-call.""" + import sys + + with patch.dict( + sys.modules, + { + "litellm.proxy.proxy_server": _fake_proxy_server( + {"allow_client_side_credentials": True} + ) + }, + ): + captured = await _run_advisor_and_capture_subcall_kwargs() + assert captured["api_key"] == "sk-other" + assert captured["api_base"] == "https://other.example" diff --git a/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py b/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py index 32838701949..bc26268ee92 100644 --- a/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py +++ b/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py @@ -66,9 +66,8 @@ class TestAPISerpentConfig: assert headers["X-API-Key"] == "test-api-key" assert headers["Content-Type"] == "application/json" - @patch("litellm.llms.apiserpent.search.transformation.get_secret_str") - def test_validate_environment_without_api_key(self, mock_get_secret): - mock_get_secret.return_value = None + def test_validate_environment_without_api_key(self, monkeypatch): + monkeypatch.delenv("APISERPENT_API_KEY", raising=False) with pytest.raises(ValueError, match="APISERPENT_API_KEY is not set"): APISerpentSearchConfig().validate_environment({}) diff --git a/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py b/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py new file mode 100644 index 00000000000..a1353d57038 --- /dev/null +++ b/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py @@ -0,0 +1,329 @@ +""" +Regression tests for the host-aware server-credential fallback guard in +``BaseSearchConfig``. + +A caller-supplied ``api_base`` is honored when building the request URL, so +falling back to a server-configured secret while the caller controls the host +would send the operator's credential to an attacker. The guard must refuse that +combination for every provider that carries a server-managed secret, while +leaving keyless providers and legitimate operator overrides untouched. +""" + +from typing import Dict, Tuple, Type +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm.llms.apiserpent.search.transformation import APISerpentSearchConfig +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + _is_trusted_search_api_base, +) +from litellm.llms.brave.search.transformation import BraveSearchConfig +from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig +from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig +from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig +from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig +from litellm.llms.google_pse.search.transformation import GooglePSESearchConfig +from litellm.llms.linkup.search.transformation import LinkupSearchConfig +from litellm.llms.parallel_ai.search.transformation import ParallelAISearchConfig +from litellm.llms.perplexity.search.transformation import PerplexitySearchConfig +from litellm.llms.searchapi.search.transformation import SearchAPIConfig +from litellm.llms.searxng.search.transformation import SearXNGSearchConfig +from litellm.llms.serper.search.transformation import SerperSearchConfig +from litellm.llms.tavily.search.transformation import TavilySearchConfig +from litellm.llms.tinyfish.search.transformation import TinyfishSearchConfig +from litellm.llms.you_com.search.transformation import YouComSearchConfig + +ATTACKER_BASE = "https://attacker.example.com" + +# Every *_API_BASE override env var that could otherwise mark the attacker host +# as trusted; cleared before each test so the suite is hermetic. +_BASE_ENV_VARS = ( + "SERPER_API_BASE", + "TAVILY_API_BASE", + "PERPLEXITY_API_BASE", + "APISERPENT_API_BASE", + "EXA_API_BASE", + "BRAVE_API_BASE", + "FIRECRAWL_API_BASE", + "LINKUP_API_BASE", + "SEARCHAPI_API_BASE", + "GOOGLE_PSE_API_BASE", + "PARALLEL_AI_API_BASE", + "YOUCOM_API_BASE", + "SEARXNG_API_BASE", + "DATAFORSEO_API_BASE", + "TINYFISH_API_BASE", + "CRW_API_BASE", +) + + +@pytest.fixture(autouse=True) +def _clear_base_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + for var in _BASE_ENV_VARS: + monkeypatch.delenv(var, raising=False) + + +# (config, {server secret env vars}, caller_api_key honored as-is, extra env for full validate) +ProviderSpec = Tuple[Type[BaseSearchConfig], Dict[str, str], str, Dict[str, str]] + +PROVIDERS: Tuple[ProviderSpec, ...] = ( + (SerperSearchConfig, {"SERPER_API_KEY": "srv"}, "caller-key", {}), + (TavilySearchConfig, {"TAVILY_API_KEY": "srv"}, "caller-key", {}), + (PerplexitySearchConfig, {"PERPLEXITYAI_API_KEY": "srv"}, "caller-key", {}), + (APISerpentSearchConfig, {"APISERPENT_API_KEY": "srv"}, "caller-key", {}), + (ExaAISearchConfig, {"EXA_API_KEY": "srv"}, "caller-key", {}), + (BraveSearchConfig, {"BRAVE_API_KEY": "srv"}, "caller-key", {}), + (FirecrawlSearchConfig, {"FIRECRAWL_API_KEY": "srv"}, "caller-key", {}), + (LinkupSearchConfig, {"LINKUP_API_KEY": "srv"}, "caller-key", {}), + (SearchAPIConfig, {"SEARCHAPI_API_KEY": "srv"}, "caller-key", {}), + ( + GooglePSESearchConfig, + {"GOOGLE_PSE_API_KEY": "srv"}, + "caller-key", + {"GOOGLE_PSE_ENGINE_ID": "engine"}, + ), + (ParallelAISearchConfig, {"PARALLEL_API_KEY": "srv"}, "caller-key", {}), + (YouComSearchConfig, {"YOUCOM_API_KEY": "srv"}, "caller-key", {}), + (SearXNGSearchConfig, {"SEARXNG_API_KEY": "srv"}, "caller-key", {}), + ( + DataForSEOSearchConfig, + {"DATAFORSEO_LOGIN": "srv", "DATAFORSEO_PASSWORD": "pw"}, + "login:password", + {}, + ), + (TinyfishSearchConfig, {"TINYFISH_API_KEY": "srv"}, "caller-key", {}), + (FastCRWSearchConfig, {"CRW_API_KEY": "srv"}, "caller-key", {}), +) + +_IDS = tuple(spec[0].__name__ for spec in PROVIDERS) + + +@pytest.mark.parametrize( + "config_cls, server_env, caller_key, extra_env", PROVIDERS, ids=_IDS +) +def test_server_secret_refused_for_caller_api_base( + config_cls: Type[BaseSearchConfig], + server_env: Dict[str, str], + caller_key: str, + extra_env: Dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + for key, value in {**server_env, **extra_env}.items(): + monkeypatch.setenv(key, value) + + with pytest.raises(ValueError, match="Refusing to send the server-configured"): + config_cls().validate_environment(headers={}, api_base=ATTACKER_BASE) + + +@pytest.mark.parametrize( + "config_cls, server_env, caller_key, extra_env", PROVIDERS, ids=_IDS +) +def test_caller_supplied_key_is_honored_for_custom_api_base( + config_cls: Type[BaseSearchConfig], + server_env: Dict[str, str], + caller_key: str, + extra_env: Dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + for key, value in {**server_env, **extra_env}.items(): + monkeypatch.setenv(key, value) + + # An explicit caller key is the caller's own credential, so pointing it at + # the caller's own host must be allowed. + config_cls().validate_environment( + headers={}, api_key=caller_key, api_base=ATTACKER_BASE + ) + + +@pytest.mark.parametrize( + "config_cls, server_env, caller_key, extra_env", PROVIDERS, ids=_IDS +) +def test_server_secret_used_without_caller_api_base( + config_cls: Type[BaseSearchConfig], + server_env: Dict[str, str], + caller_key: str, + extra_env: Dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + for key, value in {**server_env, **extra_env}.items(): + monkeypatch.setenv(key, value) + + # No caller-supplied api_base -> the request targets the trusted default, so + # the server secret is still used and nothing is refused. + config_cls().validate_environment(headers={}) + + +def test_keyless_provider_allows_caller_api_base( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SEARXNG_API_KEY", raising=False) + + headers = SearXNGSearchConfig().validate_environment( + headers={}, api_base="https://my-searxng.internal" + ) + + assert "Authorization" not in headers + + +def test_operator_env_base_override_is_trusted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SERPER_API_KEY", "srv") + monkeypatch.setenv("SERPER_API_BASE", "https://serper.internal.corp") + + # Mirrors the second validate_environment call in the search handler, which + # receives the already-resolved operator base as api_base. + headers = SerperSearchConfig().validate_environment( + headers={}, api_base="https://serper.internal.corp/search" + ) + + assert headers["X-API-KEY"] == "srv" + + +class TestResolveServerApiKey: + def test_caller_key_short_circuits(self) -> None: + result = BaseSearchConfig().resolve_server_api_key( + caller_api_key="mine", + caller_api_base=ATTACKER_BASE, + key_env_vars=("SERPER_API_KEY",), + base_env_var="SERPER_API_BASE", + default_api_base="https://google.serper.dev", + ) + assert result == "mine" + + def test_returns_none_when_no_server_secret( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("SEARXNG_API_KEY", raising=False) + result = BaseSearchConfig().resolve_server_api_key( + caller_api_key=None, + caller_api_base=ATTACKER_BASE, + key_env_vars=("SEARXNG_API_KEY",), + base_env_var="SEARXNG_API_BASE", + default_api_base=None, + ) + assert result is None + + def test_first_set_env_var_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "second") + result = BaseSearchConfig().resolve_server_api_key( + caller_api_key=None, + caller_api_base=None, + key_env_vars=("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"), + base_env_var="PARALLEL_AI_API_BASE", + default_api_base="https://api.parallel.ai", + ) + assert result == "second" + + +class TestIsTrustedSearchApiBase: + def test_matches_default_host(self) -> None: + assert _is_trusted_search_api_base( + "https://google.serper.dev/search", "https://google.serper.dev", None + ) + + def test_foreign_host_untrusted(self) -> None: + assert not _is_trusted_search_api_base( + ATTACKER_BASE, "https://google.serper.dev", None + ) + + def test_env_override_host_trusted(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SERPER_API_BASE", "https://serper.internal.corp") + assert _is_trusted_search_api_base( + "https://serper.internal.corp/search", + "https://google.serper.dev", + "SERPER_API_BASE", + ) + + def test_schemeless_candidate_untrusted(self) -> None: + # Without a scheme urlsplit puts the value in the path, leaving an empty + # netloc; an unparseable host must never be treated as trusted. + assert not _is_trusted_search_api_base( + "attacker.example.com", "https://google.serper.dev", None + ) + + +@pytest.mark.asyncio +async def test_asearch_does_not_leak_server_key_to_caller_api_base( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """End-to-end regression on the reported vector: a search call with a foreign + api_base and no caller key must fail without any outbound request carrying the + server-configured key.""" + monkeypatch.setenv("SERPER_API_KEY", "sk-server-secret") + monkeypatch.delenv("SERPER_API_BASE", raising=False) + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post, + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + new_callable=AsyncMock, + ) as mock_get, + ): + with pytest.raises(Exception): + await litellm.asearch( + query="secrets", + search_provider="serper", + api_base=ATTACKER_BASE, + ) + + mock_post.assert_not_called() + mock_get.assert_not_called() + + +@pytest.mark.parametrize( + "provider, key_env, server_key, extra_env", + [ + ("searchapi", "SEARCHAPI_API_KEY", "sk-server-searchapi", {}), + ( + "google_pse", + "GOOGLE_PSE_API_KEY", + "sk-server-google", + {"GOOGLE_PSE_ENGINE_ID": "engine-id"}, + ), + ], +) +@pytest.mark.asyncio +async def test_query_param_key_not_leaked_with_dummy_caller_key( + provider: str, + key_env: str, + server_key: str, + extra_env: Dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Providers that send the key as a URL query param resolve it in + transform_search_request, not validate_environment. A caller who passes a + dummy api_key to clear the validate_environment short-circuit must not cause + the server key to be placed in the URL sent to their own api_base.""" + monkeypatch.setenv(key_env, server_key) + for name, value in extra_env.items(): + monkeypatch.setenv(name, value) + + captured: Dict[str, str] = {} + + async def fake_get(self, *args, **kwargs): # type: ignore[no-untyped-def] + captured["url"] = kwargs.get("url") or (args[0] if args else "") + raise RuntimeError("stop after capturing the outbound url") + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + fake_get, + ): + with pytest.raises(Exception): + await litellm.asearch( + query="secrets", + search_provider=provider, + api_key="sk-CALLER-DUMMY", + api_base=ATTACKER_BASE, + ) + + assert captured["url"], "expected an outbound request to be attempted" + assert server_key not in captured["url"] + assert "sk-CALLER-DUMMY" in captured["url"] diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index 3f91f6ac26e..2d5242d510f 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -163,6 +163,134 @@ def test_aws_profile_path_not_cached_in_iam_cache(): assert mock_profile.call_count == 2 +def test_get_credentials_does_not_expand_request_env_reference(): + """ + A parameter of the form os.environ/ reaching get_credentials is left as-is + rather than expanded against the process environment, so the downstream auth + helper only ever receives the literal value. + """ + env = _os_environ_without_aws_keys() + env["SERVER_ONLY_VALUE"] = "config-managed-value" + base = BaseAWSLLM() + with patch.dict(os.environ, env, clear=True), patch.object( + base, + "_auth_with_aws_profile", + return_value=(Credentials("ak", "sk", None), None), + ) as mock_profile: + base.get_credentials(aws_profile_name="os.environ/SERVER_ONLY_VALUE") + + assert mock_profile.call_args.args[0] == "os.environ/SERVER_ONLY_VALUE" + assert "config-managed-value" not in str(mock_profile.call_args) + + +def test_get_credentials_falls_back_to_ambient_aws_profile_name_env(): + """ + The fixed AWS_* ambient fallback keeps working: an unset aws_profile_name + resolves from the AWS_PROFILE_NAME environment variable. + """ + env = _os_environ_without_aws_keys() + env["AWS_PROFILE_NAME"] = "ambient-profile" + base = BaseAWSLLM() + with patch.dict(os.environ, env, clear=True), patch.object( + base, + "_auth_with_aws_profile", + return_value=(Credentials("ak", "sk", None), None), + ) as mock_profile: + base.get_credentials(aws_profile_name=None) + + assert mock_profile.call_args.args[0] == "ambient-profile" + + +def test_get_credentials_ambient_fallback_resolves_aws_external_id(): + """ + Each unset param falls back to its own AWS_* env var. Regression for an index + misalignment between the value list and the env-name list, which left + AWS_EXTERNAL_ID unresolved. + """ + env = _os_environ_without_aws_keys() + env["AWS_EXTERNAL_ID"] = "ext-from-env" + base = BaseAWSLLM() + with patch.dict(os.environ, env, clear=True), patch.object( + base, + "_auth_with_aws_role", + return_value=(Credentials("ak", "sk", "tok"), None), + ) as mock_role: + base.get_credentials( + aws_role_name="arn:aws:iam::123456789012:role/x", + aws_session_name="s", + ) + + assert mock_role.call_args.kwargs["aws_external_id"] == "ext-from-env" + + +def _capturing_sts_client(captured: Dict[str, Any]) -> MagicMock: + sts = MagicMock() + + def _assume(**params): + captured["WebIdentityToken"] = params.get("WebIdentityToken") + return { + "Credentials": { + "AccessKeyId": "AKIA", + "SecretAccessKey": "sk", + "SessionToken": "tok", + }, + "PackedPolicySize": 10, + } + + sts.assume_role_with_web_identity.side_effect = _assume + return sts + + +@pytest.mark.parametrize( + "token_ref", + ["os.environ/SERVER_ONLY_VALUE", "SERVER_ONLY_VALUE"], + ids=["os_environ_prefix", "bare_env_name"], +) +def test_web_identity_token_env_reference_not_expanded(token_ref): + """ + A web-identity token that is an environment-variable reference (an os.environ/ + prefix, or a bare name matching an env var) is rejected rather than expanded, so + the process-environment value is never used as the token. + """ + env = _os_environ_without_aws_keys() + env["SERVER_ONLY_VALUE"] = "server-only-value" + captured: Dict[str, Any] = {} + base = BaseAWSLLM() + with patch.dict(os.environ, env, clear=True), patch( + "boto3.client", side_effect=lambda *a, **k: _capturing_sts_client(captured) + ), patch("boto3.Session", return_value=MagicMock()): + with pytest.raises(AwsAuthError): + base.get_credentials( + aws_web_identity_token=token_ref, + aws_role_name="arn:aws:iam::123456789012:role/x", + aws_session_name="s", + aws_sts_endpoint="https://custom-sts.example", + ) + + assert "server-only-value" not in str(captured) + + +def test_web_identity_token_oidc_reference_still_resolved(): + """ + The env-reference guard does not over-reject: an oidc/ reference still flows to + get_secret (mocked to None here), surfacing the existing 401 rather than the 400 + used for rejected env-var references. + """ + base = BaseAWSLLM() + env = _os_environ_without_aws_keys() + with patch.dict(os.environ, env, clear=True), patch( + "litellm.llms.bedrock.base_aws_llm.get_secret", return_value=None + ): + with pytest.raises(AwsAuthError) as exc: + base.get_credentials( + aws_web_identity_token="oidc/circleci/", + aws_role_name="arn:aws:iam::123456789012:role/x", + aws_session_name="s", + ) + + assert exc.value.status_code == 401 + + def test_web_identity_path_not_cached_in_iam_cache(): base = BaseAWSLLM() with patch.object( diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index 6298eeb25e9..3893ca474db 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -158,6 +158,68 @@ def test_deepseek_cris(): assert bedrock_route == "converse" +def test_application_inference_profile_arn_routes_to_converse(): + """ + Regression for #18258: a bare application-inference-profile ARN passed as + `bedrock/arn:...` must route to converse. The ARN ends in an opaque id with + no provider substring, so the invoke path cannot build a provider-native + body and raises "Unknown provider=None". Converse needs no provider, so it + is the correct route. + """ + route = BedrockModelInfo.get_bedrock_route( + model="bedrock/arn:aws:bedrock:us-west-2:123412341234:application-inference-profile/a1b2c3" + ) + assert route == "converse" + + +def test_explicit_invoke_prefix_wins_over_application_inference_profile_arn(): + """ + An explicit invoke/ prefix is respected even for an application-inference-profile + ARN; only the bare `bedrock/arn:...` form is auto-routed to converse. The + explicit invoke path remains a dead end for these ARNs (no provider can be + derived, so completion raises "Unknown provider=None") by design: a caller + that explicitly asks for invoke gets invoke. The auto-route only rescues the + documented bare form. + """ + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model = "bedrock/invoke/arn:aws:bedrock:us-west-2:123412341234:application-inference-profile/a1b2c3" + assert BedrockModelInfo.get_bedrock_route(model) == "invoke" + assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None + + +def test_system_defined_inference_profile_arn_still_routes_to_converse(): + """ + A system-defined cross-region inference-profile ARN embeds a known model, so + get_base_model resolves it and it already routes to converse. Guards that the + application-inference-profile fix does not change this working case. + """ + route = BedrockModelInfo.get_bedrock_route( + model="bedrock/arn:aws:bedrock:us-east-1:123:inference-profile/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + ) + assert route == "converse" + + +def test_other_opaque_arn_types_still_route_to_invoke(): + """ + Only application-inference-profile ARNs are auto-routed to converse. Other + opaque ARNs (provisioned-model, imported-model, custom-model-deployment) + also yield no invoke provider, but they are frequently invoke-only with + provider-specific body formats, so routing them to converse could break + them. Guards the deliberate scope against an over-broad "any opaque ARN -> + converse" generalization. + """ + for arn_segment in ( + "provisioned-model/abcdefgh1234", + "imported-model/abcdefgh1234", + "custom-model-deployment/abcdefgh1234", + ): + route = BedrockModelInfo.get_bedrock_route( + model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" + ) + assert route == "invoke", f"{arn_segment} should stay on invoke route" + + def test_govcloud_cross_region_inference_prefix(): """ Test that GovCloud models with cross-region inference prefix (us-gov.) are parsed correctly diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/test_litellm/llms/bedrock/test_mantle.py index bbefdd621f0..f7f8f582abc 100644 --- a/tests/test_litellm/llms/bedrock/test_mantle.py +++ b/tests/test_litellm/llms/bedrock/test_mantle.py @@ -125,6 +125,74 @@ def test_mantle_messages_url_construction(): assert url == "https://bedrock-mantle.us-east-1.api.aws/anthropic/v1/messages" +_VPC_ENDPOINT = "https://vpce-0a1b2c3d.bedrock-mantle.us-gov-west-1.vpce.amazonaws.com" + + +def test_mantle_chat_url_honors_api_base_host(): + config = AmazonMantleConfig() + url = config.get_complete_url( + api_base=_VPC_ENDPOINT, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={"aws_region_name": "us-gov-west-1"}, + litellm_params={}, + ) + assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + + +def test_mantle_chat_url_honors_api_base_full_path_without_duplication(): + config = AmazonMantleConfig() + full = f"{_VPC_ENDPOINT}/anthropic/v1/messages" + url = config.get_complete_url( + api_base=full, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={"aws_region_name": "us-gov-west-1"}, + litellm_params={}, + ) + assert url == full + + +def test_mantle_messages_url_honors_api_base_host(): + config = AmazonMantleMessagesConfig() + url = config.get_complete_url( + api_base=_VPC_ENDPOINT, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={"aws_region_name": "us-gov-west-1"}, + litellm_params={}, + ) + assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + assert "api.aws" not in url + + +def test_mantle_messages_url_honors_api_base_with_trailing_slash(): + config = AmazonMantleMessagesConfig() + url = config.get_complete_url( + api_base=f"{_VPC_ENDPOINT}/", + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={"aws_region_name": "us-gov-west-1"}, + litellm_params={}, + ) + assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + + +def test_mantle_messages_url_honors_aws_bedrock_runtime_endpoint(): + config = AmazonMantleMessagesConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={ + "aws_region_name": "us-gov-west-1", + "aws_bedrock_runtime_endpoint": _VPC_ENDPOINT, + }, + litellm_params={}, + ) + assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + + def test_mantle_transform_request_strips_prefix_and_adds_model(): config = AmazonMantleConfig() request = config.transform_request( @@ -247,3 +315,35 @@ async def test_mantle_anthropic_messages_sends_workspace_header_and_clean_body() assert requests[0]["path"] == "/anthropic/v1/messages" assert requests[0]["headers"]["anthropic-workspace"] == "proj_abc123def456" assert "aws_bedrock_project_id" not in requests[0]["body"] + + +@pytest.mark.asyncio +async def test_mantle_anthropic_messages_routes_to_vpc_api_base(): + import litellm + + urls = [] + + async def mock_post(self, url, data=None, headers=None, **kwargs): + urls.append(str(url)) + return _anthropic_response(str(url)) + + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): + await litellm.anthropic_messages( + model="bedrock/mantle/anthropic.claude-mythos-preview", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + api_base=_VPC_ENDPOINT, + aws_access_key_id="fake-key", + aws_secret_access_key="fake-secret", + aws_region_name="us-gov-west-1", + ) + finally: + await litellm.close_litellm_async_clients() + + assert len(urls) == 1 + assert urls[0] == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + assert "api.aws" not in urls[0] diff --git a/tests/test_litellm/llms/cloudflare/test_cloudflare_transformation.py b/tests/test_litellm/llms/cloudflare/test_cloudflare_transformation.py index cecb6024de1..1a46015ceb9 100644 --- a/tests/test_litellm/llms/cloudflare/test_cloudflare_transformation.py +++ b/tests/test_litellm/llms/cloudflare/test_cloudflare_transformation.py @@ -3,25 +3,190 @@ import pytest from litellm.llms.cloudflare.chat.transformation import CloudflareChatConfig -def test_get_complete_url_encodes_model_path_segment(): +def test_supported_params_include_tools_and_tool_choice(): config = CloudflareChatConfig() - assert ( - config.get_complete_url( - api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run/", - api_key="cf-key", - model="@cf/meta/llama?x=1#frag", - optional_params={}, - litellm_params={}, - ) - == "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/%40cf/meta/llama%3Fx%3D1%23frag" + params = config.get_supported_openai_params(model="@cf/meta/llama-2-7b-chat-int8") + + assert "tools" in params + assert "tool_choice" in params + assert "stream" in params + assert "max_tokens" in params + + +def test_get_complete_url_defaults_to_openai_compatible_endpoint(monkeypatch): + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + config = CloudflareChatConfig() + + url = config.get_complete_url( + api_base=None, + api_key="cf-key", + model="@cf/meta/llama-2-7b-chat-int8", + optional_params={}, + litellm_params={}, ) - with pytest.raises(ValueError, match="dot path segment"): + assert ( + url + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1/chat/completions" + ) + assert "/ai/run/" not in url + + +def test_get_complete_url_appends_chat_completions_to_explicit_base(): + config = CloudflareChatConfig() + + url = config.get_complete_url( + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1", + api_key="cf-key", + model="@cf/meta/llama-2-7b-chat-int8", + optional_params={}, + litellm_params={}, + ) + + assert ( + url + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1/chat/completions" + ) + assert "/ai/run/" not in url + + +def test_get_complete_url_is_idempotent_for_full_base(): + config = CloudflareChatConfig() + + url = config.get_complete_url( + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1/chat/completions", + api_key="cf-key", + model="@cf/meta/llama-2-7b-chat-int8", + optional_params={}, + litellm_params={}, + ) + + assert ( + url + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1/chat/completions" + ) + + +def test_get_complete_url_falls_back_to_account_id_when_base_is_empty(monkeypatch): + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + config = CloudflareChatConfig() + + url = config.get_complete_url( + api_base="", + api_key="cf-key", + model="@cf/meta/llama-2-7b-chat-int8", + optional_params={}, + litellm_params={}, + ) + + assert ( + url + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1/chat/completions" + ) + + +def test_get_complete_url_raises_when_account_id_and_base_missing(monkeypatch): + monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) + config = CloudflareChatConfig() + + with pytest.raises(ValueError, match="Missing CLOUDFLARE_ACCOUNT_ID"): config.get_complete_url( - api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run/", + api_base=None, api_key="cf-key", - model="../../accounts/other", + model="@cf/meta/llama-2-7b-chat-int8", optional_params={}, litellm_params={}, ) + + +def test_get_complete_url_raises_when_account_id_is_empty(monkeypatch): + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", " ") + config = CloudflareChatConfig() + + with pytest.raises(ValueError, match="Missing CLOUDFLARE_ACCOUNT_ID"): + config.get_complete_url( + api_base=None, + api_key="cf-key", + model="@cf/meta/llama-2-7b-chat-int8", + optional_params={}, + litellm_params={}, + ) + + +def test_get_complete_url_migrates_legacy_ai_run_base(): + config = CloudflareChatConfig() + + url = config.get_complete_url( + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run/", + api_key="cf-key", + model="@cf/meta/llama-2-7b-chat-int8", + optional_params={}, + litellm_params={}, + ) + + assert ( + url + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1/chat/completions" + ) + assert "/ai/run" not in url + + +def test_transform_request_passes_tools_through_in_openai_format(): + config = CloudflareChatConfig() + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + messages = [{"role": "user", "content": "weather in nyc?"}] + + body = config.transform_request( + model="@cf/meta/llama-2-7b-chat-int8", + messages=messages, + optional_params={"tools": tools, "tool_choice": "auto"}, + litellm_params={}, + headers={}, + ) + + assert body["messages"] == messages + assert body["model"] == "@cf/meta/llama-2-7b-chat-int8" + assert body["tools"] == tools + assert body["tool_choice"] == "auto" + + +def test_validate_environment_requires_api_key(): + config = CloudflareChatConfig() + + with pytest.raises(ValueError, match="Missing Cloudflare API Key"): + config.validate_environment( + headers={}, + model="@cf/meta/llama-2-7b-chat-int8", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + +def test_validate_environment_sets_bearer_and_content_type(): + config = CloudflareChatConfig() + + headers = config.validate_environment( + headers={}, + model="@cf/meta/llama-2-7b-chat-int8", + messages=[], + optional_params={}, + litellm_params={}, + api_key="cf-key", + ) + + assert headers["Authorization"] == "Bearer cf-key" + assert headers["Content-Type"] == "application/json" diff --git a/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py b/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py index 88072cd7760..46c37e6af6c 100644 --- a/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py +++ b/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py @@ -2,10 +2,8 @@ Unit tests for Cohere Rerank Guardrail Translation Handler """ -import asyncio import os import sys -from typing import List, Optional, Tuple import pytest @@ -94,6 +92,74 @@ class TestInputProcessing: "id": "doc2", } + @pytest.mark.asyncio + async def test_process_query_and_instruction(self): + """Both query and instruction are guardrailed; documents untouched""" + handler = CohereRerankHandler() + guardrail = MockGuardrail(guardrail_name="test") + + data = { + "model": "qwen3-reranker", + "query": "What is machine learning?", + "instruction": "Rank by relevance to ML research", + "documents": ["Doc 1", "Doc 2"], + } + + result = await handler.process_input_messages(data, guardrail) + + # Both user-controlled text fields are scanned and written back + assert result["query"] == "What is machine learning? [GUARDRAILED]" + assert result["instruction"] == "Rank by relevance to ML research [GUARDRAILED]" + # Documents unchanged + assert result["documents"] == ["Doc 1", "Doc 2"] + + @pytest.mark.asyncio + async def test_instruction_masked_with_pii(self): + """A masking guardrail rewrites instruction, not just query""" + + class PIIMaskingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: dict, request_data: dict, input_type: str, **kwargs + ) -> dict: + texts = inputs.get("texts", []) + return {"texts": [t.replace("John Doe", "[NAME_REDACTED]") for t in texts]} + + handler = CohereRerankHandler() + guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") + + data = { + "model": "qwen3-reranker", + "query": "find records", + "instruction": "prioritize anything authored by John Doe", + "documents": ["Doc 1"], + } + + result = await handler.process_input_messages(data, guardrail) + + # The sensitive value in instruction is sanitized before forwarding + assert "John Doe" not in result["instruction"] + assert "[NAME_REDACTED]" in result["instruction"] + assert result["documents"] == ["Doc 1"] + + @pytest.mark.asyncio + async def test_non_string_instruction_not_scanned(self): + """A non-string instruction is left as-is (only strings are scanned)""" + handler = CohereRerankHandler() + guardrail = MockGuardrail(guardrail_name="test") + + data = { + "model": "qwen3-reranker", + "query": "hello", + "instruction": 12345, # invalid type; backend will reject it + "documents": ["Doc 1"], + } + + result = await handler.process_input_messages(data, guardrail) + + # Query still guardrailed; non-string instruction untouched + assert result["query"] == "hello [GUARDRAILED]" + assert result["instruction"] == 12345 + @pytest.mark.asyncio async def test_process_no_query(self): """Test processing when query is missing""" diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index bf835b5d8f9..7bd1d7a6031 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -851,3 +851,56 @@ async def test_async_get_forwards_per_request_timeout(): } finally: await handler.close() + + +class TestDefaultCachedClientTimeoutHonorsRequestTimeout: + """Cached default httpx clients must fall back to an explicit litellm.request_timeout. + + Regression for LIT-2369: get_async_httpx_client / _get_httpx_client hardcoded a + 600s default and never consulted litellm.request_timeout, so provider calls with + no per-model timeout (e.g. Bedrock) hung for 600s. + """ + + @pytest.fixture + def restore_request_timeout(self): + original_value = litellm.request_timeout + original_flag = litellm.request_timeout_explicitly_set + try: + yield + finally: + litellm.request_timeout = original_value + litellm.request_timeout_explicitly_set = original_flag + + def test_default_when_request_timeout_unset(self, restore_request_timeout): + from litellm.llms.custom_httpx.http_handler import ( + _DEFAULT_TIMEOUT, + _default_cached_client_timeout, + ) + + litellm.request_timeout = litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS + litellm.request_timeout_explicitly_set = False + assert _default_cached_client_timeout() is _DEFAULT_TIMEOUT + + def test_uses_explicit_request_timeout(self, restore_request_timeout): + from litellm.llms.custom_httpx.http_handler import ( + _default_cached_client_timeout, + ) + + litellm.request_timeout = 300 + litellm.request_timeout_explicitly_set = True + resolved = _default_cached_client_timeout() + assert resolved.read == 300.0 + assert resolved.connect == 5.0 + + def test_cached_async_client_built_with_explicit_request_timeout( + self, restore_request_timeout + ): + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + litellm.request_timeout = 300 + litellm.request_timeout_explicitly_set = True + litellm.in_memory_llm_clients_cache = LLMClientCache() + client = get_async_httpx_client(llm_provider=LlmProviders.BEDROCK) + assert client.timeout.read == 300.0 diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index f7d445d0788..64ae30daa70 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -10,13 +10,21 @@ sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path import litellm +from litellm.integrations.code_interpreter_interception.handler import ( + CodeInterpreterInterceptionLogger, + LITELLM_CODE_EXECUTION_TOOL_NAME, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import ( BaseLLMHTTPHandler, _google_genai_streaming_hidden_params, ) +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams +_ACTIVE_KEY = "_code_interpreter_interception_active" +_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key" + def test_prepare_fake_stream_request(): # Initialize the BaseLLMHTTPHandler @@ -116,6 +124,117 @@ def test_response_api_handler_streams_when_provider_transform_adds_stream(): assert client.post.call_args.kwargs["json"]["stream"] is True +def test_response_api_handler_runs_agentic_hooks_in_sync_path(monkeypatch): + handler = BaseLLMHTTPHandler() + config = Mock() + config.validate_environment.return_value = {} + config.get_complete_url.return_value = "https://chatgpt.example.com/responses" + config.transform_responses_api_request.return_value = { + "model": "gpt-5", + "input": "hi", + } + config.sign_request.return_value = ({}, None) + initial_response = Mock() + final_response = Mock() + config.transform_response_api_response.return_value = initial_response + + client = HTTPHandler(client=httpx.Client()) + client.post = Mock( + return_value=httpx.Response( + 200, + request=httpx.Request("POST", "https://chatgpt.example.com/responses"), + ) + ) + logging_obj = Mock() + + monkeypatch.setattr(handler, "_has_agentic_completion_hook", Mock(return_value=True)) + hook_mock = AsyncMock(return_value=final_response) + monkeypatch.setattr(handler, "_call_agentic_completion_hooks", hook_mock) + + response = handler.response_api_handler( + model="gpt-5", + input="hi", + responses_api_provider_config=config, + response_api_optional_request_params={}, + custom_llm_provider="openai", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + client=client, + ) + + assert response is final_response + hook_mock.assert_awaited_once() + assert hook_mock.call_args.kwargs["api_surface"] == "responses" + assert hook_mock.call_args.kwargs["messages"] == [ + {"role": "user", "content": "hi"} + ] + + +def test_response_api_handler_runs_responses_pre_call_hook_before_transform(): + handler = BaseLLMHTTPHandler() + config = Mock() + config.validate_environment.return_value = {} + config.get_complete_url.return_value = "https://api.openai.com/v1/responses" + config.sign_request.return_value = ({}, None) + initial_response = ResponsesAPIResponse( + id="resp_1", + created_at=0, + output=[], + status="completed", + model="gpt-5", + ) + config.transform_response_api_response.return_value = initial_response + + def transform_responses_api_request(**kwargs): + return { + "model": kwargs["model"], + "input": kwargs["input"], + **kwargs["response_api_optional_request_params"], + } + + config.transform_responses_api_request.side_effect = transform_responses_api_request + client = HTTPHandler(client=httpx.Client()) + client.post = Mock( + return_value=httpx.Response( + 200, + request=httpx.Request("POST", "https://api.openai.com/v1/responses"), + ) + ) + logging_obj = Mock() + logging_obj.dynamic_success_callbacks = [] + + old_callbacks = list(litellm.callbacks) + litellm.callbacks = [CodeInterpreterInterceptionLogger()] + try: + response = handler.response_api_handler( + model="gpt-5", + input="use code", + responses_api_provider_config=config, + response_api_optional_request_params={ + "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}] + }, + custom_llm_provider="openai", + litellm_params=GenericLiteLLMParams(api_key="sk-test"), + logging_obj=logging_obj, + client=client, + ) + finally: + litellm.callbacks = old_callbacks + + assert response is initial_response + transform_kwargs = config.transform_responses_api_request.call_args.kwargs + tools = transform_kwargs["response_api_optional_request_params"]["tools"] + assert not any(tool.get("type") == "code_interpreter" for tool in tools) + assert any( + tool.get("type") == "function" + and tool.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME + for tool in tools + ) + hook_litellm_params = transform_kwargs["litellm_params"] + assert hook_litellm_params.get(_ACTIVE_KEY) is True + assert hook_litellm_params.get(_SANDBOX_KEY) + + @pytest.mark.asyncio async def test_async_response_api_handler_streams_when_provider_transform_adds_stream(): handler = BaseLLMHTTPHandler() diff --git a/tests/test_litellm/llms/deepseek/chat/__init__.py b/tests/test_litellm/llms/deepseek/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py new file mode 100644 index 00000000000..ec51e5d303d --- /dev/null +++ b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -0,0 +1,103 @@ +from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig + + +def _function_tool(name: str) -> dict: + return { + "type": "function", + "function": {"name": name, "parameters": {"type": "object"}}, + } + + +def test_drop_unsupported_tools_keeps_function_tools_only(): + optional_params = { + "tools": [ + _function_tool("shell"), + {"type": "namespace", "name": "container.exec"}, + _function_tool("apply_patch"), + ], + "tool_choice": "auto", + } + + result = DeepSeekChatConfig._drop_unsupported_tools(optional_params) + + assert [tool["function"]["name"] for tool in result["tools"]] == [ + "shell", + "apply_patch", + ] + assert all(tool["type"] == "function" for tool in result["tools"]) + assert result["tool_choice"] == "auto" + + +def test_drop_unsupported_tools_drops_dangling_tool_choice_when_none_survive(): + optional_params = { + "tools": [{"type": "namespace", "name": "container.exec"}], + "tool_choice": "required", + "parallel_tool_calls": True, + "temperature": 0.2, + } + + result = DeepSeekChatConfig._drop_unsupported_tools(optional_params) + + assert "tools" not in result + assert "tool_choice" not in result + assert "parallel_tool_calls" not in result + assert result["temperature"] == 0.2 + + +def test_drop_unsupported_tools_is_noop_for_function_only(): + optional_params = { + "tools": [_function_tool("shell")], + "tool_choice": "auto", + } + + result = DeepSeekChatConfig._drop_unsupported_tools(optional_params) + + assert result is optional_params + + +def test_drop_unsupported_tools_is_noop_without_tools(): + optional_params = {"temperature": 0.7} + + result = DeepSeekChatConfig._drop_unsupported_tools(optional_params) + + assert result is optional_params + + +def test_transform_request_strips_unsupported_tools_from_body(): + config = DeepSeekChatConfig() + body = config.transform_request( + model="deepseek-chat", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "tools": [ + _function_tool("shell"), + {"type": "namespace", "name": "container.exec"}, + ], + "tool_choice": "auto", + }, + litellm_params={}, + headers={}, + ) + + assert [tool["type"] for tool in body["tools"]] == ["function"] + assert body["tools"][0]["function"]["name"] == "shell" + + +async def test_async_transform_request_strips_unsupported_tools_from_body(): + config = DeepSeekChatConfig() + body = await config.async_transform_request( + model="deepseek-chat", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "tools": [ + _function_tool("shell"), + {"type": "namespace", "name": "container.exec"}, + ], + "tool_choice": "auto", + }, + litellm_params={}, + headers={}, + ) + + assert [tool["type"] for tool in body["tools"]] == ["function"] + assert body["tools"][0]["function"]["name"] == "shell" diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index 5673ad81551..f69ba7df938 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -878,3 +878,107 @@ class TestGithubCopilotTransformResponse: litellm_params={}, encoding=None, ) + + +class TestGithubCopilotTransformParsedResponseDict: + """ + Tests for GithubCopilotConfig.transform_parsed_response_dict, the hook the + OpenAI SDK handler calls on its parsed response. That handler bypasses + transform_response, so this is the seam that repairs empty-choices responses + from newer Copilot Claude models on the live completion path. + + See: https://github.com/BerriAI/litellm/issues/30927 + """ + + def test_synthesizes_choices_from_anthropic_content(self): + config = GithubCopilotConfig() + + parsed = { + "id": "msg_vrtx_01", + "model": "claude-opus-4.8", + "object": "chat.completion", + "choices": [], + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + repaired = config.transform_parsed_response_dict(parsed) + + assert len(repaired["choices"]) == 1 + choice = repaired["choices"][0] + assert choice["message"]["content"] == "Hello!" + assert choice["finish_reason"] == "stop" + assert repaired["usage"]["prompt_tokens"] == 10 + assert repaired["usage"]["completion_tokens"] == 5 + assert repaired["usage"]["total_tokens"] == 15 + + def test_passthrough_when_choices_present(self): + config = GithubCopilotConfig() + + parsed = { + "id": "chatcmpl-1", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + } + + assert config.transform_parsed_response_dict(parsed) is parsed + + +@patch("litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client") +@patch( + "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request" +) +def test_openai_handler_repairs_github_copilot_empty_choices( + mock_request, mock_get_client +): + """ + The OpenAI SDK handler calls convert_to_model_response_object directly on the + SDK's parsed output, bypassing transform_response. convert raises APIError on + empty choices, so the handler must route github_copilot responses through + transform_parsed_response_dict first. Removing that wiring (or resolving a + config without the override) fails this test with APIError. + + See: https://github.com/BerriAI/litellm/issues/30927 + """ + from litellm.llms.openai.openai import OpenAIChatCompletion + + mock_get_client.return_value = MagicMock() + + class _FakeSDKResponse: + def model_dump(self): + return { + "id": "msg_vrtx_01", + "model": "claude-opus-4.8", + "object": "chat.completion", + "choices": [], + "content": [{"type": "text", "text": "Hi there"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 3}, + } + + mock_request.return_value = ({}, _FakeSDKResponse()) + + result = OpenAIChatCompletion().completion( + model="claude-opus-4.8", + messages=[{"role": "user", "content": "Hi"}], + model_response=ModelResponse(), + timeout=60.0, + optional_params={}, + litellm_params={}, + logging_obj=MagicMock(), + custom_llm_provider="github_copilot", + client=MagicMock(), + api_key="gh.test-key-123456789", + acompletion=False, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hi there" + assert result.choices[0].finish_reason == "stop" + mock_request.assert_called_once() diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index 9e6fa608c50..6425e815db0 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -4,6 +4,7 @@ import sys import pytest from litellm.llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig +from litellm.rerank_api.rerank_utils import get_optional_rerank_params from litellm.types.rerank import ( OptionalRerankParams, RerankBilledUnits, @@ -37,6 +38,54 @@ class TestHostedVLLMRerankTransform: assert params["rank_fields"] == ["field1"] assert params["return_documents"] is True + def test_map_cohere_rerank_params_omits_instruction_when_absent(self): + # Backward-compat: when no instruction is supplied, it must not appear + # in the mapped params (and therefore not in the outgoing request body). + params = self.config.map_cohere_rerank_params( + non_default_params=None, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + assert "instruction" not in params + + def test_map_cohere_rerank_params_passes_instruction_when_set(self): + params = self.config.map_cohere_rerank_params( + non_default_params=None, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + instruction="Rank by relevance to genomics", + ) + assert params["instruction"] == "Rank by relevance to genomics" + + def test_transform_request_includes_instruction_when_set(self): + body = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={ + "query": "test query", + "documents": ["doc1", "doc2"], + "instruction": "Rank by relevance to genomics", + }, + headers={}, + ) + assert body["instruction"] == "Rank by relevance to genomics" + + def test_transform_request_omits_instruction_when_absent(self): + # exclude_none must drop the field entirely so the body matches the + # pre-existing (instruction-less) shape exactly. + body = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={ + "query": "test query", + "documents": ["doc1", "doc2"], + }, + headers={}, + ) + assert "instruction" not in body + def test_map_cohere_rerank_params_raises_on_max_chunks_per_doc(self): with pytest.raises( ValueError, match="Hosted VLLM does not support max_chunks_per_doc" @@ -74,6 +123,7 @@ class TestHostedVLLMRerankTransform: } result = self.config._transform_response(response_dict) assert result.id == "abc123" + assert result.results is not None assert len(result.results) == 2 assert result.results[0]["index"] == 0 assert result.results[0]["relevance_score"] == 0.9 @@ -94,3 +144,32 @@ class TestHostedVLLMRerankTransform: } with pytest.raises(ValueError, match="Missing required fields in the result="): self.config._transform_response(response_dict) + + +class TestGetOptionalRerankParamsInstruction: + """`instruction` is threaded through get_optional_rerank_params only when set.""" + + def setup_method(self): + self.config = HostedVLLMRerankConfig() + self.model = "hosted-vllm-model" + + def test_instruction_threaded_when_set(self): + params = get_optional_rerank_params( + rerank_provider_config=self.config, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + instruction="Rank by relevance to genomics", + ) + assert params["instruction"] == "Rank by relevance to genomics" + + def test_instruction_absent_when_not_set(self): + params = get_optional_rerank_params( + rerank_provider_config=self.config, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + assert "instruction" not in params diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py index 95ade4290e9..417dd4a767c 100644 --- a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -305,6 +305,34 @@ class TestMoonshotConfig: assert len(result["messages"]) == 2 assert result["messages"][1]["content"] == "Please select a tool to handle the current issue." + def test_tool_choice_required_does_not_mutate_input_messages(self): + """tool_choice='required' must not mutate the caller's messages list. + + The handling appends a "select a tool" user message; building it in + place corrupts the caller's conversation history and makes + transform_request non-idempotent across retries. + """ + config = MoonshotChatConfig() + + messages = [{"role": "user", "content": "What's the weather like?"}] + + for _ in range(2): + optional_params = { + "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + } + result = config.transform_request( + model="moonshot-v1-8k", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + # The returned request carries the extra message. + assert len(result["messages"]) == 2 + # The caller's list is untouched, so repeated calls stay idempotent. + assert messages == [{"role": "user", "content": "What's the weather like?"}] + def test_tool_choice_non_required_preserved(self): """Test that non-'required' tool_choice values are preserved""" config = MoonshotChatConfig() diff --git a/tests/test_litellm/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/test_litellm/llms/openai/transcriptions/test_transcription_duration_hidden.py index 2b287e456a1..703fa13cbc9 100644 --- a/tests/test_litellm/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/test_litellm/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -13,7 +13,52 @@ from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, ) -from litellm.types.utils import TranscriptionResponse +from litellm.types.utils import ( + TranscriptionResponse, + TranscriptionUsageDurationObject, +) + + +class TestDiarizedJsonUsageParsing: + """gpt-4o-transcribe / diarized_json returns a fractional `usage.seconds`.""" + + def test_fractional_duration_seconds_does_not_raise(self): + """ + A diarized_json response carries usage={"type": "duration", "seconds": }. + OpenAI specs `seconds` as a float, so a fractional value must parse cleanly + instead of raising and getting retried until the upstream rate-limits. + """ + response_object = { + "text": "speaker_1: Olá", + "task": "transcribe", + "duration": 295.8, + "segments": [ + { + "id": "seg_001", + "speaker": "speaker_1", + "start": 0.0, + "end": 1.0, + "text": "Olá", + "type": "transcript.text.segment", + } + ], + "usage": {"type": "duration", "seconds": 295.8}, + } + + result = convert_to_model_response_object( + response_object=response_object, + model_response_object=TranscriptionResponse(), + response_type="audio_transcription", + ) + + assert isinstance(result.usage, TranscriptionUsageDurationObject) + assert result.usage.seconds == 295.8 + + def test_usage_duration_object_accepts_float_seconds(self): + assert ( + TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds + == 295.8 + ) class TestTranscriptionDurationNotInResponseBody: diff --git a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py index b5c1a86205b..7be295826e3 100644 --- a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py +++ b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py @@ -293,7 +293,10 @@ class TestParallelAISearch: ], ) @pytest.mark.asyncio - async def test_custom_api_base_appends_v1_search(self, api_base): + async def test_custom_api_base_appends_v1_search(self, api_base, monkeypatch): + # Operator points at an internal base via the env override (a trusted + # host), so the server key is still used and the URL is normalized. + monkeypatch.setenv("PARALLEL_AI_API_BASE", api_base) with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock, @@ -303,7 +306,6 @@ class TestParallelAISearch: await litellm.asearch( query="AI developments", search_provider="parallel_ai", - api_base=api_base, ) call_args = mock_post.call_args @@ -312,6 +314,23 @@ class TestParallelAISearch: == "https://proxy.internal.example.com/v1/search" ) + @pytest.mark.asyncio + async def test_caller_api_base_without_key_is_refused(self, monkeypatch): + # A caller-supplied api_base (untrusted host) while relying on the + # server key must be refused without any outbound request. + monkeypatch.setenv("PARALLEL_API_KEY", "server-secret") + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + with pytest.raises(Exception, match="Refusing to send"): + await litellm.asearch( + query="AI developments", + search_provider="parallel_ai", + api_base="https://attacker.example.com", + ) + mock_post.assert_not_called() + @pytest.mark.asyncio async def test_missing_api_key_raises(self, monkeypatch): monkeypatch.delenv("PARALLEL_API_KEY", raising=False) diff --git a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py index 5496486765c..9870d30d488 100644 --- a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py +++ b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py @@ -63,23 +63,17 @@ class TestTinyfishSearchConfig: assert headers["X-API-Key"] == "sk-tinyfish-test" assert headers["Accept"] == "application/json" - def test_validate_environment_from_env(self): + def test_validate_environment_from_env(self, monkeypatch): + monkeypatch.setenv("TINYFISH_API_KEY", "sk-from-env") config = TinyfishSearchConfig() - with patch( - "litellm.llms.tinyfish.search.transformation.get_secret_str", - return_value="sk-from-env", - ): - headers = config.validate_environment(headers={}) + headers = config.validate_environment(headers={}) assert headers["X-API-Key"] == "sk-from-env" - def test_validate_environment_missing_key(self): + def test_validate_environment_missing_key(self, monkeypatch): + monkeypatch.delenv("TINYFISH_API_KEY", raising=False) config = TinyfishSearchConfig() - with patch( - "litellm.llms.tinyfish.search.transformation.get_secret_str", - return_value=None, - ): - with pytest.raises(ValueError, match="TINYFISH_API_KEY"): - config.validate_environment(headers={}) + with pytest.raises(ValueError, match="TINYFISH_API_KEY"): + config.validate_environment(headers={}) def test_validate_environment_uses_api_base_kwarg(self): config = TinyfishSearchConfig() diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 1ebd704be34..1f171496cce 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -346,3 +346,74 @@ def test_vertex_does_not_warn_when_dropping_non_guardrail_session_update(caplog) "Vertex AI Realtime" in record.message and "session.update" in record.message for record in caplog.records ) + + +async def test_async_realtime_does_not_forward_client_query_params_to_vertex_backend( + monkeypatch, +): + """Regression: forwarding client ?model=/?intent= to the Vertex Live WSS URL causes 1007 errors. + + Exercises ``async_realtime`` end-to-end so that re-adding ``_append_query_params`` + (the reverted bug) would push ``model=``/``intent=`` onto the backend URL and fail here. + """ + import websockets + + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + + captured = {} + + def fake_connect(url, *args, **kwargs): + captured["url"] = url + raise RuntimeError("stop before establishing the backend connection") + + monkeypatch.setattr(websockets, "connect", fake_connect) + + await BaseLLMHTTPHandler().async_realtime( + model="gemini-live-2.5-flash-native-audio", + websocket=AsyncMock(), + logging_obj=MagicMock(), + provider_config=cfg, + headers={}, + query_params={ + "model": "gemini-live-2.5-flash-native-audio", + "intent": "chat", + }, + ) + + assert "?" not in captured["url"] + assert "model=" not in captured["url"] + assert "intent=" not in captured["url"] + + +def test_vertex_function_call_output_omits_id(): + """Regression: Vertex Live rejects ``id`` on toolResponse.functionResponses (1007).""" + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + cfg._tool_call_id_to_name["call_abc123"] = "terminate_call" + + messages = cfg.transform_realtime_request( + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": "call_abc123", + "output": '{"status": "ok"}', + }, + } + ), + "gemini-live-2.5-flash-native-audio", + session_configuration_request="existing", + ) + + assert len(messages) == 1 + payload = json.loads(messages[0]) + function_response = payload["toolResponse"]["functionResponses"][0] + assert "id" not in function_response + assert function_response["name"] == "terminate_call" + assert function_response["response"] == {"status": "ok"} diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py new file mode 100644 index 00000000000..7e028064e4c --- /dev/null +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -0,0 +1,333 @@ +"""Tests for the optional Rust-backed OCR path (``litellm/ocr/rust_bridge.py``).""" + +import importlib +import sys +import types + +import httpx +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse + +# `litellm/__init__.py` does `from .ocr.main import *`, which binds the `ocr` +# function onto `litellm.ocr` and shadows the submodule, so import the modules +# explicitly via importlib rather than attribute traversal. +ocr_main = importlib.import_module("litellm.ocr.main") +rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") + +MODEL = "mistral/mistral-ocr-latest" +DOCUMENT = {"type": "document_url", "document_url": "https://example.com/doc.pdf"} + +FAKE_OCR_RESPONSE = { + "pages": [{"index": 0, "markdown": "hello world"}], + "model": "mistral-ocr-2505-completion", + "document_annotation": None, + "usage_info": {"pages_processed": 1}, + "object": "ocr", +} + + +class RecordingBridge: + """A fake ``RustOcr`` callable that records the args it was handed.""" + + def __init__(self): + self.calls = [] + + def __call__( + self, model, document, api_key, api_base, optional_params, timeout_seconds + ): + self.calls.append( + { + "model": model, + "document": document, + "api_key": api_key, + "api_base": api_base, + "optional_params": optional_params, + "timeout_seconds": timeout_seconds, + } + ) + return dict(FAKE_OCR_RESPONSE) + + +class RecordingLogging: + """A spy standing in for ``LiteLLMLoggingObj`` to capture ``pre_call``.""" + + def __init__(self): + self.pre_call_kwargs = None + + def pre_call(self, *, input, api_key, additional_args): + self.pre_call_kwargs = { + "input": input, + "api_key": api_key, + "additional_args": additional_args, + } + + +class FakeOCRConfig: + """A stand-in ``BaseOCRConfig`` that echoes the request it would build.""" + + def validate_environment( + self, *, headers, model, api_key, api_base, litellm_params + ): + return {"authorization": f"Bearer {api_key}"} + + def get_complete_url(self, *, api_base, model, optional_params, litellm_params): + return f"{api_base or 'https://api.mistral.ai/v1'}/ocr" + + +@pytest.fixture(autouse=True) +def _reset_rust_flag(): + """Keep the global toggle isolated between tests.""" + rust_bridge.use_litellm_rust(False, ocr=None) + yield + rust_bridge.use_litellm_rust(False, ocr=None) + + +@pytest.fixture +def fake_bridge(): + """Enable the Rust path with an injected recording bridge (no native wheel).""" + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + return bridge + + +def test_use_litellm_rust_toggles_flag(): + assert rust_bridge.rust_ocr_enabled() is False + litellm.use_litellm_rust() + assert rust_bridge.rust_ocr_enabled() is True + litellm.use_litellm_rust(False) + assert rust_bridge.rust_ocr_enabled() is False + + +def test_load_rust_ocr_returns_injected_impl(): + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + assert rust_bridge.load_rust_ocr() is bridge + + +def test_toggle_without_ocr_arg_preserves_injected_impl(): + """Regression: routine enable/disable calls must not clobber a prior injection. + + Earlier, ``use_litellm_rust()`` unconditionally assigned the keyword default + of ``None`` to ``_rust_ocr_impl``, silently dropping a custom bridge whenever + a caller toggled the flag without re-passing ``ocr=``. + """ + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + + litellm.use_litellm_rust(False) + assert rust_bridge.load_rust_ocr() is bridge + litellm.use_litellm_rust(True) + assert rust_bridge.load_rust_ocr() is bridge + + +def test_explicit_ocr_none_clears_injected_impl(): + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + + litellm.use_litellm_rust(True, ocr=None) + assert rust_bridge.load_rust_ocr() is None + + +def test_load_rust_ocr_none_when_extension_absent(): + """With no injected impl and no compiled wheel, the loader returns None so the + caller degrades to the Python path instead of raising ImportError.""" + litellm.use_litellm_rust(True) # no impl injected; extension isn't built in CI + assert rust_bridge.load_rust_ocr() is None + + +def test_load_rust_ocr_uses_compiled_extension(monkeypatch): + """With no injected impl but a compiled ``litellm_python_bridge`` importable, + the loader returns the extension's ``ocr`` callable. The native wheel isn't + built in CI, so stand in a fake module via ``sys.modules``.""" + fake_module = types.ModuleType("litellm_python_bridge") + fake_module.ocr = lambda **kwargs: dict(FAKE_OCR_RESPONSE) # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "litellm_python_bridge", fake_module) + + litellm.use_litellm_rust(True) # enabled, no impl injected -> import the extension + assert rust_bridge.load_rust_ocr() is fake_module.ocr + + +def test_timeout_to_seconds_handles_float_timeout_and_none(): + assert ocr_main._timeout_to_seconds(12.5) == 12.5 + assert ocr_main._timeout_to_seconds(None) is None + assert ocr_main._timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0 + + +def test_run_rust_ocr_forwards_args_and_wraps_response(): + bridge = RecordingBridge() + logging_obj = RecordingLogging() + + response = ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=logging_obj, + provider_config=FakeOCRConfig(), + resolve_api_key=lambda _name: None, + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-test", + api_base="https://proxy.internal", + optional_params={"include_image_base64": True}, + litellm_params={}, + timeout_seconds=12.5, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + call = bridge.calls[0] + assert call == { + "model": "mistral-ocr-latest", + "document": DOCUMENT, + "api_key": "sk-test", + "api_base": "https://proxy.internal", + "optional_params": {"include_image_base64": True}, + "timeout_seconds": 12.5, + } + + +def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): + """No explicit api_key: the resolver (get_secret_str in production) supplies it, + so secret-manager backends (AWS/Azure/GCP/Vault) work like the Python path.""" + bridge = RecordingBridge() + + ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=RecordingLogging(), + provider_config=FakeOCRConfig(), + resolve_api_key=lambda name: ( + "sk-from-vault" if name == "MISTRAL_API_KEY" else None + ), + model="mistral-ocr-latest", + document=DOCUMENT, + api_key=None, + api_base=None, + optional_params={}, + litellm_params={}, + timeout_seconds=None, + ) + + assert bridge.calls[0]["api_key"] == "sk-from-vault" + + +def test_run_rust_ocr_prefers_explicit_key_over_resolver(): + bridge = RecordingBridge() + resolver_calls = [] + + def _resolver(name): + resolver_calls.append(name) + return "sk-from-vault" + + ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=RecordingLogging(), + provider_config=FakeOCRConfig(), + resolve_api_key=_resolver, + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-explicit", + api_base=None, + optional_params={}, + litellm_params={}, + timeout_seconds=None, + ) + + assert bridge.calls[0]["api_key"] == "sk-explicit" + assert resolver_calls == [] # resolver never consulted when a key is supplied + + +def test_run_rust_ocr_runs_pre_call_logging(): + """The Rust shortcut must run pre_call so callbacks and spend tracking fire.""" + logging_obj = RecordingLogging() + + ocr_main._run_rust_ocr( + rust_ocr=RecordingBridge(), + logging_obj=logging_obj, + provider_config=FakeOCRConfig(), + resolve_api_key=lambda _name: None, + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-test", + api_base="https://api.mistral.ai/v1", + optional_params={"include_image_base64": True}, + litellm_params={}, + timeout_seconds=None, + ) + + assert logging_obj.pre_call_kwargs is not None + assert logging_obj.pre_call_kwargs["input"] == "OCR document processing" + additional_args = logging_obj.pre_call_kwargs["additional_args"] + complete_input = additional_args["complete_input_dict"] + assert complete_input["document"] == DOCUMENT + assert complete_input["include_image_base64"] is True + # The logged request mirrors what Rust sends: resolved URL + headers. + assert additional_args["api_base"] == "https://api.mistral.ai/v1/ocr" + assert additional_args["headers"] == {"authorization": "Bearer sk-test"} + + +def test_ocr_routes_to_rust_when_enabled(fake_bridge): + response = litellm.ocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + include_image_base64=True, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + assert len(fake_bridge.calls) == 1 + call = fake_bridge.calls[0] + # Provider prefix is stripped before reaching the bridge. + assert call["model"] == "mistral-ocr-latest" + assert call["document"] == DOCUMENT + assert call["api_key"] == "sk-test" + # Raw OCR params ride along in optional_params; Rust filters to supported keys. + assert call["optional_params"].get("include_image_base64") is True + + +def test_ocr_forwards_timeout_to_rust(fake_bridge): + """Caller-supplied timeout must flow into the Rust bridge so the fixed 600s + client ceiling doesn't silently override shorter deadlines.""" + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test", timeout=12.5) + + assert fake_bridge.calls[0]["timeout_seconds"] == 12.5 + + +def test_ocr_passes_default_request_timeout_to_rust(fake_bridge): + """When no explicit timeout is given, the library default (request_timeout) + must still be forwarded so the Rust path matches the Python path's deadline.""" + from litellm.constants import request_timeout + + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert fake_bridge.calls[0]["timeout_seconds"] == float(request_timeout) + + +def test_ocr_does_not_route_to_rust_when_disabled(): + """With the flag off, the bridge must not be consulted even if an impl exists.""" + bridge = RecordingBridge() + litellm.use_litellm_rust(False, ocr=bridge) + + assert rust_bridge.rust_ocr_enabled() is False + # The impl stays available for injection, but the disabled flag gates usage, + # so ocr() never reaches the Rust path (asserted via the enabled-path test). + assert bridge.calls == [] + + +def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): + """Rust enabled but no bridge available (no injected impl, no compiled wheel): + ocr() must degrade to the Python HTTP handler instead of raising.""" + litellm.use_litellm_rust(True) # enabled, but load_rust_ocr() returns None in CI + + captured = {} + + def fake_handler_ocr(**kwargs): + captured["called"] = True + return OCRResponse(pages=[], model="mistral-ocr-latest", object="ocr") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", fake_handler_ocr) + + response = litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert captured.get("called") is True # Python path was used + assert isinstance(response, OCRResponse) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index ab42ee1e979..20fd5c1d86a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -15,7 +15,11 @@ from starlette.datastructures import Headers from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) -from litellm.proxy._types import SpecialHeaders, UserAPIKeyAuth +from litellm.proxy._types import ( + SpecialHeaders, + SpecialMCPServerNames, + UserAPIKeyAuth, +) @pytest.mark.asyncio @@ -166,6 +170,53 @@ class TestMCPRequestHandler: mock_key_servers.assert_called_once_with(user_api_key_auth) mock_team_servers.assert_called_once_with(user_api_key_auth) + @pytest.mark.parametrize("team_servers", [[], ["team_server1", "team_server2"]]) + async def test_no_mcp_servers_sentinel_returns_empty(self, team_servers): + """A key scoped to the no-mcp-servers sentinel resolves to zero servers, + overriding team inheritance and never leaking the sentinel marker.""" + user_api_key_auth = UserAPIKeyAuth( + api_key="test-key", user_id="test-user", team_id="test-team" + ) + key_object_permission = MagicMock() + key_object_permission.mcp_servers = [ + SpecialMCPServerNames.no_mcp_servers.value + ] + + with patch.object( + MCPRequestHandler, + "_get_key_object_permission", + return_value=key_object_permission, + ), patch.object( + MCPRequestHandler, + "_get_allowed_mcp_servers_for_team", + new_callable=AsyncMock, + return_value=team_servers, + ): + result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) + + assert result == [] + + async def test_get_allowed_mcp_servers_for_key_returns_sentinel_marker(self): + """_get_allowed_mcp_servers_for_key surfaces the sentinel unexpanded so the + caller can short-circuit, ignoring any other entries on the key.""" + user_api_key_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user") + key_object_permission = MagicMock() + key_object_permission.mcp_servers = [ + SpecialMCPServerNames.no_mcp_servers.value, + "some-other-server", + ] + + with patch.object( + MCPRequestHandler, + "_get_key_object_permission", + return_value=key_object_permission, + ): + result = await MCPRequestHandler._get_allowed_mcp_servers_for_key( + user_api_key_auth + ) + + assert result == [SpecialMCPServerNames.no_mcp_servers.value] + async def test_permission_inheritance_edge_cases(self): """Test edge cases in permission inheritance""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_httpx_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_httpx_auth.py new file mode 100644 index 00000000000..9eab089bac6 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_httpx_auth.py @@ -0,0 +1,44 @@ +"""Tests for the concrete httpx.Auth objects the resolver returns. + +NoOpAuth must attach nothing; StaticHeaderAuth must set exactly the configured header. These +pin the header emission the api_key family and passthrough depend on. +""" + +import httpx + +from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + NoOpAuth, + StaticHeaderAuth, +) + + +def _apply(auth: httpx.Auth, request: httpx.Request) -> httpx.Request: + flow = auth.auth_flow(request) + sent = next(flow) + flow.close() + return sent + + +def test_noop_auth_attaches_no_authorization_header(): + request = httpx.Request("GET", "https://upstream.example.com/mcp") + _apply(NoOpAuth(), request) + assert "authorization" not in request.headers + + +def test_static_header_auth_defaults_to_authorization(): + request = httpx.Request("GET", "https://upstream.example.com/mcp") + _apply(StaticHeaderAuth("Bearer abc"), request) + assert request.headers["Authorization"] == "Bearer abc" + + +def test_static_header_auth_honors_custom_header_name(): + request = httpx.Request("GET", "https://upstream.example.com/mcp") + _apply(StaticHeaderAuth("raw-key", header_name="X-API-Key"), request) + assert request.headers["X-API-Key"] == "raw-key" + assert "authorization" not in request.headers + + +def test_static_header_auth_masks_credential_from_introspection(): + auth = StaticHeaderAuth("Bearer super-secret-token") + assert "super-secret-token" not in repr(auth) + assert "super-secret-token" not in str(vars(auth)) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py new file mode 100644 index 00000000000..7885617aa46 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py @@ -0,0 +1,57 @@ +"""Tests for the resolver dispatch skeleton. + +Every mode must reach its own arm and, until that arm is built, return a typed +`not_implemented` CredError rather than silently producing no credential. Parametrizing over +one config per mode also guards reachability: if a `case` were dropped, that mode would fall to +the `assert_never` tail and raise here instead of returning the stub. +""" + +import pytest +from pydantic import SecretStr + +from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + ApiKeyConfig, + AuthorizationCodeConfig, + AuthSpecKind, + AwsSigV4Config, + ClientCredentialsConfig, + Error, + NoneConfig, + PassthroughConfig, + ServerSpec, + SharedKey, + Subject, + TokenExchangeConfig, + UpstreamCredentialProvider, +) + +_ONE_CONFIG_PER_MODE = [ + (AuthSpecKind.none, NoneConfig()), + (AuthSpecKind.api_key, ApiKeyConfig(key_source=SharedKey(value=SecretStr("k")))), + (AuthSpecKind.passthrough, PassthroughConfig()), + (AuthSpecKind.client_credentials, ClientCredentialsConfig()), + (AuthSpecKind.token_exchange, TokenExchangeConfig()), + (AuthSpecKind.authorization_code, AuthorizationCodeConfig()), + (AuthSpecKind.aws_sigv4, AwsSigV4Config(region="us-east-1")), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind, config", _ONE_CONFIG_PER_MODE) +async def test_every_mode_reaches_its_arm_and_returns_not_implemented(kind, config): + spec = ServerSpec( + server_id="s", resource="https://upstream.example.com", config=config + ) + subject = Subject(tenant_id="", subject_id="") + + result = await UpstreamCredentialProvider().resolve_credentials(subject, spec) + + assert isinstance(result, Error) + assert result.error.tag == "not_implemented" + assert kind.value in result.error.summary + + +def test_all_seven_modes_are_covered(): + # Guards that the parametrization (and therefore the dispatch) spans every AuthSpecKind, so a + # newly added mode without a test row is caught here rather than slipping through. + assert {kind for kind, _ in _ONE_CONFIG_PER_MODE} == set(AuthSpecKind) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_result.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_result.py new file mode 100644 index 00000000000..5d26a530214 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_result.py @@ -0,0 +1,20 @@ +"""Smoke test for the outbound_credentials Result union. + +Result is trivial frozen dataclasses; its load-bearing guarantee (no `.ok` access before +the Error arm is eliminated) is a type-checker property, not a runtime one. This pins only +the runtime contract consumers rely on: each arm carries its payload and discriminates by +type. The union is exercised for real where it is used (see PR2's parse_auth_spec_kind). +""" + +from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + Error, + Ok, + Result, +) + + +def test_ok_and_error_carry_payload_and_discriminate(): + ok: Result[int, str] = Ok(5) + err: Result[int, str] = Error("boom") + assert isinstance(ok, Ok) and ok.ok == 5 + assert isinstance(err, Error) and err.error == "boom" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_types.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_types.py new file mode 100644 index 00000000000..43b3612a5f2 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_types.py @@ -0,0 +1,150 @@ +"""Construction-time tests for the outbound_credentials vocabulary. + +The point of the typed seam is that illegal mode/field combinations are unrepresentable: +a config missing a required field, an unknown mode, or a mismatched discriminated-union +source must fail at construction, not at resolve time. These tests pin that, plus the +CredError tag/summary surface and the derived auth_spec_kind. Each assertion fails if the +corresponding guarantee is mutated away. +""" + +import pytest +from pydantic import SecretStr, TypeAdapter, ValidationError + +from litellm.proxy._experimental.mcp_server.outbound_credentials import ( + Ambient, + ApiKeyConfig, + AuthConfig, + AuthSpecKind, + AwsSigV4Config, + Byok, + CredError, + Error, + NoneConfig, + Ok, + ServerSpec, + SharedKey, + StaticKeys, + parse_auth_spec_kind, +) + +_AUTH_CONFIG = TypeAdapter(AuthConfig) + + +def test_parse_auth_spec_kind_accepts_known_mode(): + result = parse_auth_spec_kind("token_exchange") + assert isinstance(result, Ok) + assert result.ok is AuthSpecKind.token_exchange + + +def test_parse_auth_spec_kind_rejects_unknown_mode(): + result = parse_auth_spec_kind("totally_made_up") + assert isinstance(result, Error) + assert result.error.tag == "unsupported_mode" + assert "totally_made_up" in result.error.summary + + +@pytest.mark.parametrize( + "factory, expected_tag", + [ + (CredError.of_unauthorized, "unauthorized"), + (CredError.of_misconfigured, "misconfigured"), + (CredError.of_upstream_unavailable, "upstream_unavailable"), + (CredError.of_unsupported_mode, "unsupported_mode"), + (CredError.of_precondition_required, "precondition_required"), + (CredError.of_not_implemented, "not_implemented"), + ], +) +def test_crederror_factory_sets_the_matching_tag(factory, expected_tag): + err = factory("detail text") + assert err.tag == expected_tag + assert "detail text" in err.summary + + +def test_apikeyconfig_requires_a_key_source(): + with pytest.raises(ValidationError): + ApiKeyConfig() # type: ignore[call-arg] + + +def test_sharedkey_requires_a_value(): + with pytest.raises(ValidationError): + SharedKey() # type: ignore[call-arg] + + +def test_static_keys_require_id_and_secret(): + with pytest.raises(ValidationError): + StaticKeys(access_key_id="AKIA") # type: ignore[call-arg] + + +def test_aws_sigv4_requires_a_region(): + with pytest.raises(ValidationError): + AwsSigV4Config() # type: ignore[call-arg] + + +def test_aws_sigv4_defaults_to_the_ambient_credential_chain(): + cfg = AwsSigV4Config(region="us-east-1") + assert isinstance(cfg.credentials, Ambient) + assert cfg.service == "bedrock-agentcore" + + +def test_authconfig_discriminates_on_kind(): + api_key = _AUTH_CONFIG.validate_python( + {"kind": "api_key", "key_source": {"source": "shared", "value": "k"}} + ) + assert isinstance(api_key, ApiKeyConfig) + assert isinstance(api_key.key_source, SharedKey) + + none = _AUTH_CONFIG.validate_python({"kind": "none"}) + assert isinstance(none, NoneConfig) + + +def test_authconfig_rejects_unknown_kind(): + with pytest.raises(ValidationError): + _AUTH_CONFIG.validate_python({"kind": "not_a_mode"}) + + +def test_apikeysource_discriminates_and_rejects_unknown_source(): + byok = ApiKeyConfig.model_validate({"key_source": {"source": "byok"}}) + assert isinstance(byok.key_source, Byok) + + with pytest.raises(ValidationError): + ApiKeyConfig.model_validate({"key_source": {"source": "mystery"}}) + + +def test_server_spec_derives_auth_spec_kind_from_config(): + spec = ServerSpec( + server_id="s1", + resource="https://api.example.com", + config=NoneConfig(), + ) + assert spec.auth_spec_kind is AuthSpecKind.none + + api_spec = ServerSpec( + server_id="s2", + resource="https://api.example.com", + config=ApiKeyConfig(key_source=SharedKey(value=SecretStr("k"))), + ) + assert api_spec.auth_spec_kind is AuthSpecKind.api_key + + +def test_api_key_header_placement(): + default = ApiKeyConfig(key_source=SharedKey(value=SecretStr("tok"))) + assert default.header("tok") == ("Authorization", "Bearer tok") + + raw = ApiKeyConfig( + header_name="X-API-Key", + value_prefix="", + key_source=SharedKey(value=SecretStr("tok")), + ) + assert raw.header("tok") == ("X-API-Key", "tok") + + +def test_configs_are_frozen(): + cfg = NoneConfig() + with pytest.raises(ValidationError): + cfg.kind = AuthSpecKind.api_key # type: ignore[misc] + + +def test_secrets_do_not_leak_in_repr(): + key = SharedKey(value=SecretStr("super-secret")) + assert "super-secret" not in repr(key) + assert key.value.get_secret_value() == "super-secret" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index c86ae966f21..f44552c2943 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -6293,3 +6293,153 @@ async def test_get_allowed_mcp_servers_from_mcp_server_names_empty_list_fails_cl ) assert result == [] + + +class TestProxyExceptionToHttpException: + """Auth failures reach the MCP ASGI handlers as ProxyException, not + HTTPException. The handlers must map them back to their real status and + headers; otherwise they fall through to the generic 500 handler, dropping + the 401 + WWW-Authenticate challenge an OAuth client needs to re-authenticate + and surfacing the tool call as a cancelled/terminated session. + """ + + def test_preserves_401_status_and_www_authenticate_header(self): + from litellm.proxy._experimental.mcp_server.server import ( + _proxy_exception_to_http_exception, + ) + from litellm.proxy._types import ProxyException + + exc = ProxyException( + message="Authentication Error, invalid token", + type="auth_error", + param="key", + code=401, + headers={"WWW-Authenticate": 'Bearer resource_metadata="/x"'}, + ) + + http_exc = _proxy_exception_to_http_exception(exc) + + assert http_exc.status_code == 401 + assert http_exc.detail == "Authentication Error, invalid token" + assert http_exc.headers["WWW-Authenticate"] == 'Bearer resource_metadata="/x"' + + def test_preserves_403_status(self): + from litellm.proxy._experimental.mcp_server.server import ( + _proxy_exception_to_http_exception, + ) + from litellm.proxy._types import ProxyException + + http_exc = _proxy_exception_to_http_exception( + ProxyException( + message="Forbidden", type="auth_error", param="key", code=403 + ) + ) + + assert http_exc.status_code == 403 + + def test_non_numeric_code_falls_back_to_500(self): + from litellm.proxy._experimental.mcp_server.server import ( + _proxy_exception_to_http_exception, + ) + from litellm.proxy._types import ProxyException + + # ProxyException normalises code to the string "None" when unset. + http_exc = _proxy_exception_to_http_exception( + ProxyException(message="boom", type="server_error", param=None, code=None) + ) + + assert http_exc.status_code == 500 + + +class TestStreamableHttpAuthErrorMapping: + """End-to-end guard for the handler wiring: a ProxyException from auth must + propagate as the real HTTPException (401 + WWW-Authenticate), not be + flattened to a generic 500 by the catch-all handler. + """ + + @pytest.mark.asyncio + async def test_streamable_http_propagates_proxy_exception_as_401(self): + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._types import ProxyException + + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/some_server", + "headers": [(b"x-litellm-api-key", b"sk-bad")], + } + + async def receive(): + return {"type": "http.request", "body": b"{}", "more_body": False} + + sent = [] + + async def send(message): + sent.append(message) + + auth_failure = ProxyException( + message="Authentication Error, invalid token", + type="auth_error", + param="key", + code=401, + headers={"WWW-Authenticate": "Bearer"}, + ) + + with patch.object( + mcp_module, + "extract_mcp_auth_context", + new=AsyncMock(side_effect=auth_failure), + ): + with pytest.raises(HTTPException) as exc_info: + await mcp_module.handle_streamable_http_mcp(scope, receive, send) + + assert exc_info.value.status_code == 401 + assert exc_info.value.headers["WWW-Authenticate"] == "Bearer" + # Must not have emitted a 500 body via the generic catch-all. + assert not any( + m.get("type") == "http.response.start" and m.get("status") == 500 + for m in sent + ) + + @pytest.mark.asyncio + async def test_sse_propagates_proxy_exception_as_401(self): + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._types import ProxyException + + scope = { + "type": "http", + "method": "GET", + "path": "/mcp/some_server", + "headers": [(b"x-litellm-api-key", b"sk-bad")], + } + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + sent = [] + + async def send(message): + sent.append(message) + + auth_failure = ProxyException( + message="Authentication Error, invalid token", + type="auth_error", + param="key", + code=401, + headers={"WWW-Authenticate": "Bearer"}, + ) + + with patch.object( + mcp_module, + "extract_mcp_auth_context", + new=AsyncMock(side_effect=auth_failure), + ): + with pytest.raises(HTTPException) as exc_info: + await mcp_module.handle_sse_mcp(scope, receive, send) + + assert exc_info.value.status_code == 401 + assert exc_info.value.headers["WWW-Authenticate"] == "Bearer" + assert not any( + m.get("type") == "http.response.start" and m.get("status") == 500 + for m in sent + ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 1b815b7a1c9..8dbee1daa36 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -2948,6 +2948,41 @@ class TestMCPServerManager: assert "test_server_1" in result assert "test_server_2" in result + @pytest.mark.asyncio + async def test_no_mcp_servers_sentinel_blocks_allow_all_keys(self): + """A key scoped to no-mcp-servers gets zero servers even when allow_all_keys + servers exist, and the inner resolver is never consulted.""" + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth + + manager = MCPServerManager() + object_permission = LiteLLM_ObjectPermissionTable( + object_permission_id="perm_no_mcp", + mcp_servers=["no-mcp-servers"], + mcp_access_groups=[], + ) + user_api_key_auth = UserAPIKeyAuth( + api_key="sk-test", + user_id="user-123", + object_permission=object_permission, + object_permission_id="perm_no_mcp", + ) + + with patch.object( + manager, "get_allow_all_keys_server_ids", return_value=["global-server"] + ), patch.object( + MCPRequestHandler, + "get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=["leaked-server"], + ) as mock_inner: + result = await manager.get_allowed_mcp_servers(user_api_key_auth) + + assert result == [] + mock_inner.assert_not_called() + @pytest.mark.asyncio async def test_get_allowed_mcp_servers_anonymous_delegate_requires_oauth2(self): """Anonymous delegated auth listing should only include oauth2 servers.""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py index 2ffa997bdde..b41cdfb1576 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py @@ -93,6 +93,37 @@ class TestApplyToolsetScope: await _apply_toolset_scope(auth, "toolset-123") assert exc_info.value.status_code == 403 + @pytest.mark.asyncio + @pytest.mark.parametrize("user_role", [None, LitellmUserRoles.PROXY_ADMIN.value]) + async def test_no_mcp_servers_sentinel_denies_toolset_access(self, user_role): + """A key scoped to the no-mcp-servers sentinel cannot reach a toolset it + would otherwise be granted (even as admin); the opt-out covers the + toolset path, which replaces mcp_servers and would drop the sentinel.""" + from starlette.exceptions import HTTPException + + from litellm.proxy._experimental.mcp_server.server import _apply_toolset_scope + + op = LiteLLM_ObjectPermissionTable( + object_permission_id="test", + mcp_servers=["no-mcp-servers"], + mcp_toolsets=["toolset-123"], + ) + auth = UserAPIKeyAuth( + api_key="sk-test", object_permission=op, user_role=user_role + ) + + resolve = AsyncMock(return_value={"server-a": ["tool1"]}) + with patch( + "litellm.proxy._experimental.mcp_server.server." + "global_mcp_server_manager.resolve_toolset_tool_permissions", + new=resolve, + ): + with pytest.raises(HTTPException) as exc_info: + await _apply_toolset_scope(auth, "toolset-123") + + assert exc_info.value.status_code == 403 + resolve.assert_not_awaited() + class TestFetchMCPToolsetsAccess: """Tests for GET /v1/mcp/toolset access control.""" diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 2dcf040a556..c8dc0ea5ed6 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -1787,6 +1787,60 @@ async def test_reject_clientside_metadata_tags_allows_key_tags_without_client_ta assert request_body["metadata"]["tags"] == ["engineering"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [ + "/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke", + "/v1/messages", + ], +) +async def test_common_checks_metadata_route_keeps_key_tags_out_of_provider_metadata( + route, +): + """GH#30629: on routes that track tags in litellm_metadata (bedrock, /v1/messages, + responses, ...) key-level tags must land in litellm_metadata, never in the + provider-facing metadata field (Bedrock rejects non-user_id metadata with HTTP 400). + The auth-time pre-seed keys off LITELLM_METADATA_ROUTES, so hardcoding a single route + or dropping the pre-seed makes apply_key_tags_pre_auth fall back to metadata; this + guards that regression. + """ + from fastapi import Request + + from litellm.proxy.auth.auth_checks import common_checks + + request_body = {"messages": [{"role": "user", "content": "test"}]} + + mock_request = MagicMock(spec=Request) + valid_token = UserAPIKeyAuth( + token="test-token", + metadata={"tags": ["engineering"]}, + ) + + with patch( + "litellm.proxy.auth.auth_checks.get_tag_objects_batch", + new_callable=AsyncMock, + return_value={}, + ): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=valid_token, + request=mock_request, + ) + + assert result is True + assert request_body["litellm_metadata"]["tags"] == ["engineering"] + assert "metadata" not in request_body + + @pytest.mark.asyncio async def test_virtual_key_soft_budget_check_with_user_obj(): """Test _virtual_key_soft_budget_check includes user_email when user_obj is provided""" @@ -3775,3 +3829,54 @@ async def test_inference_route_still_enforces_team_budget(): valid_token=UserAPIKeyAuth(token="test-token", team_id="test-team"), request=MagicMock(), ) + + +@pytest.mark.asyncio +async def test_virtual_key_max_budget_error_names_the_key(): + """BudgetExceededError for a virtual key must name the key (alias + masked key) + so operators don't have to reverse-map a spend figure back to a key.""" + valid_token = UserAPIKeyAuth( + token="hashed-token", + key_alias="payments-prod", + key_name="sk-...um_g", + max_budget=10.0, + spend=0.0, + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + + with patch( + "litellm.proxy.proxy_server.get_current_spend", + new=AsyncMock(return_value=25.0), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + ) + + message = str(exc_info.value) + assert "payments-prod" in message + assert "sk-...um_g" in message + + +@pytest.mark.asyncio +async def test_virtual_key_max_budget_not_exceeded_does_not_raise(): + """Spend below the configured budget must not raise.""" + valid_token = UserAPIKeyAuth( + token="hashed-token", + key_alias="payments-prod", + max_budget=10.0, + spend=0.0, + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.budget_alerts = AsyncMock() + + with patch( + "litellm.proxy.proxy_server.get_current_spend", + new=AsyncMock(return_value=1.0), + ): + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + ) diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index e652c109987..cd8cf10d037 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -2160,3 +2160,48 @@ class TestGetRequestRouteTemplate: lambda self: (_ for _ in ()).throw(RuntimeError("boom")) ) assert get_request_route_template(req) is None + + +class TestIsRequestBodySafeBlocksModelList: + """model_list is an SDK-only field with no proxy API meaning; it must + be rejected from the request body regardless of any opt-in.""" + + def test_model_list_rejected_with_no_opt_in(self): + with pytest.raises(ValueError, match="model_list is not allowed"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "model_list": [{"model_name": "x", "litellm_params": {}}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_model_list_rejected_even_with_proxy_wide_opt_in(self): + with pytest.raises(ValueError, match="model_list is not allowed"): + is_request_body_safe( + request_body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "model_list": [], + }, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-4", + ) + + def test_normal_body_still_passes(self): + assert ( + is_request_body_safe( + request_body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + }, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + is True + ) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 52ba1dbcfbd..d623149ff6a 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -403,6 +403,48 @@ def test_virtual_key_llm_api_routes_rejects_mcp_multi_segment_admin_subpaths( assert exc_info.value.status_code == 403 +@pytest.mark.parametrize( + "route, method", + [ + ("/mcp", "POST"), + ("/mcp/", "POST"), + ("/mcp/my-server", "POST"), # matches the /mcp/{subpath} pattern + ("/mcp/tools", "GET"), + ("/mcp/tools/list", "POST"), + ("/mcp/tools/call", "POST"), + ("/mcp-rest/tools/list", "GET"), + ("/mcp-rest/tools/call", "POST"), + ("/v1/mcp/tools", "GET"), + ], +) +def test_virtual_key_llm_api_routes_allows_mcp_inference_endpoints(route, method): + """Every MCP inference/discovery endpoint must be reachable by virtual keys + scoped to allowed_routes=["llm_api_routes"], the default the Create Key UI + applies. + + /v1/mcp/tools is the most recent addition: before it joined this group a key + could list tools via /mcp/tools/list and /mcp-rest/tools/list but got a 403 + on the equivalent /v1/mcp/tools. Unlike /v1/mcp/server, none of these paths + have a management write counterpart, so they live directly in + `mcp_inference_routes` rather than behind a method-aware carve-out. + """ + + assert RouteChecks.is_llm_api_route(route=route) is True + + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["llm_api_routes"], + ) + + result = RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + request=_mock_request(method), + ) + + assert result is True + + def test_spend_logs_v2_classified_as_management_not_llm_api(): """Paginated spend logs are a management/spend read route, not an LLM API.""" diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 27c1b04fbd9..7219ab58799 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -2689,6 +2689,67 @@ async def test_centralized_common_checks_runs_for_standard_auth(): setattr(_proxy_server_mod, k, v) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [ + "/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke", + "/v1/messages", + ], +) +async def test_centralized_common_checks_routes_header_tags_to_litellm_metadata(route): + """GH#30629: on LITELLM_METADATA_ROUTES the tag-budget read resolves to + litellm_metadata, so the litellm_metadata pre-seed must run before + apply_client_tag_policy_pre_auth merges x-litellm-tags. Otherwise header tags + land in metadata and silently escape _tag_max_budget_check. This guards the + pre-seed call site in _run_centralized_common_checks; dropping it routes header + tags back into metadata. + """ + import litellm.proxy.proxy_server as _proxy_server_mod + from fastapi import Request + from starlette.datastructures import URL + + token = UserAPIKeyAuth(api_key="sk-test", user_id="u1") + request = Request( + scope={ + "type": "http", + "method": "POST", + "headers": [(b"x-litellm-tags", b"tenant:acme")], + "query_string": b"", + } + ) + request._url = URL(url=route) + request_data: dict = {"model": "us.anthropic.claude-sonnet-4-6"} + + attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None) + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.auth.user_api_key_auth._reserve_budget_after_common_checks", + new_callable=AsyncMock, + ), + ): + await _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data=request_data, + route=route, + ) + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + assert request_data["litellm_metadata"]["tags"] == ["tenant:acme"] + assert "metadata" not in request_data + + @pytest.mark.asyncio async def test_centralized_common_checks_skipped_for_custom_auth_without_flag(): """Existing RPS guarantee: custom-auth deployments without diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py index 21d41e0992b..ad0e7010325 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py @@ -13,7 +13,10 @@ from litellm.proxy.management_endpoints.scim.scim_transformations import ( ScimTransformations, ) from litellm.types.proxy.management_endpoints.scim_v2 import ( + SCIM_ENTERPRISE_USER_SCHEMA, + SCIMEnterpriseUser, SCIMPatchOperation, + SCIMUser, ) @@ -149,6 +152,77 @@ class TestScimTransformations: assert scim_user.name.givenName == "Test" assert scim_user.name.familyName == "User" + @pytest.mark.asyncio + async def test_transform_user_with_enterprise_metadata(self, mock_prisma_client): + mock_client, mock_find_unique = mock_prisma_client + mock_find_unique.return_value = None + + user = LiteLLM_UserTable( + user_id="user-ent", + user_email="ent@example.com", + user_alias=None, + teams=[], + created_at=None, + updated_at=None, + metadata={ + "scim_enterprise": {"costCenter": "CC-42", "department": "Platform"} + }, + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_client): + scim_user = await ScimTransformations.transform_litellm_user_to_scim_user( + user + ) + + assert scim_user.enterprise_user is not None + assert scim_user.enterprise_user.costCenter == "CC-42" + assert scim_user.enterprise_user.department == "Platform" + assert SCIM_ENTERPRISE_USER_SCHEMA in scim_user.schemas + + @pytest.mark.asyncio + async def test_transform_user_without_enterprise_metadata_omits_schema( + self, mock_user, mock_prisma_client + ): + mock_client, mock_find_unique = mock_prisma_client + team1 = LiteLLM_TeamTable( + team_id="team-1", team_alias="Team One", members_with_roles=[] + ) + team2 = LiteLLM_TeamTable( + team_id="team-2", team_alias="Team Two", members_with_roles=[] + ) + mock_find_unique.side_effect = [team1, team2] + + with patch("litellm.proxy.proxy_server.prisma_client", mock_client): + scim_user = await ScimTransformations.transform_litellm_user_to_scim_user( + mock_user + ) + + assert scim_user.enterprise_user is None + assert SCIM_ENTERPRISE_USER_SCHEMA not in scim_user.schemas + + def test_scim_user_serialization_omits_absent_enterprise_urn(self): + without_enterprise = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + id="user-1", + userName="user@example.com", + ) + dumped = without_enterprise.model_dump(by_alias=True) + assert SCIM_ENTERPRISE_USER_SCHEMA not in dumped + assert "enterprise_user" not in dumped + assert SCIM_ENTERPRISE_USER_SCHEMA not in dumped["schemas"] + + with_enterprise = SCIMUser( + schemas=[ + "urn:ietf:params:scim:schemas:core:2.0:User", + SCIM_ENTERPRISE_USER_SCHEMA, + ], + id="user-2", + userName="ent@example.com", + enterprise_user=SCIMEnterpriseUser(costCenter="CC-42"), + ) + dumped_ent = with_enterprise.model_dump(by_alias=True) + assert dumped_ent[SCIM_ENTERPRISE_USER_SCHEMA]["costCenter"] == "CC-42" + @pytest.mark.asyncio async def test_transform_litellm_team_to_scim_group( self, mock_team, mock_prisma_client diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index ad893012807..7f5aee51f51 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -15,15 +15,19 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( _extract_group_member_ids, _handle_team_membership_changes, _process_group_patch_operations, + _recompute_scim_member_roles, create_group, create_user, + delete_group, get_users, get_service_provider_config, + patch_group, patch_user, update_group, update_user, ) from litellm.types.proxy.management_endpoints.scim_v2 import ( + SCIM_ENTERPRISE_USER_SCHEMA, SCIMGroup, SCIMMember, SCIMPatchOp, @@ -115,6 +119,59 @@ async def test_create_user_defaults_to_viewer(mocker, monkeypatch): assert called_args.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY +@pytest.mark.asyncio +async def test_create_user_ingests_enterprise_extension(mocker, monkeypatch): + """A SCIM create payload carrying the enterprise extension block should land + in the created user's metadata under scim_enterprise""" + + scim_user = SCIMUser.model_validate( + { + "schemas": [ + "urn:ietf:params:scim:schemas:core:2.0:User", + SCIM_ENTERPRISE_USER_SCHEMA, + ], + "userName": "ent-user", + "name": {"familyName": "User", "givenName": "Ent"}, + "emails": [{"value": "ent@example.com"}], + SCIM_ENTERPRISE_USER_SCHEMA: { + "costCenter": "CC-42", + "department": "Platform", + }, + } + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + + new_user_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id="ent-user")), + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await create_user(user=scim_user) + + created_metadata = new_user_mock.call_args.kwargs["data"].metadata + assert created_metadata["scim_enterprise"] == { + "costCenter": "CC-42", + "department": "Platform", + } + + @pytest.mark.asyncio async def test_create_user_uses_default_internal_user_params_role(mocker, monkeypatch): """If role is set in default_internal_user_params, new user should use that role""" @@ -1720,3 +1777,1015 @@ async def test_process_group_patch_operations_with_flag_false_rejects( assert exc_info.value.status_code == 400 assert "does not exist" in str(exc_info.value.detail) assert "new-user-1" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_create_user_grants_admin_when_in_scim_admin_group(mocker, monkeypatch): + """When scim_admin_group is configured and a created user's groups include it, + the user is provisioned as PROXY_ADMIN.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="new-admin", + emails=[SCIMUserEmail(value="new-admin@example.com")], + groups=[SCIMUserGroup(value="litellm-admins", display="LiteLLM Admins")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + new_user_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id="new-admin")), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await create_user(user=scim_user) + + called_args = new_user_mock.call_args.kwargs["data"] + assert called_args.user_role == LitellmUserRoles.PROXY_ADMIN + + +@pytest.mark.asyncio +async def test_create_user_keeps_default_when_not_in_scim_admin_group( + mocker, monkeypatch +): + """When scim_admin_group is configured but the user's groups don't include it, + the user keeps the non-admin default role.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="regular-user", + emails=[SCIMUserEmail(value="regular@example.com")], + groups=[SCIMUserGroup(value="engineering", display="Engineering")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + new_user_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id="regular-user")), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await create_user(user=scim_user) + + called_args = new_user_mock.call_args.kwargs["data"] + assert called_args.user_role == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_update_user_demotes_admin_when_removed_from_scim_admin_group( + mocker, monkeypatch +): + """Core demotion test: a PUT whose new groups no longer include the configured + admin group must re-evaluate the role and write the non-admin default, so an + admin removed from the IdP group is demoted without re-login.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + existing_user = mocker.MagicMock() + existing_user.teams = ["litellm-admins"] + existing_user.metadata = {} + + updated_user = { + "user_id": "demote-me", + "user_email": "demote@example.com", + "user_alias": None, + "teams": ["engineering"], + "metadata": "{}", + } + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="demote-me", + emails=[SCIMUserEmail(value="demote@example.com")], + groups=[SCIMUserGroup(value="engineering", display="Engineering")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value=updated_user + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes", + AsyncMock(), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await update_user(user_id="demote-me", user=scim_user) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_update_user_does_not_force_role_when_scim_admin_group_unset( + mocker, monkeypatch +): + """When scim_admin_group is unset, PUT must not touch user_role (current + behavior preserved).""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + existing_user = mocker.MagicMock() + existing_user.teams = ["litellm-admins"] + existing_user.metadata = {} + + updated_user = { + "user_id": "no-touch", + "user_email": "no-touch@example.com", + "user_alias": None, + "teams": ["litellm-admins"], + "metadata": "{}", + } + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="no-touch", + emails=[SCIMUserEmail(value="no-touch@example.com")], + groups=[SCIMUserGroup(value="litellm-admins", display="LiteLLM Admins")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value=updated_user + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes", + AsyncMock(), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await update_user(user_id="no-touch", user=scim_user) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert "user_role" not in call_args[1]["data"] + + +@pytest.mark.asyncio +async def test_update_user_demotes_when_default_params_lack_user_role( + mocker, monkeypatch +): + """Regression: default_internal_user_params set without a user_role key must + still resolve to the non-admin default on demotion, not silently skip and + leave the user PROXY_ADMIN.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr( + "litellm.default_internal_user_params", {"max_budget": 10}, raising=False + ) + + existing_user = mocker.MagicMock() + existing_user.teams = ["litellm-admins"] + existing_user.metadata = {} + + updated_user = { + "user_id": "demote-me", + "user_email": "demote@example.com", + "user_alias": None, + "teams": ["engineering"], + "metadata": "{}", + } + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="demote-me", + emails=[SCIMUserEmail(value="demote@example.com")], + groups=[SCIMUserGroup(value="engineering", display="Engineering")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value=updated_user + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes", + AsyncMock(), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await update_user(user_id="demote-me", user=scim_user) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_patch_user_demotes_admin_when_removed_from_scim_admin_group( + mocker, monkeypatch +): + """PATCH that drops the admin team from the resulting team set must write the + non-admin default, mirroring the PUT demotion path.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + existing_user = mocker.MagicMock() + existing_user.teams = ["litellm-admins"] + existing_user.metadata = {} + + patch_ops = SCIMPatchOp( + schemas=["urn:ietf:params:scim:api:messages:2.0:PatchOp"], + Operations=[ + SCIMPatchOperation( + op="replace", path="groups", value=[{"value": "engineering"}] + ) + ], + ) + + updated_user = { + "user_id": "demote-me", + "user_alias": None, + "teams": ["engineering"], + "metadata": "{}", + } + + engineering_team = mocker.MagicMock() + engineering_team.team_alias = "Engineering" + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value=updated_user + ) + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=engineering_team + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes", + AsyncMock(), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock( + return_value=SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="demote-me", + ) + ), + ) + + await patch_user(user_id="demote-me", patch_ops=patch_ops) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_patch_user_grants_admin_by_team_display_name(mocker, monkeypatch): + """PATCH carries groups as team ids, so admin-group matching must fall back to + each team's display name; an admin group configured as a human-readable alias + grants PROXY_ADMIN even when the team id differs.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "LiteLLM Admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + existing_user = mocker.MagicMock() + existing_user.teams = [] + existing_user.metadata = {} + + patch_ops = SCIMPatchOp( + schemas=["urn:ietf:params:scim:api:messages:2.0:PatchOp"], + Operations=[ + SCIMPatchOperation( + op="replace", path="groups", value=[{"value": "team-abc-123"}] + ) + ], + ) + + updated_user = { + "user_id": "promote-me", + "user_alias": None, + "teams": ["team-abc-123"], + "metadata": "{}", + } + + admin_team = mocker.MagicMock() + admin_team.team_alias = "LiteLLM Admins" + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value=updated_user + ) + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=admin_team + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists", + AsyncMock(return_value=existing_user), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes", + AsyncMock(), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock( + return_value=SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="promote-me", + ) + ), + ) + + await patch_user(user_id="promote-me", patch_ops=patch_ops) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.PROXY_ADMIN + + +def _scim_admin_prisma(mocker, *, user_teams): + """Prisma double whose user resolves to user_teams and whose teams expose an + alias equal to their id, used by the role-recompute helper tests.""" + user = mocker.MagicMock() + user.user_id = "member-1" + user.teams = user_teams + + def _team_find_unique(where): + team = mocker.MagicMock() + team.team_alias = where["team_id"] + return team + + prisma = mocker.MagicMock() + prisma.db = mocker.MagicMock() + prisma.db.litellm_usertable = mocker.MagicMock() + prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user) + prisma.db.litellm_usertable.update = AsyncMock(return_value=user) + prisma.db.litellm_teamtable = mocker.MagicMock() + prisma.db.litellm_teamtable.find_unique = AsyncMock(side_effect=_team_find_unique) + return prisma + + +@pytest.mark.asyncio +async def test_recompute_scim_member_roles_demotes_when_not_in_admin_group( + mocker, monkeypatch +): + """The shared recompute helper writes the non-admin default for a member whose + resulting teams no longer include the configured admin group.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + prisma = _scim_admin_prisma(mocker, user_teams=["engineering"]) + + await _recompute_scim_member_roles(prisma, ["member-1"]) + + call_args = prisma.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_recompute_scim_member_roles_grants_when_in_admin_group( + mocker, monkeypatch +): + """The shared recompute helper grants PROXY_ADMIN when a member's resulting + teams include the configured admin group.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + prisma = _scim_admin_prisma(mocker, user_teams=["litellm-admins"]) + + await _recompute_scim_member_roles(prisma, ["member-1"]) + + call_args = prisma.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.PROXY_ADMIN + + +@pytest.mark.asyncio +async def test_recompute_scim_member_roles_noop_when_admin_group_unset( + mocker, monkeypatch +): + """With scim_admin_group unset the recompute helper must not touch any role, + preserving current behavior for SCIM group writes.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + + prisma = _scim_admin_prisma(mocker, user_teams=["litellm-admins"]) + + await _recompute_scim_member_roles(prisma, ["member-1"]) + + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_update_group_recomputes_roles_for_changed_members(mocker): + """PUT /Groups must recompute the global role for every member whose + membership changed, so an admin dropped from the admin group is demoted.""" + from litellm.proxy._types import LiteLLM_TeamTable, Member + from litellm.proxy.management_endpoints.scim.scim_transformations import ( + ScimTransformations, + ) + + group_id = "test-team-123" + existing_team = LiteLLM_TeamTable( + team_id=group_id, + team_alias="Admins", + members=["user1", "user2"], + members_with_roles=[ + Member(user_id="user1", role="user"), + Member(user_id="user2", role="user"), + ], + metadata={}, + ) + scim_group_update = SCIMGroup( + schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"], + id=group_id, + displayName="Admins", + members=[SCIMMember(value="user2"), SCIMMember(value="user3")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_teamtable.update = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=mocker.MagicMock() + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + AsyncMock(), + ) + recompute_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", + AsyncMock(), + ) + mocker.patch.object( + ScimTransformations, + "transform_litellm_team_to_scim_group", + AsyncMock(return_value=scim_group_update), + ) + + await update_group(group_id=group_id, group=scim_group_update) + + recompute_mock.assert_awaited_once() + assert set(recompute_mock.call_args[0][1]) == {"user1", "user3"} + + +@pytest.mark.asyncio +async def test_patch_group_recomputes_roles_for_changed_members(mocker): + """PATCH /Groups must recompute the global role for every member whose + membership changed, mirroring the PUT path.""" + from litellm.proxy._types import LiteLLM_TeamTable, Member + from litellm.proxy.management_endpoints.scim.scim_transformations import ( + ScimTransformations, + ) + + group_id = "test-team-123" + existing_team = LiteLLM_TeamTable( + team_id=group_id, + team_alias="Admins", + members=["user1", "user2"], + members_with_roles=[ + Member(user_id="user1", role="user"), + Member(user_id="user2", role="user"), + ], + metadata={}, + ) + patch_ops = SCIMPatchOp( + schemas=["urn:ietf:params:scim:api:messages:2.0:PatchOp"], + Operations=[ + SCIMPatchOperation(op="remove", path="members", value=[{"value": "user1"}]) + ], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_teamtable.update = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=mocker.MagicMock() + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + AsyncMock(), + ) + recompute_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", + AsyncMock(), + ) + mocker.patch.object( + ScimTransformations, + "transform_litellm_team_to_scim_group", + AsyncMock( + return_value=SCIMGroup( + schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"], + id=group_id, + displayName="Admins", + ) + ), + ) + + await patch_group(group_id=group_id, patch_ops=patch_ops) + + recompute_mock.assert_awaited_once() + assert set(recompute_mock.call_args[0][1]) == {"user1"} + + +@pytest.mark.asyncio +async def test_delete_group_recomputes_roles_for_members(mocker): + """DELETE /Groups must recompute the global role for the team's members, so + deleting the admin group demotes everyone who was only admin through it.""" + from litellm.proxy._types import Member + + existing_team = mocker.MagicMock() + existing_team.members_with_roles = [ + Member(user_id="user1", role="user"), + Member(user_id="user2", role="user"), + ] + + member = mocker.MagicMock() + member.teams = ["test-team-123"] + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_teamtable.delete = AsyncMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=member) + mock_prisma_client.db.litellm_usertable.update = AsyncMock() + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + recompute_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", + AsyncMock(), + ) + + await delete_group(group_id="test-team-123") + + recompute_mock.assert_awaited_once() + assert list(recompute_mock.call_args[0][1]) == ["user1", "user2"] + + +@pytest.mark.asyncio +async def test_handle_existing_user_by_email_applies_role_when_admin_group_set(mocker): + """When admin_group is configured, re-upserting an existing email persists the + resolved role so a now-non-admin user can't keep a stale PROXY_ADMIN.""" + existing_user = mocker.MagicMock() + existing_user.user_id = "old-user-id" + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock( + return_value=existing_user + ) + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value={"user_id": "new-user-id"} + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=mocker.MagicMock()), + ) + + new_user_request = NewUserRequest( + user_id="new-user-id", + user_email="test@example.com", + teams=["engineering"], + metadata={}, + auto_create_key=False, + user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ) + + await UserProvisionerHelpers.handle_existing_user_by_email( + prisma_client=mock_prisma_client, + new_user_request=new_user_request, + admin_group="litellm-admins", + ) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_handle_existing_user_by_email_leaves_role_when_admin_group_unset(mocker): + """With admin_group unset, the existing-email upsert must not write user_role, + preserving current behavior when the feature is off.""" + existing_user = mocker.MagicMock() + existing_user.user_id = "old-user-id" + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock( + return_value=existing_user + ) + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value={"user_id": "new-user-id"} + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=mocker.MagicMock()), + ) + + new_user_request = NewUserRequest( + user_id="new-user-id", + user_email="test@example.com", + teams=["engineering"], + metadata={}, + auto_create_key=False, + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + await UserProvisionerHelpers.handle_existing_user_by_email( + prisma_client=mock_prisma_client, + new_user_request=new_user_request, + ) + + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert "user_role" not in call_args[1]["data"] + + +@pytest.mark.asyncio +async def test_create_user_existing_email_upsert_demotes_when_admin_group_set( + mocker, monkeypatch +): + """End-to-end create wiring: a SCIM POST that upserts an existing email while + the user is not in the admin group must write the non-admin default, not leave + a stale PROXY_ADMIN.""" + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + + scim_user = SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + userName="returning-user", + emails=[SCIMUserEmail(value="returning@example.com")], + groups=[SCIMUserGroup(value="engineering", display="Engineering")], + ) + + existing_user = mocker.MagicMock() + existing_user.user_id = "returning-user" + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock( + return_value=existing_user + ) + mock_prisma_client.db.litellm_usertable.update = AsyncMock( + return_value={"user_id": "returning-user"} + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + new_user_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_user", + AsyncMock(return_value=NewUserRequest(user_id="returning-user")), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock(return_value=scim_user), + ) + + await create_user(user=scim_user) + + new_user_mock.assert_not_called() + call_args = mock_prisma_client.db.litellm_usertable.update.call_args + assert call_args[1]["data"]["user_role"] == LitellmUserRoles.INTERNAL_USER_VIEW_ONLY + + +@pytest.mark.asyncio +async def test_create_group_recomputes_roles_for_members(mocker): + """POST /Groups must recompute the global role for the new team's members, so a + team created with the admin-group display name elevates its members.""" + from litellm.proxy.management_endpoints.scim.scim_transformations import ( + ScimTransformations, + ) + + group_id = "admin-team-1" + scim_group = SCIMGroup( + schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"], + id=group_id, + displayName="LiteLLM Admins", + members=[SCIMMember(value="user1"), SCIMMember(value="user2")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=mocker.MagicMock() + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.new_team", + AsyncMock(return_value=mocker.MagicMock()), + ) + recompute_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", + AsyncMock(), + ) + mocker.patch.object( + ScimTransformations, + "transform_litellm_team_to_scim_group", + AsyncMock(return_value=scim_group), + ) + + await create_group(group=scim_group) + + recompute_mock.assert_awaited_once() + assert set(recompute_mock.call_args[0][1]) == {"user1", "user2"} + + +@pytest.mark.asyncio +async def test_update_group_rename_recomputes_retained_members(mocker): + """A PUT that renames the group (alias changes) but leaves membership unchanged + must still recompute retained members, since a rename can flip whether the + group matches scim_admin_group by display name.""" + from litellm.proxy._types import LiteLLM_TeamTable, Member + from litellm.proxy.management_endpoints.scim.scim_transformations import ( + ScimTransformations, + ) + + group_id = "test-team-123" + existing_team = LiteLLM_TeamTable( + team_id=group_id, + team_alias="LiteLLM Admins", + members=["user1"], + members_with_roles=[Member(user_id="user1", role="user")], + metadata={}, + ) + scim_group_update = SCIMGroup( + schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"], + id=group_id, + displayName="Engineering", + members=[SCIMMember(value="user1")], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_teamtable.update = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=mocker.MagicMock() + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + AsyncMock(), + ) + recompute_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", + AsyncMock(), + ) + mocker.patch.object( + ScimTransformations, + "transform_litellm_team_to_scim_group", + AsyncMock(return_value=scim_group_update), + ) + + await update_group(group_id=group_id, group=scim_group_update) + + recompute_mock.assert_awaited_once() + assert set(recompute_mock.call_args[0][1]) == {"user1"} + + +@pytest.mark.asyncio +async def test_patch_group_rename_recomputes_retained_members(mocker): + """A PATCH that renames the group (displayName op) but leaves membership + unchanged must still recompute retained members, mirroring the PUT path.""" + from litellm.proxy._types import LiteLLM_TeamTable, Member + from litellm.proxy.management_endpoints.scim.scim_transformations import ( + ScimTransformations, + ) + + group_id = "test-team-123" + existing_team = LiteLLM_TeamTable( + team_id=group_id, + team_alias="LiteLLM Admins", + members=["user1"], + members_with_roles=[Member(user_id="user1", role="user")], + metadata={}, + ) + patch_ops = SCIMPatchOp( + schemas=["urn:ietf:params:scim:api:messages:2.0:PatchOp"], + Operations=[ + SCIMPatchOperation(op="replace", path="displayName", value="Engineering") + ], + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_teamtable.update = AsyncMock( + return_value=existing_team + ) + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=mocker.MagicMock() + ) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership", + AsyncMock(), + ) + recompute_mock = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", + AsyncMock(), + ) + mocker.patch.object( + ScimTransformations, + "transform_litellm_team_to_scim_group", + AsyncMock( + return_value=SCIMGroup( + schemas=["urn:ietf:params:scim:schemas:core:2.0:Group"], + id=group_id, + displayName="Engineering", + ) + ), + ) + + await patch_group(group_id=group_id, patch_ops=patch_ops) + + recompute_mock.assert_awaited_once() + assert set(recompute_mock.call_args[0][1]) == {"user1"} diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 2d2c18bb46e..b882090e8f1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -638,6 +638,84 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): assert key_data.metrics.spend == 10.0 +def _daily_user_spend_record(*, user_id, api_key, spend): + """A LiteLLM_DailyUserSpend row as the per-user breakdown reads it.""" + return SimpleNamespace( + date="2024-01-01", + user_id=user_id, + api_key=api_key, + model="gpt-4", + model_group="gpt-4", + custom_llm_provider="openai", + mcp_namespaced_tool_name=None, + endpoint="/chat/completions", + spend=spend, + prompt_tokens=10, + completion_tokens=5, + cache_read_input_tokens=0, + cache_creation_input_tokens=0, + api_requests=1, + successful_requests=1, + failed_requests=0, + ) + + +@pytest.mark.asyncio +async def test_get_daily_activity_applies_resolve_entity_metadata_to_breakdown(): + """Regression for LIT-3889: the Spend Per User chart showed raw UUIDs. + + /user/daily/activity used to pass entity_metadata_field=None, so every + user entity in the breakdown carried empty metadata and the dashboard had + nothing to render but the user_id UUID. The page-scoped resolver must put + the resolved email/alias onto the entity metadata so the UI can label it, + while a spender with no email on file still falls back to the raw UUID. + """ + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + + records = [ + _daily_user_spend_record(user_id="user-with-email", api_key="key-1", spend=7.0), + _daily_user_spend_record(user_id="user-no-email", api_key="key-2", spend=3.0), + ] + + mock_table = MagicMock() + mock_table.count = AsyncMock(return_value=len(records)) + mock_table.find_many = AsyncMock(return_value=records) + mock_prisma.db.litellm_dailyuserspend = mock_table + mock_prisma.db.litellm_verificationtoken = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + seen_user_ids = {} + + async def resolver(page_records): + seen_user_ids["ids"] = {r.user_id for r in page_records} + return {"user-with-email": {"user_email": "spender@example.com"}} + + result = await get_daily_activity( + prisma_client=mock_prisma, + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id=None, + entity_metadata_field=None, + start_date="2024-01-01", + end_date="2024-01-01", + model=None, + api_key=None, + page=1, + page_size=1000, + resolve_entity_metadata=resolver, + ) + + # Resolver is driven by the user_ids actually on the page + assert seen_user_ids["ids"] == {"user-with-email", "user-no-email"} + + entities = result.results[0].breakdown.entities + # Email is on the entity metadata so the UI labels the chart with it + assert entities["user-with-email"].metadata["user_email"] == "spender@example.com" + # No email on file -> empty metadata -> UI falls back to the UUID + assert entities["user-no-email"].metadata == {} + + class TestAdjustDatesForTimezone: """ Regression tests for the timezone double-counting bug. @@ -758,6 +836,8 @@ class TestBuildAggregatedSqlQuery: ] assert "model = $4" in sql assert "api_key = $5" in sql + + @pytest.mark.asyncio async def test_get_daily_activity_aggregated_empty_result_set(): """Regression test for the empty-range 500. diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 627958cef93..b4602e0ad8b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -2,6 +2,7 @@ import json import os import sys from datetime import datetime, timezone +from types import SimpleNamespace import pytest from fastapi.testclient import TestClient @@ -20,6 +21,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.management_endpoints.internal_user_endpoints import ( LiteLLM_UserTableWithKeyCount, + _resolve_user_email_metadata, _update_internal_user_params, get_user_key_counts, get_users, @@ -657,6 +659,56 @@ async def test_get_users_includes_timestamps(mocker): assert user_response.key_count == 0 +@pytest.mark.asyncio +async def test_get_users_redacts_scim_enterprise_metadata(mocker): + """ + /user/list must strip scim_enterprise from each user's metadata while leaving + the rest of the metadata intact, matching the user-info endpoints. + """ + mock_prisma_client = mocker.MagicMock() + + mock_user_row = mocker.MagicMock() + mock_user_row.user_id = "listed-user" + mock_user_row.model_dump.return_value = { + "user_id": "listed-user", + "user_email": "listed@example.com", + "user_role": "internal_user", + "metadata": { + "scim_metadata": {"givenName": "Jane", "familyName": "Doe"}, + "scim_enterprise": {"costCenter": "CC-42", "department": "Platform"}, + }, + } + + async def mock_find_many(*args, **kwargs): + return [mock_user_row] + + async def mock_count(*args, **kwargs): + return 1 + + mock_prisma_client.db.litellm_usertable.find_many = mock_find_many + mock_prisma_client.db.litellm_usertable.count = mock_count + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + async def mock_get_user_key_counts(*args, **kwargs): + return {"listed-user": 0} + + mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints.get_user_key_counts", + mock_get_user_key_counts, + ) + + admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + response = await get_users( + page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None + ) + + listed = response["users"][0] + assert listed.metadata == { + "scim_metadata": {"givenName": "Jane", "familyName": "Doe"} + } + assert "scim_enterprise" not in (listed.metadata or {}) + + def test_validate_sort_params(): """ Test that validate_sort_params returns None if sort_by is None @@ -2167,6 +2219,94 @@ async def test_user_info_v2_proxy_admin_can_query_any_user(mocker): assert response.metadata == {"team": "engineering"} +@pytest.mark.asyncio +async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker): + """ + SCIM enterprise attributes are persisted in metadata for reporting, but + /v2/user/info must not surface them; the rest of metadata is preserved. + """ + from fastapi import Request + + from litellm.proxy._types import UserInfoV2Response + from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2 + + mock_prisma_client = mocker.MagicMock() + + mock_user_row = mocker.MagicMock() + mock_user_row.model_dump.return_value = { + "user_id": "target-user-123", + "user_email": "target@example.com", + "metadata": { + "scim_metadata": {"givenName": "Jane", "familyName": "Doe"}, + "scim_enterprise": { + "costCenter": "CC-42", + "department": "Platform", + "employeeNumber": "E-1001", + }, + }, + "teams": ["team-1"], + } + + async def mock_find_unique(*args, **kwargs): + if kwargs.get("where", {}).get("user_id") == "target-user-123": + return mock_user_row + return None + + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( + side_effect=mock_find_unique + ) + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + mock_request = mocker.MagicMock(spec=Request) + + admin_key = UserAPIKeyAuth( + user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + response = await user_info_v2( + request=mock_request, + user_id="target-user-123", + user_api_key_dict=admin_key, + ) + + assert isinstance(response, UserInfoV2Response) + assert response.metadata == { + "scim_metadata": {"givenName": "Jane", "familyName": "Doe"} + } + assert "scim_enterprise" not in (response.metadata or {}) + + +def test_build_user_info_response_redacts_scim_enterprise_metadata(): + """ + The shared /user/info builder strips scim_enterprise from the returned user row + while leaving every other metadata key intact. + """ + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _build_user_info_response, + ) + + user_row = { + "user_id": "target-user-123", + "metadata": { + "scim_metadata": {"givenName": "Jane"}, + "scim_enterprise": {"costCenter": "CC-42"}, + }, + } + + response = _build_user_info_response( + user_id="target-user-123", + user_info=user_row, + keys=None, + team_list=[], + teams_1=None, + ) + + assert response.user_info is not None + assert response.user_info["metadata"] == {"scim_metadata": {"givenName": "Jane"}} + assert "scim_enterprise" not in response.user_info["metadata"] + + @pytest.mark.asyncio async def test_user_info_v2_internal_user_can_query_self(mocker): """ @@ -2960,3 +3100,56 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker): user_request=user_request, user_api_key_dict=admin_caller ) assert result is not None + + +@pytest.mark.asyncio +async def test_resolve_user_email_metadata_maps_page_user_ids_to_email(mocker): + """Regression for LIT-3889. + + The Spend Per User chart rendered raw UUIDs because the per-user activity + breakdown carried no email. This resolver must turn the user_ids on the + page into {user_id: {user_email, user_alias}} so the chart can label each + spender, and it must only look up the user_ids actually present (not the + whole user table). + """ + + mock_prisma_client = mocker.MagicMock() + find_many = mocker.AsyncMock( + return_value=[ + SimpleNamespace( + user_id="u1", user_email="alice@example.com", user_alias="Alice" + ), + SimpleNamespace(user_id="u2", user_email=None, user_alias="bob-alias"), + ] + ) + mock_prisma_client.db.litellm_usertable.find_many = find_many + + records = [ + SimpleNamespace(user_id="u1"), + SimpleNamespace(user_id="u1"), # duplicate -> deduped + SimpleNamespace(user_id="u2"), + ] + + result = await _resolve_user_email_metadata(mock_prisma_client, records) + + assert result == { + "u1": {"user_email": "alice@example.com", "user_alias": "Alice"}, + "u2": {"user_email": None, "user_alias": "bob-alias"}, + } + where_arg = find_many.call_args.kwargs["where"] + assert set(where_arg["user_id"]["in"]) == {"u1", "u2"} + + +@pytest.mark.asyncio +async def test_resolve_user_email_metadata_skips_db_when_no_user_ids(mocker): + """No user_ids on the page (e.g. all spend is unattributed) means no query.""" + mock_prisma_client = mocker.MagicMock() + find_many = mocker.AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_usertable.find_many = find_many + + records = [SimpleNamespace(user_id=None), SimpleNamespace(user_id="")] + + result = await _resolve_user_email_metadata(mock_prisma_client, records) + + assert result == {} + find_many.assert_not_called() diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 0b5b5fb6ceb..f40904e234d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1197,6 +1197,136 @@ class TestListMCPServers: # Non-admin viewers get no env var config at all (not even names). assert result.env_vars is None + @pytest.mark.asyncio + async def test_fetch_single_mcp_server_sanitizes_for_view_only_admin(self): + """PROXY_ADMIN_VIEW_ONLY must NOT see credential-bearing fields. + + It previously passed the _user_has_admin_view gate (which also grants view-only + admins) and only had the explicit `credentials` field cleared, leaking secrets + embedded in url/static_headers/env_vars. Only a FULL PROXY_ADMIN may see those. + This test exercises the real role helpers (no patching of the gate).""" + mock_server = LiteLLM_MCPServerTable.model_construct( + server_id="leaky-server", + server_name="Leaky Server", + alias="Leaky Server", + transport=MCPTransport.http, + url="https://leaky.example.com/mcp?api_key=sk-embedded-in-url", + static_headers={"Authorization": "Bearer sk-secret-header"}, + env={"UPSTREAM_TOKEN": "sk-secret-env"}, + env_vars=[ + {"name": "GLOBAL_KEY", "value": "super-secret", "scope": "global"}, + ], + credentials={"auth_value": "sk-explicit-credential"}, + ) + + mock_prisma_client = MagicMock() + + mock_health_result = generate_mock_mcp_server_db_record( + server_id="leaky-server", alias="Leaky Server" + ) + mock_health_result.status = "healthy" + mock_health_result.last_health_check = datetime.now() + mock_health_result.health_check_error = None + + mock_manager = MagicMock() + mock_manager.add_server = AsyncMock() + mock_manager.health_check_server = AsyncMock(return_value=mock_health_result) + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=mock_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_mcp_server, + ) + + result = await fetch_mcp_server( + request=_make_mock_request(), + server_id="leaky-server", + user_api_key_dict=mock_user_auth, + ) + + assert result.server_id == "leaky-server" + assert result.credentials is None + assert result.url is None + assert result.static_headers is None + assert result.env == {} + assert result.env_vars is None + + @pytest.mark.asyncio + async def test_fetch_single_mcp_server_full_admin_still_sees_secrets(self): + """the fix must not over-redact for FULL PROXY_ADMIN, + who needs url/static_headers/env to populate the edit form.""" + mock_server = LiteLLM_MCPServerTable.model_construct( + server_id="admin-server", + server_name="Admin Server", + alias="Admin Server", + transport=MCPTransport.http, + url="https://admin.example.com/mcp", + static_headers={"Authorization": "Bearer sk-secret-header"}, + credentials={"auth_value": "sk-explicit-credential"}, + ) + + mock_prisma_client = MagicMock() + + mock_health_result = generate_mock_mcp_server_db_record( + server_id="admin-server", alias="Admin Server" + ) + mock_health_result.status = "healthy" + mock_health_result.last_health_check = datetime.now() + mock_health_result.health_check_error = None + + mock_manager = MagicMock() + mock_manager.add_server = AsyncMock() + mock_manager.health_check_server = AsyncMock(return_value=mock_health_result) + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.PROXY_ADMIN + ) + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma_client, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=mock_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_mcp_server, + ) + + result = await fetch_mcp_server( + request=_make_mock_request(), + server_id="admin-server", + user_api_key_dict=mock_user_auth, + ) + + # credentials field is always redacted; the rest must survive for full admin. + assert result.credentials is None + assert result.url == "https://admin.example.com/mcp" + assert result.static_headers == {"Authorization": "Bearer sk-secret-header"} + class TestTeamScopedMCPServerAccess: """Tests for cross-team information disclosure and restricted key bypass fixes.""" @@ -3338,6 +3468,10 @@ async def test_store_mcp_oauth_user_credential_returns_status(): return_value=generate_mock_mcp_server_db_record(server_id=server_id) ), ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.store_user_oauth_credential", new=AsyncMock(return_value=None), @@ -3693,18 +3827,12 @@ def _server_with_env_vars(server_id: str = "srv-env"): @pytest.mark.asyncio -@pytest.mark.parametrize( - "user_role, expected_global_value", - [ - (LitellmUserRoles.PROXY_ADMIN, "super-secret"), - (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ""), - ], -) -async def test_fetch_single_mcp_server_redacts_global_env_for_view_only_admin( - user_role, expected_global_value -): - """Read-only admins must not receive admin-supplied global env var secrets; - full admins still see them so the edit form can pre-fill.""" +async def test_fetch_single_mcp_server_env_vars_full_admin_vs_view_only(): + """full admins see admin-supplied global env var secrets so the edit + form can pre-fill; read-only admins now go through the non-admin sanitizer, which + drops env_vars entirely (the names alone, e.g. ADMIN_API_KEY, leak what secrets the + admin configured). Previously the view-only case merely blanked the global value + while keeping the names, which still leaked configuration metadata.""" server = _server_with_env_vars() health_result = generate_mock_mcp_server_db_record(server_id=server.server_id) @@ -3712,34 +3840,39 @@ async def test_fetch_single_mcp_server_redacts_global_env_for_view_only_admin( health_result.last_health_check = datetime.now() health_result.health_check_error = None - with ( - patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", - return_value=MagicMock(), - ), - patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", - AsyncMock(return_value=server), - ), - patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.add_server", - AsyncMock(return_value=None), - ), - patch( - "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server", - AsyncMock(return_value=health_result), - ), - ): - result = await mgmt_endpoints.fetch_mcp_server( - request=_make_mock_request(), - server_id=server.server_id, - user_api_key_dict=generate_mock_user_api_key_auth(user_role=user_role), - ) + async def _fetch(user_role): + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.add_server", + AsyncMock(return_value=None), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.health_check_server", + AsyncMock(return_value=health_result), + ), + ): + return await mgmt_endpoints.fetch_mcp_server( + request=_make_mock_request(), + server_id=server.server_id, + user_api_key_dict=generate_mock_user_api_key_auth(user_role=user_role), + ) - by_name = {ev.name: ev for ev in result.env_vars} - assert by_name["ADMIN_API_KEY"].value == expected_global_value - # Per-user placeholders are always preserved. + full_admin = await _fetch(LitellmUserRoles.PROXY_ADMIN) + by_name = {ev.name: ev for ev in full_admin.env_vars} + assert by_name["ADMIN_API_KEY"].value == "super-secret" assert by_name["USER_TOKEN"].value == "placeholder-hint" + + view_only = await _fetch(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + assert view_only.env_vars is None + # The source record must never be mutated. assert {ev.name: ev.value for ev in server.env_vars}[ "ADMIN_API_KEY" @@ -3747,18 +3880,67 @@ async def test_fetch_single_mcp_server_redacts_global_env_for_view_only_admin( @pytest.mark.asyncio -@pytest.mark.parametrize( - "user_role, expected_global_value", - [ - (LitellmUserRoles.PROXY_ADMIN, "super-secret"), - (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ""), - ], -) -async def test_fetch_all_mcp_servers_redacts_global_env_for_view_only_admin( - user_role, expected_global_value -): +async def test_fetch_all_mcp_servers_env_vars_full_admin_vs_view_only(): + """same posture as the single-server fetch. Full admins + keep the env var values; view-only admins get env_vars dropped via the non-admin + sanitizer rather than only having the global value blanked.""" server = _server_with_env_vars() + async def _fetch_all(user_role): + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", + return_value="view_all", + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager.get_all_mcp_servers_unfiltered", + AsyncMock(return_value=[server]), + ), + patch( + "litellm.proxy.proxy_server.prisma_client", + None, + ), + ): + return await mgmt_endpoints.fetch_all_mcp_servers( + user_api_key_dict=generate_mock_user_api_key_auth(user_role=user_role), + ) + + full_admin = await _fetch_all(LitellmUserRoles.PROXY_ADMIN) + by_name = {ev.name: ev for ev in full_admin[0].env_vars} + assert by_name["ADMIN_API_KEY"].value == "super-secret" + assert by_name["USER_TOKEN"].value == "placeholder-hint" + + view_only = await _fetch_all(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + assert view_only[0].env_vars is None + + assert {ev.name: ev.value for ev in server.env_vars}[ + "ADMIN_API_KEY" + ] == "super-secret" + + +def _leaky_list_server() -> "LiteLLM_MCPServerTable": + """A server whose url/static_headers/env carry embedded secrets, for the + list-endpoint sanitization tests. ``model_construct`` skips validation so + the raw values survive verbatim.""" + return LiteLLM_MCPServerTable.model_construct( + server_id="leaky-list-server", + server_name="Leaky List Server", + alias="Leaky List Server", + transport=MCPTransport.http, + url="https://leaky.example.com/mcp?api_key=sk-embedded-in-url", + static_headers={"Authorization": "Bearer sk-secret-header"}, + env={"UPSTREAM_TOKEN": "sk-secret-env"}, + env_vars=[ + {"name": "GLOBAL_KEY", "value": "super-secret", "scope": "global"}, + ], + credentials={"auth_value": "sk-explicit-credential"}, + ) + + +async def _fetch_all_via_view_all(user_role: LitellmUserRoles): + """Drive GET /v1/mcp/server in view_all mode for the given role using the + real role helpers (the full-admin gate is never patched).""" + server = _leaky_list_server() with ( patch( "litellm.proxy.management_endpoints.mcp_management_endpoints._get_user_mcp_management_mode", @@ -3776,13 +3958,46 @@ async def test_fetch_all_mcp_servers_redacts_global_env_for_view_only_admin( result = await mgmt_endpoints.fetch_all_mcp_servers( user_api_key_dict=generate_mock_user_api_key_auth(user_role=user_role), ) + return server, result - by_name = {ev.name: ev for ev in result[0].env_vars} - assert by_name["ADMIN_API_KEY"].value == expected_global_value - assert by_name["USER_TOKEN"].value == "placeholder-hint" - assert {ev.name: ev.value for ev in server.env_vars}[ - "ADMIN_API_KEY" - ] == "super-secret" + +@pytest.mark.asyncio +async def test_list_mcp_servers_sanitized_for_view_only_admin(): + """PROXY_ADMIN_VIEW_ONLY listing servers must go through the non-admin + sanitizer: url and static_headers cleared, env emptied, env_vars dropped. + A mutation swapping _user_is_full_admin() back to _user_has_admin_view() + (which also grants view-only admins) would return the raw url/headers and + fail this. The real role helpers are exercised; the gate is not patched.""" + source, result = await _fetch_all_via_view_all( + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + + assert len(result) == 1 + sanitized = result[0] + assert sanitized.server_id == "leaky-list-server" + assert sanitized.url is None + assert sanitized.static_headers is None + assert sanitized.env == {} + assert sanitized.env_vars is None + assert sanitized.credentials is None + + # The source record must never be mutated by sanitization. + assert source.url == "https://leaky.example.com/mcp?api_key=sk-embedded-in-url" + assert source.static_headers == {"Authorization": "Bearer sk-secret-header"} + + +@pytest.mark.asyncio +async def test_list_mcp_servers_full_admin_still_sees_secrets(): + """The view-only redaction must not over-redact for a FULL PROXY_ADMIN, + who needs url/static_headers to populate the edit form. Only the explicit + credentials field is cleared for full admins on the list endpoint.""" + _, result = await _fetch_all_via_view_all(LitellmUserRoles.PROXY_ADMIN) + + assert len(result) == 1 + raw = result[0] + assert raw.url == "https://leaky.example.com/mcp?api_key=sk-embedded-in-url" + assert raw.static_headers == {"Authorization": "Bearer sk-secret-header"} + assert raw.credentials is None def _make_env_var_server( @@ -4347,8 +4562,13 @@ class TestMCPUserEnvVarsAccessControl: ), patch.object( mgmt_endpoints, - "get_all_mcp_servers_for_user", - AsyncMock(return_value=[_make_env_var_server(server_id="other")]), + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object( + mgmt_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", + AsyncMock(return_value=["other"]), ), patch.object(mgmt_endpoints, "get_user_env_vars", get_user_env_vars), ): @@ -4378,7 +4598,12 @@ class TestMCPUserEnvVarsAccessControl: ), patch.object( mgmt_endpoints, - "get_all_mcp_servers_for_user", + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object( + mgmt_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch.object(mgmt_endpoints, "merge_user_env_vars", merge_mock), @@ -4412,7 +4637,12 @@ class TestMCPUserEnvVarsAccessControl: ), patch.object( mgmt_endpoints, - "get_all_mcp_servers_for_user", + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object( + mgmt_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch.object(mgmt_endpoints, "delete_user_env_vars", delete_mock), @@ -4444,8 +4674,13 @@ class TestMCPUserEnvVarsAccessControl: ), patch.object( mgmt_endpoints, - "get_all_mcp_servers_for_user", - AsyncMock(return_value=[server]), + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object( + mgmt_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", + AsyncMock(return_value=["srv-1"]), ), patch.object( mgmt_endpoints, @@ -4465,13 +4700,13 @@ class TestMCPUserEnvVarsAccessControl: @pytest.mark.asyncio async def test_admin_bypasses_access_check(self): - """Proxy admins must not be filtered by get_all_mcp_servers_for_user.""" + """Proxy admins must not be filtered by the allowed-server check.""" server = _make_env_var_server( server_id="srv-1", env_vars=_ENV_VARS_MIXED, static_headers=_STATIC_HEADERS_MIXED, ) - access_list_mock = AsyncMock(return_value=[]) + allowed_mock = AsyncMock(return_value=[]) with ( patch.object( mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() @@ -4480,7 +4715,9 @@ class TestMCPUserEnvVarsAccessControl: mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=server) ), patch.object( - mgmt_endpoints, "get_all_mcp_servers_for_user", access_list_mock + mgmt_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", + allowed_mock, ), patch.object( mgmt_endpoints, "get_user_env_vars", AsyncMock(return_value={}) @@ -4494,22 +4731,34 @@ class TestMCPUserEnvVarsAccessControl: ), ) assert result.server_id == "srv-1" - access_list_mock.assert_not_awaited() + allowed_mock.assert_not_awaited() @pytest.mark.asyncio async def test_non_admin_gets_403_not_404_for_inaccessible_server(self): - """Authorization must run before the existence check so a non-admin - cannot distinguish "server does not exist" (404) from "server exists but - you lack access" (403) and enumerate server IDs.""" - get_mcp_server_mock = AsyncMock(return_value=None) + """A non-admin cannot distinguish "server does not exist" (404) from + "server exists but you lack access" (403): both collapse to 403 so server + ids stay non-enumerable, even when neither the DB nor the registry has the + server.""" with ( patch.object( mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() ), - patch.object(mgmt_endpoints, "get_mcp_server", get_mcp_server_mock), + patch.object( + mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None) + ), patch.object( mgmt_endpoints, - "get_all_mcp_servers_for_user", + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object( + mgmt_endpoints.global_mcp_server_manager, + "get_mcp_server_by_id", + MagicMock(return_value=None), + ), + patch.object( + mgmt_endpoints.global_mcp_server_manager, + "get_allowed_mcp_servers", AsyncMock(return_value=[]), ), ): @@ -4522,7 +4771,6 @@ class TestMCPUserEnvVarsAccessControl: ), ) assert exc.value.status_code == 403 - get_mcp_server_mock.assert_not_awaited() def test_oauth2_flow_accepted_on_create_request(): @@ -4572,3 +4820,217 @@ def test_oauth2_flow_defaults_to_none_when_omitted(): assert ( LiteLLM_MCPServerTable(server_id="srv-1", transport="http").oauth2_flow is None ) + + +class TestPerUserCredentialConfigServerResolution: + """Per-user credential and env-var endpoints must resolve config-defined MCP + servers, which live only in the in-memory registry and never get a DB row, so + a user can store their BYOK key / OAuth token / env vars against them. The + same allowed-server authorization the MCP gateway enforces also gates these + writes for non-admins. + """ + + # 32-char sha256 stable id, the shape a config.yaml server gets. + CONFIG_SERVER_ID = "3a6a3f8633340371b49562c8c4682da9" + + def _registry_only_manager(self, *, is_byok: bool = False): + """A manager mock where the server exists only in the registry (DB miss).""" + config_server = generate_mock_mcp_server_config_record( + server_id=self.CONFIG_SERVER_ID, name="Config Server" + ) + record = generate_mock_mcp_server_db_record( + server_id=self.CONFIG_SERVER_ID + ).model_copy(update={"is_byok": is_byok}) + manager = MagicMock() + manager.get_mcp_server_by_id = MagicMock( + side_effect=lambda sid: ( + config_server if sid == self.CONFIG_SERVER_ID else None + ) + ) + manager._build_mcp_server_table = MagicMock(return_value=record) + manager.get_allowed_mcp_servers = AsyncMock(return_value=[]) + return manager + + @pytest.mark.asyncio + async def test_store_oauth_credential_resolves_config_server_for_admin(self): + """OBO token persists for a config-defined server (DB miss, registry hit). + Before the registry fallback this raised 404 "MCP Server not found".""" + manager = self._registry_only_manager() + store_mock = AsyncMock(return_value=None) + with ( + patch.object( + mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() + ), + patch.object( + mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None) + ), + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object(mgmt_endpoints, "store_user_oauth_credential", store_mock), + patch.object( + mgmt_endpoints, + "get_user_oauth_credential", + AsyncMock(return_value={"expires_at": None}), + ), + ): + result = await mgmt_endpoints.store_mcp_oauth_user_credential( + server_id=self.CONFIG_SERVER_ID, + payload=mgmt_endpoints.MCPOAuthUserCredentialRequest( + access_token="tok", expires_in=3600 + ), + user_api_key_dict=generate_mock_user_api_key_auth(user_id="admin"), + ) + assert result.has_credential is True + store_mock.assert_awaited_once() + manager.get_mcp_server_by_id.assert_called_once_with(self.CONFIG_SERVER_ID) + + @pytest.mark.asyncio + async def test_store_byok_credential_resolves_config_server_for_admin(self): + """BYOK key persists for a config-defined BYOK server (DB miss, registry + hit). Before the registry fallback this raised 404.""" + manager = self._registry_only_manager(is_byok=True) + store_mock = AsyncMock(return_value=None) + with ( + patch.object( + mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() + ), + patch.object( + mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None) + ), + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object(mgmt_endpoints, "store_user_credential", store_mock), + ): + result = await mgmt_endpoints.store_mcp_user_credential( + server_id=self.CONFIG_SERVER_ID, + payload=mgmt_endpoints.MCPUserCredentialRequest(credential="my-key"), + user_api_key_dict=generate_mock_user_api_key_auth(user_id="admin"), + ) + assert result.has_credential is True + store_mock.assert_awaited_once() + + @pytest.mark.asyncio + async def test_store_oauth_credential_forbidden_for_non_admin_without_access(self): + """A non-admin storing a credential for a server not in their allowed set + gets 403 and no row is written (the store endpoints had no authz before).""" + manager = self._registry_only_manager() + manager.get_allowed_mcp_servers = AsyncMock(return_value=[]) + store_mock = AsyncMock(return_value=None) + with ( + patch.object( + mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() + ), + patch.object( + mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None) + ), + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object( + mgmt_endpoints, + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object(mgmt_endpoints, "store_user_oauth_credential", store_mock), + ): + with pytest.raises(HTTPException) as exc: + await mgmt_endpoints.store_mcp_oauth_user_credential( + server_id=self.CONFIG_SERVER_ID, + payload=mgmt_endpoints.MCPOAuthUserCredentialRequest( + access_token="tok", expires_in=3600 + ), + user_api_key_dict=generate_mock_user_api_key_auth( + user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER + ), + ) + assert exc.value.status_code == 403 + store_mock.assert_not_awaited() + + @pytest.mark.asyncio + async def test_store_oauth_credential_allowed_for_non_admin_with_access(self): + """A non-admin with the config server in their allowed set persists the + token; proves the non-admin authz uses the registry-aware allowed set.""" + manager = self._registry_only_manager() + manager.get_allowed_mcp_servers = AsyncMock( + return_value=[self.CONFIG_SERVER_ID] + ) + store_mock = AsyncMock(return_value=None) + with ( + patch.object( + mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() + ), + patch.object( + mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None) + ), + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object( + mgmt_endpoints, + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object(mgmt_endpoints, "store_user_oauth_credential", store_mock), + patch.object( + mgmt_endpoints, + "get_user_oauth_credential", + AsyncMock(return_value={"expires_at": None}), + ), + ): + result = await mgmt_endpoints.store_mcp_oauth_user_credential( + server_id=self.CONFIG_SERVER_ID, + payload=mgmt_endpoints.MCPOAuthUserCredentialRequest( + access_token="tok", expires_in=3600 + ), + user_api_key_dict=generate_mock_user_api_key_auth( + user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER + ), + ) + assert result.has_credential is True + store_mock.assert_awaited_once() + + @pytest.mark.asyncio + async def test_store_env_vars_resolves_config_server_for_non_admin_with_access( + self, + ): + """Per-user env vars persist for a config server a non-admin may access. + The non-admin path previously used a DB-only access list that never + included config servers, so this 403'd before the fix.""" + env_var_server = _make_env_var_server( + server_id=self.CONFIG_SERVER_ID, + env_vars=_ENV_VARS_MIXED, + static_headers=_STATIC_HEADERS_MIXED, + ) + manager = MagicMock() + manager.get_mcp_server_by_id = MagicMock( + return_value=generate_mock_mcp_server_config_record( + server_id=self.CONFIG_SERVER_ID + ) + ) + manager._build_mcp_server_table = MagicMock(return_value=env_var_server) + manager.get_allowed_mcp_servers = AsyncMock( + return_value=[self.CONFIG_SERVER_ID] + ) + merge_mock = AsyncMock(return_value={"CORP_USERNAME": "alice"}) + with ( + patch.object( + mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock() + ), + patch.object( + mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None) + ), + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object( + mgmt_endpoints, + "build_effective_auth_contexts", + AsyncMock(return_value=[object()]), + ), + patch.object(mgmt_endpoints, "merge_user_env_vars", merge_mock), + ): + result = await mgmt_endpoints.store_mcp_user_env_vars( + server_id=self.CONFIG_SERVER_ID, + payload=mgmt_endpoints.MCPUserEnvVarsRequest( + values={"CORP_USERNAME": "alice"} + ), + user_api_key_dict=generate_mock_user_api_key_auth( + user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER + ), + ) + merge_mock.assert_awaited_once() + _, _, _, updates, _ = merge_mock.await_args.args + assert updates == {"CORP_USERNAME": "alice"} + assert result.server_id == self.CONFIG_SERVER_ID diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index 965580e8758..c77ac11ffc1 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -14,6 +14,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( _extract_requested_mcp_access_groups, _extract_requested_mcp_server_ids, _resolve_team_allowed_mcp_servers, + _rewrite_object_permission_mcp_servers, _set_object_permission, validate_key_mcp_servers_against_team, validate_key_search_tools_against_team, @@ -111,6 +112,31 @@ def test_extract_requested_mcp_server_ids_none(): assert _extract_requested_mcp_server_ids({}) == set() +def test_extract_requested_mcp_server_ids_excludes_no_mcp_servers_sentinel(): + obj_perm = {"mcp_servers": ["no-mcp-servers", "server-1"]} + assert _extract_requested_mcp_server_ids(obj_perm) == {"server-1"} + + +def test_rewrite_object_permission_mcp_servers_preserves_sentinel(): + obj_perm = {"mcp_servers": ["no-mcp-servers", "alias-1"]} + _rewrite_object_permission_mcp_servers(obj_perm, {"alias-1": {"server-1"}}) + assert obj_perm["mcp_servers"] == ["no-mcp-servers", "server-1"] + + +@pytest.mark.asyncio +async def test_validate_no_mcp_servers_sentinel_passes_and_preserved(): + """A key scoped to no-mcp-servers passes team validation untouched, keeping the + sentinel so it is not mistaken for an unknown server and rejected.""" + team_obj = _make_team_obj(mcp_servers=["server-1"]) + obj_perm = {"mcp_servers": ["no-mcp-servers"]} + result = await validate_key_mcp_servers_against_team( + object_permission=obj_perm, + team_obj=team_obj, + ) + assert result == obj_perm + assert obj_perm["mcp_servers"] == ["no-mcp-servers"] + + # ---- Tests for _extract_requested_mcp_access_groups ---- diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index b800c82c75d..947a7a64beb 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -1885,3 +1885,310 @@ class TestNonStreamingResponseRedaction: leaked = logging_obj.model_call_details.get("complete_streaming_response") assert leaked is None assert redacted.choices[0].message.content == "redacted-by-litellm" + + +def _sse_bytes(data: dict) -> bytes: + return f"event: {data['type']}\ndata: {json.dumps(data)}\n\n".encode() + + +class TestAnthropicUsageOnlyFallback: + """When stream_chunk_builder cannot reassemble a large/agentic stream (returns + None or raises), Anthropic still emits token usage in the message_start / + message_delta SSE events. The handler must recover usage-only so the request is + priced instead of being dropped from SpendLogs while Anthropic billed the tokens.""" + + _CHUNKS = [ + _sse_bytes( + { + "type": "message_start", + "message": { + "model": "claude-3-5-haiku-20241022", + "usage": { + "input_tokens": 100, + "cache_read_input_tokens": 40, + "cache_creation_input_tokens": 20, + "output_tokens": 1, + }, + }, + } + ), + _sse_bytes( + { + "type": "message_delta", + "usage": { + "output_tokens": 55, + "server_tool_use": {"web_search_requests": 2}, + }, + } + ), + ] + + def test_build_usage_only_recovers_cache_inclusive_usage(self): + response = ( + AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=self._CHUNKS, model="claude-3-5-haiku-20241022" + ) + ) + assert response is not None + usage = response.usage + # prompt_tokens must be cache-inclusive (input + cache_read + cache_creation) + assert usage.prompt_tokens == 160 + assert usage.completion_tokens == 55 + assert usage._cache_read_input_tokens == 40 + assert usage._cache_creation_input_tokens == 20 + assert usage.prompt_tokens_details.cached_tokens == 40 + assert usage.server_tool_use.web_search_requests == 2 + + def test_build_usage_only_returns_none_without_usage_events(self): + chunks = [_sse_bytes({"type": "content_block_delta", "delta": {"text": "hi"}})] + assert ( + AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=chunks, model="claude-3-5-haiku-20241022" + ) + is None + ) + + def test_build_usage_only_recovers_cache_split_server_tools_and_model(self): + # the model is "unknown" up-front and only the 5m/1h cache split is sent + # (no flat cache_creation_input_tokens); web/tool-search and geo arrive in + # message_delta. All must be recovered and priced, not left at $0. + chunks = [ + "event: ping\ndata: [DONE]\n\n", # ignored sentinel between real events + _sse_bytes( + { + "type": "message_start", + "message": { + "model": "claude-opus-4-6", + "usage": { + "input_tokens": 80, + "output_tokens": 1, + "cache_creation": { + "ephemeral_5m_input_tokens": 12, + "ephemeral_1h_input_tokens": 8, + }, + "inference_geo": "us", + }, + }, + } + ), + _sse_bytes( + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use"}, + "usage": { + "output_tokens": 40, + "cache_read_input_tokens": 5, + "inference_geo": "us", + "server_tool_use": { + "web_search_requests": 1, + "tool_search_requests": 3, + }, + }, + } + ), + ] + response = ( + AnthropicPassthroughLoggingHandler._build_usage_only_response_from_chunks( + all_chunks=chunks, model="unknown" + ) + ) + assert response is not None + assert response.model == "claude-opus-4-6" + # the real stop_reason is surfaced, not a hardcoded "stop" + assert response.choices[0].finish_reason == "tool_calls" + usage = response.usage + # 80 input + 20 cache_creation (derived from 12+8) + 5 cache_read + assert usage.prompt_tokens == 105 + assert usage.completion_tokens == 40 + assert usage._cache_creation_input_tokens == 20 + assert usage._cache_read_input_tokens == 5 + assert usage.server_tool_use.web_search_requests == 1 + assert usage.server_tool_use.tool_search_requests == 3 + + @pytest.mark.parametrize( + "event_str,expected", + [ + ("data: [DONE]", None), + ("data: ", None), + ("data: {not-json", None), + ("event: ping", None), + ('data: {"a": 1}', {"a": 1}), + ], + ) + def test_extract_sse_data_handles_malformed_and_sentinel_lines( + self, event_str, expected + ): + assert ( + AnthropicPassthroughLoggingHandler._extract_sse_data(event_str) == expected + ) + + def _real_logging_obj(self): + from litellm.litellm_core_utils.litellm_logging import Logging as RealLoggingObj + + logging_obj = RealLoggingObj( + model="claude-3-5-haiku-20241022", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="1", + ) + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + return logging_obj + + @patch("litellm.completion_cost") + @patch.object( + AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response" + ) + def test_handler_falls_back_when_assembly_returns_none( + self, mock_assemble, mock_cost + ): + mock_assemble.return_value = None + mock_cost.return_value = 0.0021 + logging_obj = self._real_logging_obj() + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/anthropic/v1/messages", + request_body={"model": "claude-3-5-haiku-20241022", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=list(self._CHUNKS), + end_time=datetime.now(), + ) + + assert result["result"] is not None + assert result["result"].usage.completion_tokens == 55 + assert result["kwargs"]["response_cost"] == 0.0021 + + @patch("litellm.completion_cost") + @patch.object( + AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response" + ) + def test_handler_falls_back_when_assembly_raises(self, mock_assemble, mock_cost): + import litellm + + mock_assemble.side_effect = litellm.APIError( + status_code=500, + message="boom", + llm_provider="anthropic", + model="claude-3-5-haiku-20241022", + ) + mock_cost.return_value = 0.0021 + logging_obj = self._real_logging_obj() + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/anthropic/v1/messages", + request_body={"model": "claude-3-5-haiku-20241022", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=list(self._CHUNKS), + end_time=datetime.now(), + ) + + # a raise from stream_chunk_builder must be treated like a None result, + # not propagate out and drop the request from SpendLogs + assert result["result"] is not None + assert result["result"].usage.completion_tokens == 55 + assert result["kwargs"]["response_cost"] == 0.0021 + + @patch.object( + AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response" + ) + def test_handler_returns_none_when_no_usage_recoverable(self, mock_assemble): + # assembly fails AND the chunks carry no usage event, so there is nothing + # to price; the handler must return None rather than fabricate a response + mock_assemble.return_value = None + logging_obj = self._real_logging_obj() + chunks = [_sse_bytes({"type": "content_block_delta", "delta": {"text": "hi"}})] + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/anthropic/v1/messages", + request_body={"model": "claude-3-5-haiku-20241022", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=chunks, + end_time=datetime.now(), + ) + + assert result["result"] is None + assert result["kwargs"] == {} + + @patch.object( + AnthropicPassthroughLoggingHandler, "_build_usage_only_response_from_chunks" + ) + @patch.object( + AnthropicPassthroughLoggingHandler, "_build_complete_streaming_response" + ) + def test_handler_does_not_crash_when_usage_only_fallback_raises( + self, mock_assemble, mock_fallback + ): + # if the usage-only fallback itself raises, it must be treated as None and + # drop gracefully, not propagate out and crash the success handler + mock_assemble.return_value = None + mock_fallback.side_effect = Exception("fallback boom") + logging_obj = self._real_logging_obj() + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/anthropic/v1/messages", + request_body={"model": "claude-3-5-haiku-20241022", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=list(self._CHUNKS), + end_time=datetime.now(), + ) + + assert result["result"] is None + assert result["kwargs"] == {} + + +class TestAnthropicResponseCostRecordedOnModelCallDetails: + """The pass-through success path reads spend from + model_call_details["response_cost"], not from kwargs, so the streaming payload + builder must record it there or streaming pass-through logs $0.""" + + def test_create_payload_records_response_cost_on_model_call_details(self): + from litellm.types.utils import Choices, Message, ModelResponse + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.get_router_model_id.return_value = None + logging_obj.litellm_params = {} + logging_obj.litellm_call_id = "test-call-id" + + response = ModelResponse( + id="test-id", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="hello", role="assistant"), + ) + ], + created=1234567890, + model="claude-3-7-sonnet-20250219", + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ) + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=response, + model="claude-3-7-sonnet-20250219", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + assert ( + logging_obj.model_call_details["response_cost"] == kwargs["response_cost"] + ) + assert logging_obj.model_call_details["response_cost"] > 0 diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 362f4986c62..379e219b29b 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -2682,15 +2682,19 @@ async def test_add_litellm_data_to_request_adds_headers_to_metadata(): version="1.0", ) - # Verify headers are added to metadata for guardrails - assert "metadata" in result, "metadata should be present in result" - assert "headers" in result["metadata"], "headers should be present in metadata" + # Verify headers are added to litellm_metadata for guardrails. + # Bedrock passthrough uses litellm_metadata to prevent key-level + # tags from leaking into the provider payload (GH#30629). + assert "litellm_metadata" in result, "litellm_metadata should be present in result" + assert ( + "headers" in result["litellm_metadata"] + ), "headers should be present in litellm_metadata" assert isinstance( - result["metadata"]["headers"], dict + result["litellm_metadata"]["headers"], dict ), "headers should be a dictionary" # Verify specific headers are accessible (important for guardrails) - headers = result["metadata"]["headers"] + headers = result["litellm_metadata"]["headers"] assert ( "user-agent" in headers or "User-Agent" in headers ), "User-Agent header should be accessible in metadata" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index f73aee77cc1..38990644154 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -118,3 +118,20 @@ async def test_chunk_processor_does_not_schedule_logging_when_no_chunks(): assert received == [] mock_route.assert_not_called() + + +def test_convert_raw_bytes_survives_truncated_multibyte_sequence(): + """A stream cut mid-multibyte-sequence (client disconnect) must still decode + via errors="replace" so the usage events already received are logged, instead + of raising UnicodeDecodeError and dropping the whole request from SpendLogs.""" + # the 3-byte "☃" (E2 98 83) is cut after 2 bytes, leaving an invalid sequence + # that strict utf-8 decode would raise on, discarding the message_delta line too + truncated_codepoint = "☃".encode("utf-8")[:2] + raw_bytes = [ + b'data: {"text": "' + truncated_codepoint, + b'\ndata: {"type": "message_delta"}\n', + ] + + lines = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes) + + assert any('"type": "message_delta"' in line for line in lines) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index e89ada5bdef..df14dc5b5dc 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -15,8 +15,6 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock -import pytest - from .conftest import VOLATILE_KEYS, normalize @@ -248,6 +246,193 @@ def test_config_field_info_field_not_in_db(client, auth_as, mock_prisma, monkeyp assert "not in DB" in response.json().get("detail", {}).get("error", "") +def test_config_field_info_redacts_nested_secret_for_view_only_admin( + client, auth_as, mock_prisma, monkeypatch +): + """A view-only admin reading a structured field must not receive nested + credentials. database_args carries aws_web_identity_token (a DynamoDB + role-assumption credential); it must come back redacted while non-secret + siblings like region_name stay visible.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + row = MagicMock() + row.param_value = { + "database_args": { + "region_name": "us-east-1", + "user_table_name": "LiteLLM_UserTable", + "aws_web_identity_token": "sk-super-secret-token", + } + } + table.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + response = client.get( + "/config/field/info", params={"field_name": "database_args"} + ) + assert response.status_code == 200 + value = response.json()["field_value"] + assert value["aws_web_identity_token"] == "REDACTED" + assert value["region_name"] == "us-east-1" + assert value["user_table_name"] == "LiteLLM_UserTable" + + +def test_config_field_info_full_admin_sees_nested_secret( + client, auth_as, mock_prisma, monkeypatch +): + """The redaction must not over-redact for a full PROXY_ADMIN, who needs + the real nested value to populate the edit form.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + row = MagicMock() + row.param_value = { + "database_args": { + "region_name": "us-east-1", + "aws_web_identity_token": "sk-super-secret-token", + } + } + table.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get( + "/config/field/info", params={"field_name": "database_args"} + ) + assert response.status_code == 200 + value = response.json()["field_value"] + assert value["aws_web_identity_token"] == "sk-super-secret-token" + assert value["region_name"] == "us-east-1" + + +def test_config_field_info_redacts_top_level_scalar_for_view_only( + client, auth_as, mock_prisma, monkeypatch +): + """The top-level scalar branch must also redact for a view-only admin. + database_url carries DB credentials and is not caught by the name masker, + so it is in the explicit secret set.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + row = MagicMock() + row.param_value = {"database_url": "postgresql://admin:p4ss@db:5432/litellm"} + table.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + response = client.get( + "/config/field/info", params={"field_name": "database_url"} + ) + assert response.status_code == 200 + assert response.json()["field_value"] == "REDACTED" + + +def test_redact_general_setting_value_recurses_list_of_dicts(): + """The list branch of the recursor redacts secret leaves inside each dict + while non-secret keys survive, and a full admin gets the value untouched.""" + from litellm.proxy import proxy_server as ps + + value = [ + {"path": "/foo", "headers": {"Authorization": "Bearer sk-x"}}, + {"path": "/bar", "client_secret": "sk-y"}, + ] + redacted = ps._redact_general_setting_value( + "some_list_field", value, is_full_admin=False + ) + assert redacted[0]["headers"]["Authorization"] == "REDACTED" + assert redacted[0]["path"] == "/foo" + assert redacted[1]["client_secret"] == "REDACTED" + assert redacted[1]["path"] == "/bar" + assert ( + ps._redact_general_setting_value("some_list_field", value, is_full_admin=True) + == value + ) + + +def test_redact_secret_values_in_obj_fails_closed_at_max_depth(): + """Past _REDACT_SECRET_MAX_DEPTH the whole subtree is replaced with + "REDACTED" rather than returned verbatim, so a secret buried below the cap + can never leak via depth-overrun. A future refactor that flips the cap + branch to fail-open would surface here.""" + from litellm.proxy import proxy_server as ps + + # leaf and wrap keys are both NON-secret so neither the key-name + # short-circuit nor the explicit-secret set catches the leak. The cap is + # the only thing standing between the secret and the response — flip the + # cap to fail-open and the secret comes back verbatim. + nested: object = {"notes": "sk-leak-bottom"} + for _ in range(ps._REDACT_SECRET_MAX_DEPTH + 2): + nested = {"wrap": nested} + + out = ps._redact_general_setting_value( + "some_struct_field", nested, is_full_admin=False + ) + # the secret must not survive anywhere in the returned tree + assert "sk-leak-bottom" not in repr(out) + + # full admin is unaffected by the cap — the value comes back untouched + admin_out = ps._redact_general_setting_value( + "some_struct_field", nested, is_full_admin=True + ) + assert admin_out is nested + + +def test_config_list_redacts_pass_through_secret_for_view_only( + client, auth_as, mock_prisma, monkeypatch +): + """/config/list must not leak pass_through_endpoints upstream credentials + to a view-only admin. pass_through_endpoints is a known secret-bearing + field, so a non-admin gets it redacted; a full admin still sees it.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + row = MagicMock() + row.param_value = {"max_parallel_requests": 3} + table.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr( + ps, + "general_settings", + { + "pass_through_endpoints": [ + { + "path": "/foo", + "target": "https://upstream.example.com", + "headers": {"Authorization": "Bearer sk-UPSTREAM-SECRET"}, + } + ] + }, + ) + + def _pass_through_value(body): + return next( + entry["field_value"] + for entry in body + if entry["field_name"] == "pass_through_endpoints" + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + view_resp = client.get( + "/config/list", params={"config_type": "general_settings"} + ) + assert view_resp.status_code == 200 + assert "sk-UPSTREAM-SECRET" not in view_resp.text + assert _pass_through_value(view_resp.json()) == "REDACTED" + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + admin_resp = client.get( + "/config/list", params={"config_type": "general_settings"} + ) + assert admin_resp.status_code == 200 + admin_value = _pass_through_value(admin_resp.json()) + assert admin_value[0]["headers"]["Authorization"] == "Bearer sk-UPSTREAM-SECRET" + + # --------------------------------------------------------------------------- # GET /config/list # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py index 017f4bd4368..3bf14c08d14 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py @@ -9,7 +9,7 @@ Pins (PR2): from __future__ import annotations -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest @@ -128,6 +128,104 @@ def test_v1_model_info_no_model_list_error(client, auth_as, null_router, path): assert "LLM Model List not loaded" in response.text +# --------------------------------------------------------------------------- +# GET /model/info — team BYOK scoping (issue #30983) +# --------------------------------------------------------------------------- + +_BYOK_TEAM_ID = "team-abc" +_BYOK_PUBLIC_NAME = "my-byok-gpt-4" +_BYOK_INTERNAL_NAME = f"model_name_{_BYOK_TEAM_ID}_0123456789abcdef" + + +@pytest.fixture +def byok_team_router(monkeypatch): + """Router holding one team-scoped BYOK deployment for team `team-abc`. + + Mirrors how a team's own-key BYOK model lives in the router: the routing + key is an internal mangled name while the public name lives in + `model_info.team_public_model_name`. + """ + byok_deployment = { + "model_name": _BYOK_INTERNAL_NAME, + "litellm_params": {"model": "openai/gpt-4"}, + "model_info": { + "id": "byok-deployment-id", + "db_model": True, + "team_id": _BYOK_TEAM_ID, + "team_public_model_name": _BYOK_PUBLIC_NAME, + }, + } + + router = MagicMock() + router.model_list = [byok_deployment] + router.get_model_list_from_model_alias = MagicMock(return_value=[]) + router.get_model_names = MagicMock(return_value=[]) + router.get_model_access_groups = MagicMock(return_value={}) + router.get_model_ids = MagicMock(return_value=[]) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", [byok_deployment]) + monkeypatch.setattr(proxy_server, "user_model", None) + yield router + + +@pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"]) +def test_model_info_team_key_sees_own_byok_model(client, auth_as, byok_team_router, mock_prisma, monkeypatch, path): + """Regression for #30983: a team key (user_id=None) must see its own + team's BYOK model under the public name. + + Before the fix `_get_caller_byok_team_scope` keyed only off the bound + user's team memberships, returned an empty set for a team key, and the + BYOK row was dropped -> `{"data": []}`. + """ + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + mock_prisma.db.litellm_usertable.find_unique.return_value = None + + with auth_as( + role=LitellmUserRoles.INTERNAL_USER, + user_id=None, + team_id=_BYOK_TEAM_ID, + team_models=[_BYOK_PUBLIC_NAME], + ): + response = client.get(path) + + assert response.status_code == 200 + data = response.json()["data"] + surfaced_names = [m.get("model_name") for m in data] + assert _BYOK_PUBLIC_NAME in surfaced_names + assert _BYOK_INTERNAL_NAME not in surfaced_names + + +@pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"]) +def test_model_info_team_key_cannot_see_other_teams_byok_model( + client, auth_as, byok_team_router, mock_prisma, monkeypatch, path +): + """A team key for a different team must NOT see team-abc's BYOK row. + + Guards the fix from over-broadening into a cross-team metadata leak. + """ + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + mock_prisma.db.litellm_usertable.find_unique.return_value = None + + with auth_as( + role=LitellmUserRoles.INTERNAL_USER, + user_id=None, + team_id="other-team", + team_models=[_BYOK_PUBLIC_NAME], + ): + response = client.get(path) + + assert response.status_code == 200 + data = response.json()["data"] + surfaced_names = [m.get("model_name") for m in data] + assert _BYOK_PUBLIC_NAME not in surfaced_names + assert _BYOK_INTERNAL_NAME not in surfaced_names + + # --------------------------------------------------------------------------- # GET /model_group/info # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index 40d590132aa..25e84fb59a7 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -14,7 +14,11 @@ from unittest.mock import AsyncMock, MagicMock import pytest import litellm.proxy.proxy_server as ps -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_UserTable, + LitellmUserRoles, + UserAPIKeyAuth, +) from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator from litellm.proxy.proxy_server import ( _get_proxy_model_info, @@ -316,8 +320,10 @@ async def test_model_info_v1_unrestricted_key_hides_other_team_byok(monkeypatch) @pytest.mark.asyncio async def test_model_info_v1_service_key_hides_all_team_byok(monkeypatch): - """A key without a resolvable user (e.g. CI/service token) sees only - global deployments, never any team-scoped BYOK rows.""" + """A key with no resolvable user and no team (e.g. a CI/service token + created outside any team) sees only global deployments, never team-scoped + BYOK rows. A team-scoped key does see its own team's rows (issue #30983), + pinned by the /model/info route tests.""" team_row = _team_row() other_team_row = _other_team_row() global_row = { @@ -343,7 +349,7 @@ async def test_model_info_v1_service_key_hides_all_team_byok(monkeypatch): caller = UserAPIKeyAuth( user_id=None, user_role=LitellmUserRoles.INTERNAL_USER, - team_id="team-abc-123", + team_id=None, models=[], team_models=[], ) @@ -352,6 +358,106 @@ async def test_model_info_v1_service_key_hides_all_team_byok(monkeypatch): assert [m["model_info"]["id"] for m in resp["data"]] == ["global-id-1"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "find_unique", + [ + AsyncMock(return_value=MagicMock(teams=[])), + AsyncMock(return_value=None), + AsyncMock(side_effect=RuntimeError("db down")), + ], + ids=["user-not-in-team", "user-row-missing", "user-lookup-error"], +) +async def test_model_info_v1_team_key_sees_own_byok_regardless_of_user_lookup( + monkeypatch, find_unique +): + """A team-scoped key sees its own team's BYOK rows even when the bound user + is not a member of that team, has no DB row, or the lookup errors; the + key's team_id is authoritative (issue #30983). Other teams' rows stay + hidden.""" + global_row = { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [_team_row(), _other_team_row(), global_row] + router.get_model_names.return_value = ["gpt-4"] + router.get_model_access_groups.return_value = {} + + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = find_unique + + async def _populate(**kwargs): + return kwargs["all_models"] + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", prisma_client) + monkeypatch.setattr(ps, "_populate_team_access_on_models", _populate) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + caller = UserAPIKeyAuth( + user_id="user-1", + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="team-abc-123", + models=[], + team_models=[], + ) + resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) + + assert [m["model_info"]["id"] for m in resp["data"]] == ["byok-id-1", "global-id-1"] + + +@pytest.mark.asyncio +async def test_model_info_v1_user_team_membership_grants_byok(monkeypatch): + """A user's own team memberships still grant that team's BYOK rows, unioned + with any team the key itself is scoped to.""" + global_row = { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [_team_row(), _other_team_row(), global_row] + router.get_model_names.return_value = ["gpt-4"] + router.get_model_access_groups.return_value = {} + + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=MagicMock(teams=["team-other"]) + ) + + async def _populate(**kwargs): + return kwargs["all_models"] + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", prisma_client) + monkeypatch.setattr(ps, "_populate_team_access_on_models", _populate) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + caller = UserAPIKeyAuth( + user_id="user-2", + user_role=LitellmUserRoles.INTERNAL_USER, + team_id=None, + models=[], + team_models=[], + ) + resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) + + assert [m["model_info"]["id"] for m in resp["data"]] == [ + "byok-id-other", + "global-id-1", + ] + + @pytest.mark.asyncio async def test_model_info_v1_populates_access_via_team_ids(monkeypatch): """`/v1/model/info` must populate access_via_team_ids when the DB is connected.""" @@ -1258,3 +1364,83 @@ async def test_retrieve_model_by_inaccessible_public_name_404s(monkeypatch): assert exc_info.value.status_code == 404 router.get_deployment_by_model_group_name.assert_not_called() + + +def test_get_direct_access_models_expands_all_proxy_models_sentinel(): + """A user provisioned with 'all-proxy-models' has direct access to every non-team + deployment. The sentinel must resolve via get_model_ids, not be looked up as a + literal model_name (which matches nothing). Regression for GH#22791.""" + router = MagicMock() + router.get_model_ids.return_value = ["global-id-1", "global-id-2"] + router.get_model_list.return_value = [] + + user = LiteLLM_UserTable( + user_id="u", + models=[ps.SpecialModelNames.all_proxy_models.value], + teams=[], + ) + + result = ps.get_direct_access_models(user_db_object=user, llm_router=router) + + assert result == ["global-id-1", "global-id-2"] + router.get_model_ids.assert_called_once_with(exclude_team_models=True) + router.get_model_list.assert_not_called() + + +def test_get_direct_access_models_resolves_explicit_model_names(): + """Without the sentinel, only the user's explicitly listed models resolve to ids; + the all-proxy-models shortcut must not fire.""" + router = MagicMock() + router.get_model_list.return_value = [{"model_info": {"id": "gpt4o-id"}}] + + user = LiteLLM_UserTable(user_id="u", models=["gpt-4o"], teams=[]) + + result = ps.get_direct_access_models(user_db_object=user, llm_router=router) + + assert result == ["gpt4o-id"] + router.get_model_ids.assert_not_called() + router.get_model_list.assert_called_once_with(model_name="gpt-4o") + + +@pytest.mark.asyncio +async def test_populate_team_access_grants_all_proxy_models_user_direct_access( + monkeypatch, +): + """An internal user provisioned with 'all-proxy-models' and no teams must see proxy + models on the Models+Endpoints page. Before the fix _populate_team_access_on_models + marked direct_access=False, so _filter_models_to_user_accessible dropped every model + and the page was empty. Regression for GH#22791.""" + global_row = { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + + router = MagicMock() + router.get_model_ids.return_value = ["global-id-1"] + + user_row = LiteLLM_UserTable( + user_id="u", + user_role=LitellmUserRoles.INTERNAL_USER.value, + models=[ps.SpecialModelNames.all_proxy_models.value], + teams=[], + ) + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row) + + monkeypatch.setattr(ps, "get_all_team_models", AsyncMock(return_value={})) + + caller = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.INTERNAL_USER, team_models=[] + ) + + populated = await ps._populate_team_access_on_models( + user_api_key_dict=caller, + prisma_client=prisma_client, + llm_router=router, + all_models=[global_row], + ) + visible = ps._filter_models_to_user_accessible(populated) + + assert [m["model_info"]["id"] for m in visible] == ["global-id-1"] + assert visible[0]["model_info"]["direct_access"] is True diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 48d0b1deadd..d920488c352 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -201,6 +201,48 @@ def test_anthropic_provider_fields_support_byok(): ), "api_base must appear before api_key in credential_fields (matches AI21 and ANTHROPIC_TEXT convention)." +def test_bedrock_mantle_provider_fields(): + """Amazon Bedrock Mantle must be a selectable provider in the Add Model flow. + + The dropdown is driven entirely by /public/providers/fields, so a missing + entry means Mantle cannot be added through the UI at all (regression guard + for LIT-3885). The credential fields must match what the backend actually + honors: an optional bearer api_key (BYOK), the AWS SigV4 chain, a region, + and an api_base override. + """ + app_instance = FastAPI() + app_instance.include_router(router) + test_client = TestClient(app_instance) + + response = test_client.get("/public/providers/fields") + assert response.status_code == 200 + providers = response.json() + + mantle = next((p for p in providers if p["provider"] == "BedrockMantle"), None) + assert mantle is not None, "Bedrock Mantle provider entry not found" + + # provider must equal the UI provider_map key so the model dropdown resolves + # bedrock_mantle models; litellm_provider must be the backend slug. + assert mantle["provider_display_name"] == "Amazon Bedrock Mantle" + assert mantle["litellm_provider"] == "bedrock_mantle" + assert mantle["default_model_placeholder"].startswith("bedrock_mantle/") + + fields_by_key = {f["key"]: f for f in mantle["credential_fields"]} + + # Bearer-token auth is BYOK: optional and masked. + assert "api_key" in fields_by_key + assert fields_by_key["api_key"]["required"] is False + assert fields_by_key["api_key"]["field_type"] == "password" + + # AWS SigV4 fallback credentials. + assert fields_by_key["aws_access_key_id"]["field_type"] == "password" + assert fields_by_key["aws_secret_access_key"]["field_type"] == "password" + assert "aws_region_name" in fields_by_key + + # api_base override so admins can target a custom Mantle host without env access. + assert fields_by_key["api_base"]["field_type"] == "text" + + def test_google_ai_studio_provider_fields_expose_api_base(): """The Google AI Studio (gemini) credential form must let admins set a custom api_base so they can point at a Gemini-compatible gateway (e.g. a self-hosted @@ -819,3 +861,44 @@ def test_public_mcp_hub_returns_empty_when_whitelist_unset(): assert response.status_code == 200 assert response.json() == [] app.dependency_overrides.clear() + + +def test_public_mcp_hub_does_not_expose_upstream_url(): + """Regression: /public/mcp_hub is unauthenticated, so the gateway-internal + upstream url must never appear in its response even when the server has one.""" + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport + + app = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + client = TestClient(app) + + secret_url = "https://internal-only.example.com/mcp" + server = MCPServer( + server_id="listed", + name="listed", + server_name="listed", + url=secret_url, + transport=MCPTransport.http, + available_on_public_internet=True, + ) + + mock_manager = MagicMock() + mock_manager.get_public_mcp_servers.return_value = [server] + + with ( + patch("litellm.public_mcp_servers", ["listed"]), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + mock_manager, + ), + ): + response = client.get("/public/mcp_hub") + + assert response.status_code == 200 + data = response.json() + assert [item["server_id"] for item in data] == ["listed"] + assert all("url" not in item for item in data) + assert secret_url not in response.text + app.dependency_overrides.clear() diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 0b583129591..b9716c22cee 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3786,3 +3786,298 @@ async def test_ui_view_spend_logs_metadata_invalid_json_falls_back_to_empty_dict assert body["data"][0]["metadata"] == {} finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +class _FakeColdStorageLogger: + """Injectable cold storage logger that records the object key it was asked for.""" + + def __init__(self, payload): + self._payload = payload + self.requested_object_keys = [] + + async def get_proxy_server_request_from_cold_storage_with_object_key( + self, object_key + ): + self.requested_object_keys.append(object_key) + return self._payload + + +def _cold_storage_handler(payload): + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler + + logger = _FakeColdStorageLogger(payload) + return ColdStorageHandler(cold_storage_logger=logger), logger + + +@pytest.mark.parametrize( + "value, expected", + [ + (None, False), + ("", False), + (" ", False), + ("{}", False), + ("[]", False), + ("null", False), + ('{"a": 1}', True), + ({}, False), + ({"a": 1}, True), + ([], False), + ([1], True), + (5, True), + ], +) +def test_spend_log_field_has_content(value, expected): + assert spend_management_endpoints._spend_log_field_has_content(value) is expected + + +@pytest.mark.parametrize( + "metadata, expected", + [ + (None, None), + ("{}", None), + ("not-json", None), + ({"cold_storage_object_key": ""}, None), + ({"cold_storage_object_key": "k/req-1.json"}, "k/req-1.json"), + ('{"cold_storage_object_key": "k/req-2.json"}', "k/req-2.json"), + ], +) +def test_cold_storage_object_key_from_metadata(metadata, expected): + assert ( + spend_management_endpoints._cold_storage_object_key_from_metadata(metadata) + == expected + ) + + +@pytest.mark.asyncio +async def test_resolve_payload_prefers_pg_and_skips_cold_storage(): + handler, logger = _cold_storage_handler({"messages": "X", "response": "Y"}) + row = { + "messages": "{}", + "response": '{"choices": [{"message": {"content": "hi"}}]}', + "proxy_server_request": "{}", + "metadata": {"cold_storage_object_key": "k/req.json"}, + } + + resolved = await spend_management_endpoints._resolve_request_response_payload( + row, cold_storage_handler=handler + ) + + assert resolved.response == '{"choices": [{"message": {"content": "hi"}}]}' + assert logger.requested_object_keys == [] + + +@pytest.mark.asyncio +async def test_resolve_payload_fetches_from_cold_storage_when_pg_empty(): + cold_payload = { + "messages": [{"role": "user", "content": "what is 2+2"}], + "response": {"choices": [{"message": {"content": "4"}}]}, + "proxy_server_request": {"body": {"model": "gpt-4o-mini"}}, + } + handler, logger = _cold_storage_handler(cold_payload) + row = { + "messages": "{}", + "response": "{}", + "proxy_server_request": "{}", + "metadata": {"cold_storage_object_key": "llm-gateway/prod/req-42.json"}, + } + + resolved = await spend_management_endpoints._resolve_request_response_payload( + row, cold_storage_handler=handler + ) + + assert logger.requested_object_keys == ["llm-gateway/prod/req-42.json"] + assert resolved.messages == cold_payload["messages"] + assert resolved.response == cold_payload["response"] + assert resolved.proxy_server_request == cold_payload["proxy_server_request"] + + +@pytest.mark.asyncio +async def test_resolve_payload_metadata_as_json_string(): + cold_payload = {"messages": "in", "response": "out", "proxy_server_request": None} + handler, logger = _cold_storage_handler(cold_payload) + row = { + "messages": "{}", + "response": "{}", + "proxy_server_request": "{}", + "metadata": json.dumps({"cold_storage_object_key": "k/str-meta.json"}), + } + + resolved = await spend_management_endpoints._resolve_request_response_payload( + row, cold_storage_handler=handler + ) + + assert logger.requested_object_keys == ["k/str-meta.json"] + assert resolved.response == "out" + + +@pytest.mark.asyncio +async def test_resolve_payload_no_object_key_returns_empty_without_fetch(): + handler, logger = _cold_storage_handler({"messages": "should-not-be-used"}) + row = { + "messages": "{}", + "response": "{}", + "proxy_server_request": "{}", + "metadata": {}, + } + + resolved = await spend_management_endpoints._resolve_request_response_payload( + row, cold_storage_handler=handler + ) + + assert logger.requested_object_keys == [] + assert resolved == spend_management_endpoints.RequestResponsePayload( + "{}", "{}", "{}" + ) + + +@pytest.mark.asyncio +async def test_resolve_payload_cold_storage_miss_falls_back_to_pg_values(): + handler, logger = _cold_storage_handler(None) + row = { + "messages": "{}", + "response": "{}", + "proxy_server_request": "{}", + "metadata": {"cold_storage_object_key": "k/missing.json"}, + } + + resolved = await spend_management_endpoints._resolve_request_response_payload( + row, cold_storage_handler=handler + ) + + assert logger.requested_object_keys == ["k/missing.json"] + assert resolved == spend_management_endpoints.RequestResponsePayload( + "{}", "{}", "{}" + ) + + +@pytest.mark.asyncio +async def test_resolve_payload_cold_storage_exception_falls_back_to_pg_values(): + """A backend error during fetch degrades to PG values instead of bubbling a 500.""" + + class _RaisingLogger: + async def get_proxy_server_request_from_cold_storage_with_object_key( + self, object_key + ): + raise RuntimeError("cold storage backend unavailable") + + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler + + handler = ColdStorageHandler(cold_storage_logger=_RaisingLogger()) + row = { + "messages": "{}", + "response": "{}", + "proxy_server_request": "{}", + "metadata": {"cold_storage_object_key": "k/boom.json"}, + } + + resolved = await spend_management_endpoints._resolve_request_response_payload( + row, cold_storage_handler=handler + ) + + assert resolved == spend_management_endpoints.RequestResponsePayload( + "{}", "{}", "{}" + ) + + +@pytest.mark.asyncio +async def test_cold_storage_handler_uses_injected_logger(): + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler + + logger = _FakeColdStorageLogger({"messages": "in", "response": "out"}) + handler = ColdStorageHandler(cold_storage_logger=logger) + + result = await handler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key="k/req.json" + ) + + assert result == {"messages": "in", "response": "out"} + assert logger.requested_object_keys == ["k/req.json"] + + +@pytest.mark.asyncio +async def test_cold_storage_handler_returns_none_when_no_logger_configured(monkeypatch): + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler + + monkeypatch.setattr(litellm, "cold_storage_custom_logger", None, raising=False) + handler = ColdStorageHandler() + + result = await handler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key="k/req.json" + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_cold_storage_handler_resolves_configured_logger_from_registry(monkeypatch): + from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler + + logger = _FakeColdStorageLogger({"messages": "from-registry"}) + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2", raising=False) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_custom_logger_for_callback_name", + lambda name: logger if name == "s3_v2" else None, + ) + handler = ColdStorageHandler() + + result = await handler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key="k/req.json" + ) + + assert result == {"messages": "from-registry"} + assert logger.requested_object_keys == ["k/req.json"] + + +def test_ui_view_request_response_reads_from_cold_storage(client, monkeypatch): + """End-to-end: a placeholder row with a cold_storage_object_key is served from + cold storage through the detail endpoint.""" + from types import SimpleNamespace + + placeholder_row = { + "messages": "{}", + "response": "{}", + "proxy_server_request": "{}", + "metadata": {"cold_storage_object_key": "k/cold.json"}, + } + + async def _query_raw(_sql, *_args): + return [placeholder_row] + + fake_prisma = SimpleNamespace(db=SimpleNamespace(query_raw=_query_raw)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", fake_prisma) + + cold_logger = _FakeColdStorageLogger( + { + "messages": [{"role": "user", "content": "hi"}], + "response": {"choices": [{"message": {"content": "hello"}}]}, + "proxy_server_request": None, + } + ) + monkeypatch.setattr(litellm, "cold_storage_custom_logger", "s3_v2", raising=False) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_additional_logging_utils_from_custom_logger", + lambda: [], + ) + monkeypatch.setattr( + litellm.logging_callback_manager, + "get_active_custom_logger_for_callback_name", + lambda name: cold_logger if name == "s3_v2" else None, + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_1" + ) + try: + response = client.get( + "/spend/logs/ui/req-cold", + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + body = response.json() + assert body["messages"] == [{"role": "user", "content": "hi"}] + assert body["response"] == {"choices": [{"message": {"content": "hello"}}]} + assert cold_logger.requested_object_keys == ["k/cold.json"] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index ec262d75ab8..74f681bb97e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -96,6 +96,20 @@ class TestGetMetadataVariableName: request = self._make_request("/v1/embeddings") assert _get_metadata_variable_name(request) == "metadata" + def test_returns_litellm_metadata_for_bedrock_invoke(self): + # GH#30629: bedrock passthrough must use litellm_metadata + # to prevent key-level tags from leaking into provider body + request = self._make_request( + "/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke" + ) + assert _get_metadata_variable_name(request) == "litellm_metadata" + + def test_returns_litellm_metadata_for_bedrock_converse(self): + request = self._make_request( + "/bedrock/model/us.anthropic.claude-sonnet-4-6/converse" + ) + assert _get_metadata_variable_name(request) == "litellm_metadata" + def test_get_enforced_params_for_service_account_settings(): """ @@ -4256,6 +4270,99 @@ class TestApplyClientTagPolicyPreAuth: assert exc_info.value.current_cost == 0.50 assert exc_info.value.max_budget == 0.10 + @pytest.mark.asyncio + @pytest.mark.parametrize( + "route", + [ + "/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke", + "/v1/messages", + ], + ) + async def test_header_tags_visible_to_tag_max_budget_check_on_metadata_route( + self, route + ): + """Regression: on LITELLM_METADATA_ROUTES (bedrock, /v1/messages, ...), + common_checks pre-seeds ``litellm_metadata`` and writes key tags there + before ``_tag_max_budget_check`` reads from the same key. The auth wrapper + calls ``apply_client_tag_policy_pre_auth`` first, so without an earlier + pre-seed header tags land in ``metadata`` and the budget check (now + resolving to ``litellm_metadata``) silently ignores them. This test mirrors + the actual auth-time call order and verifies that an over-budget + header-supplied tag still trips ``_tag_max_budget_check``. + """ + from litellm.proxy._types import LiteLLM_BudgetTable, LiteLLM_TagTable + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.utils import ProxyLogging + + request_mock = _build_request_mock_with_headers( + {"x-litellm-tags": "tenant:acme"} + ) + data = {"model": "us.anthropic.claude-sonnet-4-6"} + valid_token = UserAPIKeyAuth( + token="test-token", + api_key="hashed-key", + metadata={}, + team_metadata={}, + ) + + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route( + request_data=data, + route=route, + ) + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth( + request=request_mock, + request_data=data, + user_api_key_dict=valid_token, + ) + + tag_object = LiteLLM_TagTable( + tag_name="tenant:acme", + spend=0.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10), + ) + + async def mock_get_current_spend( + counter_key, fallback_spend, max_budget=None, **kwargs + ): + if counter_key == "spend:tag:tenant:acme": + return 0.50 + return fallback_spend + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + MagicMock(), + ), + patch( + "litellm.proxy.proxy_server.get_current_spend", + mock_get_current_spend, + ), + patch( + "litellm.proxy.auth.auth_checks.get_tag_objects_batch", + new_callable=AsyncMock, + return_value={"tenant:acme": tag_object}, + ), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body=data, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=None), + valid_token=valid_token, + request=request_mock, + ) + assert exc_info.value.current_cost == 0.50 + assert exc_info.value.max_budget == 0.10 + + assert "metadata" not in data + assert data["litellm_metadata"]["tags"] == ["tenant:acme"] + class TestApplyKeyTagsPreAuth: def test_merges_key_tags_into_metadata(self): diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 8b10539b188..a203fcc7ec0 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1348,6 +1348,7 @@ async def test_apply_search_filter_scopes_byok_to_caller_teams(): non_admin = MagicMock(spec=UserAPIKeyAuth) non_admin.user_role = LitellmUserRoles.INTERNAL_USER non_admin.user_id = "user-mine" + non_admin.team_id = None filtered, total_count = await _apply_search_filter_to_models( all_models=[caller_team_byok, other_team_byok, public_model], @@ -1381,6 +1382,7 @@ async def test_apply_search_filter_scopes_byok_to_caller_teams(): admin = MagicMock(spec=UserAPIKeyAuth) admin.user_role = LitellmUserRoles.PROXY_ADMIN admin.user_id = "admin-1" + admin.team_id = None filtered_admin, _ = await _apply_search_filter_to_models( all_models=[caller_team_byok, other_team_byok, public_model], @@ -8362,3 +8364,89 @@ def test_preserve_redacted_plugin_keys_sets_new_and_drops_orphan_placeholder(): [{"name": "p2", "url": "https://p2", "plugin_key": "***"}], existing ) assert "plugin_key" not in new_plugin[0] + + +def _config_field_info_client(monkeypatch, user_role): + import types + from unittest.mock import AsyncMock, MagicMock + + from fastapi.testclient import TestClient + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import app + + db_record = types.SimpleNamespace( + param_value={ + "master_key": "sk-super-secret-master", + "database_url": "postgresql://user:p4ssw0rd@db:5432/litellm", + "pass_through_endpoints": [ + { + "path": "/upstream", + "target": "https://upstream.example.com", + "headers": {"Authorization": "Bearer sk-upstream-secret"}, + } + ], + "max_parallel_requests": 100, + } + ) + mock_config_table = MagicMock() + mock_config_table.find_first = AsyncMock(return_value=db_record) + mock_prisma = MagicMock() + mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="u", user_role=user_role + ) + return TestClient(app) + + +def test_config_field_info_redacts_secrets_for_view_only_admin(monkeypatch): + """/config/field/info gates on _user_has_admin_view, which also grants + PROXY_ADMIN_VIEW_ONLY. A view-only admin reading master_key/database_url verbatim is + effectively a full admin. Secret-bearing fields must come back REDACTED for anyone who + is not a FULL PROXY_ADMIN, while non-secret fields stay readable.""" + from litellm.proxy._types import LitellmUserRoles + + client = _config_field_info_client( + monkeypatch, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + try: + for secret_field in ("master_key", "database_url", "pass_through_endpoints"): + resp = client.get("/config/field/info", params={"field_name": secret_field}) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["field_value"] == "REDACTED" + assert "secret" not in str(body["field_value"]) + assert "p4ssw0rd" not in str(body["field_value"]) + + resp = client.get( + "/config/field/info", params={"field_name": "max_parallel_requests"} + ) + assert resp.status_code == 200, resp.text + assert resp.json()["field_value"] == 100 + finally: + app.dependency_overrides.clear() + + +def test_config_field_info_returns_raw_secrets_for_full_admin(monkeypatch): + """the redaction must not over-apply. A FULL PROXY_ADMIN still + needs the real master_key value to populate the admin edit form.""" + from litellm.proxy._types import LitellmUserRoles + + client = _config_field_info_client(monkeypatch, LitellmUserRoles.PROXY_ADMIN) + try: + resp = client.get("/config/field/info", params={"field_name": "master_key"}) + assert resp.status_code == 200, resp.text + assert resp.json()["field_value"] == "sk-super-secret-master" + + resp = client.get( + "/config/field/info", params={"field_name": "pass_through_endpoints"} + ) + assert resp.status_code == 200, resp.text + assert ( + resp.json()["field_value"][0]["headers"]["Authorization"] + == "Bearer sk-upstream-secret" + ) + finally: + app.dependency_overrides.clear() diff --git a/tests/test_litellm/proxy/test_read_model_list.py b/tests/test_litellm/proxy/test_read_model_list.py new file mode 100644 index 00000000000..7703d57e21d --- /dev/null +++ b/tests/test_litellm/proxy/test_read_model_list.py @@ -0,0 +1,32 @@ +"""Tests for litellm.proxy.read_model_list (Rust AI gateway config bridge).""" + +from litellm.proxy.read_model_list import read_model_list + + +def test_read_model_list_resolves_os_environ(monkeypatch, tmp_path): + """`os.environ/` markers in the model_list are resolved via ProxyConfig.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-resolved-123") + config = tmp_path / "config.yaml" + config.write_text( + "model_list:\n" + " - model_name: gpt-realtime\n" + " litellm_params:\n" + " model: openai/gpt-realtime\n" + " api_key: os.environ/OPENAI_API_KEY\n" + ) + + model_list = read_model_list(str(config)) + + assert len(model_list) == 1 + params = model_list[0]["litellm_params"] + assert model_list[0]["model_name"] == "gpt-realtime" + assert params["model"] == "openai/gpt-realtime" + assert params["api_key"] == "sk-resolved-123" + + +def test_read_model_list_missing_key_returns_empty(tmp_path): + """A config without a model_list yields an empty list, not an error.""" + config = tmp_path / "config.yaml" + config.write_text("general_settings: {}\n") + + assert read_model_list(str(config)) == [] diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py index d517c1c346f..87063bdf00b 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py @@ -105,6 +105,33 @@ def test_jsonify_team_object_converts_members_to_json_string( } +def test_jsonify_team_object_converts_budget_limits_to_json_string( + prisma_client: PrismaClient, +) -> None: + data = { + "team_id": "t1", + "budget_limits": [ + { + "budget_duration": "1d", + "max_budget": 10.0, + "reset_at": "2026-01-01T00:00:00Z", + }, + { + "budget_duration": "7d", + "max_budget": 50.0, + "reset_at": "2026-01-07T00:00:00Z", + }, + ], + "models": ["gpt-4"], + } + result = prisma_client.jsonify_team_object(data) + assert result == { + "team_id": "t1", + "budget_limits": json.dumps(data["budget_limits"]), + "models": ["gpt-4"], + } + + def test_jsonify_team_object_error_on_non_dict(prisma_client: PrismaClient) -> None: with pytest.raises(AttributeError): prisma_client.jsonify_team_object(None) # type: ignore[arg-type] diff --git a/tests/test_litellm/sandbox/test_opensandbox_sandbox.py b/tests/test_litellm/sandbox/test_opensandbox_sandbox.py new file mode 100644 index 00000000000..0d7bcbe1e53 --- /dev/null +++ b/tests/test_litellm/sandbox/test_opensandbox_sandbox.py @@ -0,0 +1,647 @@ +import json + +import httpx +import pytest + +import litellm +from litellm.llms.base_llm.sandbox.transformation import ContainerHandle +from litellm.llms.opensandbox.sandbox.transformation import ( + MAX_OUTPUT_BYTES, + OPEN_SANDBOX_DEFAULT_TEMPLATE, + OpenSandboxSandboxConfig, +) +from litellm.utils import ProviderConfigManager + +TEST_API_BASE = "https://sandbox.test/v1" + + +def http_status_error(status_code, url="http://test"): + return httpx.HTTPStatusError( + f"status {status_code}", + request=httpx.Request("GET", url), + response=httpx.Response(status_code), + ) + + +def sse(data): + return f"data: {json.dumps(data)}" + + +class FakeResponse: + def __init__(self, *, json_data=None, lines=None, status_code=200): + self._json = json_data + self._lines = lines or [] + self.status_code = status_code + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + raise http_status_error(self.status_code) + + async def aiter_lines(self): + for line in self._lines: + yield line + + +class FakeHTTPClient: + def __init__( + self, + *, + create_json=None, + sandbox_states=None, + endpoint_json=None, + endpoint_responses=None, + execute_lines=None, + delete_status=204, + execute_raises=None, + ): + self.create_json = create_json or { + "id": "osb_123", + "status": {"state": "Running"}, + "createdAt": "2026-01-01T00:00:00Z", + "entrypoint": ["/opt/code-interpreter/code-interpreter.sh"], + } + self.sandbox_states = list( + sandbox_states + or [ + { + "id": "osb_123", + "status": {"state": "Running"}, + "createdAt": "2026-01-01T00:00:00Z", + "entrypoint": ["/opt/code-interpreter/code-interpreter.sh"], + } + ] + ) + self.endpoint_json = endpoint_json or { + "endpoint": "execd.local:44772", + "headers": {"X-EXECD-ACCESS-TOKEN": "execd-token"}, + } + self.endpoint_responses = ( + list(endpoint_responses) if endpoint_responses is not None else None + ) + self.execute_lines = execute_lines or [] + self.delete_status = delete_status + self.execute_raises = execute_raises + self.calls = [] + + async def post(self, url, headers=None, json=None, stream=False, **kwargs): + self.calls.append(("POST", url, headers, json, {"stream": stream})) + if url.endswith("/sandboxes"): + return FakeResponse(json_data=self.create_json) + if url.endswith("/code"): + if self.execute_raises is not None: + raise self.execute_raises + return FakeResponse(lines=self.execute_lines) + raise AssertionError(f"unexpected POST {url}") + + async def get(self, url, headers=None, params=None, **kwargs): + self.calls.append(("GET", url, headers, None, params)) + if "/endpoints/44772" in url: + if self.endpoint_responses is not None and self.endpoint_responses: + response = self.endpoint_responses.pop(0) + if isinstance(response, Exception): + raise response + if isinstance(response, FakeResponse): + return response + return FakeResponse(json_data=response) + return FakeResponse(json_data=self.endpoint_json) + if "/sandboxes/" in url: + state = self.sandbox_states.pop(0) + return FakeResponse(json_data=state) + raise AssertionError(f"unexpected GET {url}") + + async def delete(self, url, headers=None, **kwargs): + self.calls.append(("DELETE", url, headers, None, None)) + if not (200 <= self.delete_status < 300): + raise http_status_error(self.delete_status, url) + return FakeResponse(status_code=self.delete_status) + + +def test_parse_sse_lines_maps_output_result_count_and_error(): + lines = [ + sse({"type": "stdout", "text": "hello\n"}), + sse({"type": "stderr", "text": "warn\n"}), + sse({"type": "result", "results": {"text/plain": "4"}}), + sse({"type": "execution_count", "execution_count": 7}), + sse( + { + "type": "error", + "error": { + "ename": "ValueError", + "evalue": "bad", + "traceback": ["Traceback"], + }, + } + ), + ] + + result = OpenSandboxSandboxConfig._parse_lines(lines) + + assert result.stdout == "hello\n" + assert result.stderr == "warn\n" + assert result.results == [{"text/plain": "4"}] + assert result.execution_count == 7 + assert result.error == { + "name": "ValueError", + "value": "bad", + "traceback": ["Traceback"], + } + + +def test_parse_sse_lines_skips_non_json_and_control_lines(): + lines = [ + "event: message", + "not-json", + "", + sse({"type": "stdout", "text": "ok\n"}), + ] + + result = OpenSandboxSandboxConfig._parse_lines(lines) + + assert result.stdout == "ok\n" + assert result.error is None + + +def test_parse_sse_lines_maps_fallback_shapes(): + lines = [ + "data:", + sse(["not-a-dict"]), + sse({"code": "BadRequest", "message": "nope"}), + sse({"type": "result", "text/plain": "4"}), + sse({"type": "error", "name": "RuntimeError", "text": "boom"}), + sse({"type": "execution_count", "execution_count": "8"}), + ] + + result = OpenSandboxSandboxConfig._parse_lines(lines) + + assert result.results == [{"text/plain": "4"}] + assert result.execution_count == 8 + assert result.error == { + "name": "BadRequest", + "value": "nope", + "traceback": [], + } + fallback_error = OpenSandboxSandboxConfig._parse_lines( + [sse({"type": "error", "name": "RuntimeError", "text": "boom"})] + ) + assert fallback_error.error == { + "name": "RuntimeError", + "value": "boom", + "traceback": [], + } + empty_string_error = OpenSandboxSandboxConfig._parse_lines( + [ + sse( + { + "type": "error", + "error": { + "ename": "", + "name": "FallbackName", + "evalue": "", + "value": "fallback value", + "traceback": [], + }, + } + ) + ] + ) + assert empty_string_error.error == { + "name": "", + "value": "", + "traceback": [], + } + + +def test_static_helpers_cover_defaults_and_fallbacks(monkeypatch): + def fake_secret(key): + if key == "OPEN_SANDBOX_API_KEY": + return "env-key" + if key == "OPEN_SANDBOX_API_BASE": + return TEST_API_BASE + return None + + monkeypatch.setattr( + "litellm.llms.opensandbox.sandbox.transformation.get_secret_str", + fake_secret, + ) + config = OpenSandboxSandboxConfig() + handle = ContainerHandle(id="osb", provider="opensandbox", domain="http://x/v1") + + assert config.validate_environment() == "env-key" + assert config.validate_environment(api_key="") == "" + assert config._api_key(api_key=None, handle=handle) == "env-key" + + handle._hidden_params = {"api_key": "stored-key"} + assert config._api_key(api_key=None, handle=handle) == "stored-key" + assert config._http(None) is not None + + body = config._create_body( + template=None, + timeout=None, + allow_internet_access=False, + metadata=None, + env_vars=None, + resource_limits=None, + resource_requests=None, + entrypoint=None, + network_policy={"egress": [{"domain": "example.com"}]}, + secure_access=True, + ) + assert body["networkPolicy"] == {"egress": [{"domain": "example.com"}]} + assert body["secureAccess"] is True + + other_body = config._create_body( + template=None, + timeout=None, + allow_internet_access=False, + metadata=None, + env_vars=None, + resource_limits=None, + resource_requests=None, + entrypoint=None, + network_policy=None, + secure_access=False, + ) + assert body["resourceLimits"] is not other_body["resourceLimits"] + + assert config._sandbox_state(None) is None + assert config._sandbox_state({"status": "Running"}) is None + assert config._as_str_dict(None) == {} + assert config._endpoint_base_url("http://execd.local", "https://api/v1") == ( + "http://execd.local" + ) + assert config._api_base(None) == TEST_API_BASE + assert config._api_base("https://direct.test/v1/") == "https://direct.test/v1" + assert config._as_int("9") == 9 + assert config._as_int("nope") is None + assert config._as_int(None) is None + assert isinstance( + ProviderConfigManager.get_provider_sandbox_config("opensandbox"), + OpenSandboxSandboxConfig, + ) + + +def test_api_base_requires_kwarg_or_env(monkeypatch): + monkeypatch.setattr( + "litellm.llms.opensandbox.sandbox.transformation.get_secret_str", + lambda key: None, + ) + + with pytest.raises(ValueError, match="api_base is required"): + OpenSandboxSandboxConfig._api_base(None) + + +@pytest.mark.asyncio +async def test_create_posts_default_body_and_omits_empty_api_key(): + client = FakeHTTPClient() + + handle = await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", api_base=TEST_API_BASE, client=client + ) + + method, url, headers, body, _ = client.calls[0] + assert method == "POST" + assert url == f"{TEST_API_BASE}/sandboxes" + assert "OPEN-SANDBOX-API-KEY" not in headers + assert body["image"] == {"uri": OPEN_SANDBOX_DEFAULT_TEMPLATE} + assert body["entrypoint"] == ["/opt/code-interpreter/code-interpreter.sh"] + assert body["timeout"] == 300 + assert body["resourceLimits"] == {"cpu": "1", "memory": "2Gi"} + assert body["networkPolicy"] == {"defaultAction": "deny", "egress": []} + assert handle.id == "osb_123" + assert handle._hidden_params["execd_endpoint"] == "execd.local:44772" + + +@pytest.mark.asyncio +async def test_create_can_opt_into_internet_access(): + client = FakeHTTPClient() + + await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", + api_base=TEST_API_BASE, + allow_internet_access=True, + client=client, + ) + + _, _, _, body, _ = client.calls[0] + assert "networkPolicy" not in body + + +@pytest.mark.asyncio +async def test_create_custom_options_poll_and_endpoint_resolution(): + client = FakeHTTPClient( + create_json={ + "id": "osb_pending", + "status": {"state": "Pending"}, + "createdAt": "2026-01-01T00:00:00Z", + "entrypoint": ["/bin/sh"], + }, + sandbox_states=[ + { + "id": "osb_pending", + "status": {"state": "Running"}, + "createdAt": "2026-01-01T00:00:00Z", + "entrypoint": ["/bin/sh"], + } + ], + ) + + handle = await OpenSandboxSandboxConfig().acreate_sandbox( + template="custom/image:latest", + timeout=600, + allow_internet_access=False, + api_key="osb-key", + api_base="https://sandbox.example/v1", + metadata={"suite": "unit"}, + env_vars={"PYTHONUNBUFFERED": "1"}, + resource_limits={"cpu": "500m", "memory": "512Mi"}, + resource_requests={"cpu": "250m", "memory": "256Mi"}, + entrypoint=["/bin/sh", "-lc", "sleep 3600"], + use_server_proxy=True, + client=client, + ) + + _, create_url, create_headers, body, _ = client.calls[0] + _, poll_url, poll_headers, _, _ = client.calls[1] + _, endpoint_url, endpoint_headers, _, endpoint_params = client.calls[2] + + assert create_url == "https://sandbox.example/v1/sandboxes" + assert create_headers["OPEN-SANDBOX-API-KEY"] == "osb-key" + assert body["image"] == {"uri": "custom/image:latest"} + assert body["entrypoint"] == ["/bin/sh", "-lc", "sleep 3600"] + assert body["metadata"] == {"suite": "unit"} + assert body["env"] == {"PYTHONUNBUFFERED": "1"} + assert body["resourceLimits"] == {"cpu": "500m", "memory": "512Mi"} + assert body["resourceRequests"] == {"cpu": "250m", "memory": "256Mi"} + assert body["networkPolicy"] == {"defaultAction": "deny", "egress": []} + assert poll_url == "https://sandbox.example/v1/sandboxes/osb_pending" + assert poll_headers["OPEN-SANDBOX-API-KEY"] == "osb-key" + assert endpoint_url.endswith("/sandboxes/osb_pending/endpoints/44772") + assert endpoint_headers["OPEN-SANDBOX-API-KEY"] == "osb-key" + assert endpoint_params == {"use_server_proxy": True} + assert handle.id == "osb_pending" + + +@pytest.mark.asyncio +async def test_create_waits_across_pending_state(monkeypatch): + client = FakeHTTPClient( + create_json={ + "id": "osb_pending", + "status": {"state": "Pending"}, + "createdAt": "2026-01-01T00:00:00Z", + }, + sandbox_states=[ + {"id": "osb_pending", "status": {"state": "Pending"}}, + {"id": "osb_pending", "status": {"state": "Running"}}, + ], + ) + sleeps = [] + + async def fake_sleep(interval): + sleeps.append(interval) + + monkeypatch.setattr( + "litellm.llms.opensandbox.sandbox.transformation.asyncio.sleep", fake_sleep + ) + + handle = await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", + api_base=TEST_API_BASE, + ready_timeout=1, + poll_interval=0.01, + client=client, + ) + + assert handle.id == "osb_pending" + assert sleeps == [0.01] + + +@pytest.mark.asyncio +async def test_create_raises_for_terminal_state(): + client = FakeHTTPClient( + create_json={"id": "osb_failed", "status": {"state": "Pending"}}, + sandbox_states=[ + {"id": "osb_failed", "status": {"state": "Failed"}}, + ], + ) + + with pytest.raises(ValueError, match="entered Failed"): + await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", api_base=TEST_API_BASE, client=client + ) + + +@pytest.mark.asyncio +async def test_create_times_out_waiting_for_running(): + client = FakeHTTPClient( + create_json={"id": "osb_slow", "status": {"state": "Pending"}}, + sandbox_states=[ + {"id": "osb_slow", "status": {"state": "Pending"}}, + ], + ) + + with pytest.raises(TimeoutError, match="was not Running"): + await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", + api_base=TEST_API_BASE, + ready_timeout=0, + poll_interval=0, + client=client, + ) + + +@pytest.mark.asyncio +async def test_create_waits_for_endpoint_resolution(monkeypatch): + client = FakeHTTPClient( + endpoint_responses=[ + http_status_error(404, f"{TEST_API_BASE}/sandboxes/osb_123"), + { + "endpoint": "execd.local:44772", + "headers": {"X-EXECD-ACCESS-TOKEN": "execd-token"}, + }, + ], + ) + sleeps = [] + + async def fake_sleep(interval): + sleeps.append(interval) + + monkeypatch.setattr( + "litellm.llms.opensandbox.sandbox.transformation.asyncio.sleep", fake_sleep + ) + + handle = await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", + api_base=TEST_API_BASE, + ready_timeout=1, + poll_interval=0.01, + client=client, + ) + + endpoint_calls = [call for call in client.calls if "/endpoints/44772" in call[1]] + assert handle._hidden_params["execd_endpoint"] == "execd.local:44772" + assert len(endpoint_calls) == 2 + assert sleeps == [0.01] + + +@pytest.mark.asyncio +async def test_create_raises_when_endpoint_is_missing(): + client = FakeHTTPClient(endpoint_json={"headers": {"X": "y"}}) + + with pytest.raises(TimeoutError, match="execd endpoint.*not ready"): + await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", api_base=TEST_API_BASE, ready_timeout=0, client=client + ) + + +@pytest.mark.asyncio +async def test_create_reraises_non_404_endpoint_error(): + client = FakeHTTPClient(endpoint_responses=[http_status_error(500)]) + + with pytest.raises(httpx.HTTPStatusError): + await OpenSandboxSandboxConfig().acreate_sandbox( + api_key="", api_base=TEST_API_BASE, client=client + ) + + +@pytest.mark.asyncio +async def test_run_code_resolves_bare_id_and_posts_sse_request(): + client = FakeHTTPClient( + execute_lines=[ + sse({"type": "stdout", "text": "42\n"}), + ] + ) + + result = await OpenSandboxSandboxConfig().arun_code( + container="osb_bare", + code="print(6*7)", + language="python", + api_key="", + api_base="http://sandbox.local/v1", + client=client, + ) + + endpoint_call = client.calls[0] + run_call = client.calls[1] + assert endpoint_call[0] == "GET" + assert ( + endpoint_call[1] == "http://sandbox.local/v1/sandboxes/osb_bare/endpoints/44772" + ) + assert run_call[0] == "POST" + assert run_call[1] == "http://execd.local:44772/code" + assert run_call[2]["X-EXECD-ACCESS-TOKEN"] == "execd-token" + assert run_call[3] == { + "code": "print(6*7)", + "context": {"language": "python"}, + } + assert run_call[4] == {"stream": True} + assert result.stdout == "42\n" + + +@pytest.mark.asyncio +async def test_run_code_uses_https_for_scheme_less_endpoint_when_api_base_is_https(): + client = FakeHTTPClient() + handle = ContainerHandle( + id="osb_https", provider="opensandbox", domain="https://sandbox.example/v1" + ) + handle._hidden_params = { + "execd_endpoint": "execd.example/route/44772", + "execd_headers": {}, + } + + await OpenSandboxSandboxConfig().arun_code( + container=handle, code="print(1)", client=client + ) + + assert client.calls[0][1] == "https://execd.example/route/44772/code" + + +@pytest.mark.asyncio +async def test_run_code_aborts_on_output_over_cap(): + client = FakeHTTPClient(execute_lines=["x" * (MAX_OUTPUT_BYTES + 1)]) + handle = ContainerHandle(id="osb_big", provider="opensandbox", domain="http://x/v1") + handle._hidden_params = {"execd_endpoint": "execd.local:44772", "execd_headers": {}} + + with pytest.raises(ValueError, match="exceeded"): + await OpenSandboxSandboxConfig().arun_code( + container=handle, code="print('x')", client=client + ) + + +@pytest.mark.asyncio +async def test_delete_returns_false_on_404(): + client = FakeHTTPClient(delete_status=404) + + ok = await OpenSandboxSandboxConfig().adelete_sandbox( + container="osb_gone", + api_key="", + api_base="http://sandbox.local/v1", + client=client, + ) + + assert ok is False + + +@pytest.mark.asyncio +async def test_delete_reraises_non_404_http_error(): + client = FakeHTTPClient(delete_status=500) + + with pytest.raises(httpx.HTTPStatusError): + await OpenSandboxSandboxConfig().adelete_sandbox( + container="osb_err", + api_key="", + api_base="http://sandbox.local/v1", + client=client, + ) + + +@pytest.mark.asyncio +async def test_public_lifecycle_create_run_delete(): + client = FakeHTTPClient( + execute_lines=[ + sse({"type": "stdout", "text": "42\n"}), + ] + ) + + container = await litellm.acreate_sandbox( + provider="opensandbox", api_key="", api_base=TEST_API_BASE, client=client + ) + result = await litellm.arun_code( + provider="opensandbox", + container=container, + code="print(6*7)", + api_key="", + client=client, + ) + ok = await litellm.adelete_sandbox( + provider="opensandbox", + container=container, + api_key="", + client=client, + ) + + assert container.id == "osb_123" + assert result.stdout == "42\n" + assert ok is True + + +@pytest.mark.asyncio +async def test_code_interpreter_tool_deletes_even_when_run_raises(): + client = FakeHTTPClient(execute_raises=RuntimeError("boom")) + + with pytest.raises(RuntimeError, match="boom"): + await litellm.acode_interpreter_tool( + provider="opensandbox", + code="1/0", + api_key="", + api_base=TEST_API_BASE, + client=client, + ) + + assert [call[0] for call in client.calls] == ["POST", "GET", "POST", "DELETE"] + assert client.calls[0][1].endswith("/sandboxes") + assert client.calls[1][1].endswith("/endpoints/44772") + assert client.calls[2][1].endswith("/code") + assert client.calls[3][1].endswith("/sandboxes/osb_123") diff --git a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py new file mode 100644 index 00000000000..9ca4515239a --- /dev/null +++ b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py @@ -0,0 +1,89 @@ +""" +Regression tests for the Cloudflare Workers AI text-generation catalog in the +model-cost map. + +The Cloudflare list was badly stale (only 4 ancient entries). These tests pin +the newly added current Workers AI models (sourced from Cloudflare's live +``/ai/models/search?task=Text Generation`` catalog) and guard against the root +``model_prices_and_context_window.json`` and the bundled +``litellm/model_prices_and_context_window_backup.json`` drifting out of sync for +the ``cloudflare/`` namespace. +""" + +import json +import os + +import pytest + +import litellm + +ROOT_MAP = os.path.join( + os.path.dirname(os.path.dirname(litellm.__file__)), + "model_prices_and_context_window.json", +) +BACKUP_MAP = os.path.join( + os.path.dirname(litellm.__file__), + "model_prices_and_context_window_backup.json", +) + + +@pytest.fixture(autouse=True) +def _use_local_model_cost_map(monkeypatch): + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + try: + yield + finally: + litellm.model_cost = original_model_cost + + +def _load(path: str) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _cloudflare_keys(data: dict) -> set: + return {k for k in data if k.startswith("cloudflare/")} + + +def test_glm_5_2_entry_is_present_and_well_formed(): + entry = litellm.model_cost["cloudflare/@cf/zai-org/glm-5.2"] + assert entry["litellm_provider"] == "cloudflare" + assert entry["mode"] == "chat" + assert entry["supports_function_calling"] is True + assert entry["input_cost_per_token"] > 0 + assert entry["output_cost_per_token"] > 0 + + +def test_vision_model_is_flagged_supports_vision(): + entry = litellm.model_cost["cloudflare/@cf/meta/llama-3.2-11b-vision-instruct"] + assert entry["litellm_provider"] == "cloudflare" + assert entry.get("supports_vision") is True + + +def test_additional_current_models_are_present(): + for key in ( + "cloudflare/@cf/openai/gpt-oss-120b", + "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast", + ): + entry = litellm.model_cost[key] + assert entry["litellm_provider"] == "cloudflare" + assert entry["mode"] == "chat" + assert entry["supports_function_calling"] is True + assert entry["input_cost_per_token"] > 0 + assert entry["output_cost_per_token"] > 0 + + +def test_root_and_backup_have_identical_cloudflare_keys(): + if not os.path.exists(ROOT_MAP): + pytest.skip("root cost map only ships in source checkouts") + assert _cloudflare_keys(_load(ROOT_MAP)) == _cloudflare_keys(_load(BACKUP_MAP)) + + +def test_root_and_backup_cloudflare_entries_are_byte_for_byte_equal(): + if not os.path.exists(ROOT_MAP): + pytest.skip("root cost map only ships in source checkouts") + root = {k: v for k, v in _load(ROOT_MAP).items() if k.startswith("cloudflare/")} + backup = {k: v for k, v in _load(BACKUP_MAP).items() if k.startswith("cloudflare/")} + assert root == backup diff --git a/tests/test_litellm/test_completion_timeout_resolution.py b/tests/test_litellm/test_completion_timeout_resolution.py index a76cc6f7de8..7eb79e90e60 100644 --- a/tests/test_litellm/test_completion_timeout_resolution.py +++ b/tests/test_litellm/test_completion_timeout_resolution.py @@ -63,8 +63,9 @@ def test_global_timeout_from_litellm_settings(): ) -def test_global_timeout_package_default_coerced_to_600_for_completion(): - """Package default 6000s → 600s for completion-only path.""" +def test_explicit_global_timeout_6000_is_preserved(): + """The caller passes the explicitly-configured value (or None); an explicit + 6000 must be honored, not silently coerced to 600.""" assert ( CompletionTimeout.resolve( None, @@ -73,7 +74,7 @@ def test_global_timeout_package_default_coerced_to_600_for_completion(): global_timeout=6000.0, supports_httpx_timeout=supports_httpx_timeout, ) - == 600.0 + == 6000.0 ) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index a67c5b41f36..db6f945ff78 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -505,6 +505,74 @@ def test_realtime_logging_object_allows_null_transcript_in_conversation_item_add assert logging_result.results[0]["item"]["content"][0]["transcript"] is None +def test_realtime_logging_object_does_not_validate_unknown_event_types(): + """ + A realtime session emits events outside the OpenAIRealtimeEvents union (e.g. + rate_limits.updated, response.function_call_arguments.delta). Building the + logging object must not revalidate every event against the union; doing so + produces thousands of Pydantic ValidationErrors per session, blocks the event + loop, and the raised error discards the session's usage. The events must + survive verbatim, the combined usage must be preserved, and serialization + must stay clean. + """ + import warnings + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "event_id": "ev0", "session": {"id": "s"}}, + ] + for i in range(50): + results += [ + { + "type": "rate_limits.updated", + "event_id": f"rl{i}", + "rate_limits": [{"name": "requests", "limit": 1000, "remaining": 900}], + }, + { + "type": "response.function_call_arguments.delta", + "event_id": f"fc{i}", + "delta": "{}", + }, + { + "type": "response.done", + "event_id": f"rd{i}", + "response": { + "usage": { + "input_tokens": 4, + "output_tokens": 6, + "total_tokens": 10, + } + }, + }, + ] + + usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results + ) + # On unfixed code this raises pydantic ValidationError instead of returning. + logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object( + usage=usage, + results=results, + ) + + assert logging_result.usage.total_tokens == 500 + assert len(logging_result.results) == len(results) + unknown_types = { + r["type"] + for r in logging_result.results + if r["type"] + in ("rate_limits.updated", "response.function_call_arguments.delta") + } + assert unknown_types == { + "rate_limits.updated", + "response.function_call_arguments.delta", + } + + with warnings.catch_warnings(): + warnings.simplefilter("error") + dumped = logging_result.model_dump() + assert len(dumped["results"]) == len(results) + + def test_realtime_transcription_duration_cost(monkeypatch): """ gpt-realtime-whisper transcription sessions are billed by input audio duration diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py index 6644b1389cf..c371f7442be 100644 --- a/tests/test_litellm/test_gpt_image_cost_calculator.py +++ b/tests/test_litellm/test_gpt_image_cost_calculator.py @@ -381,5 +381,93 @@ class TestCompletionCostIntegration: assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}" +class TestGPTImage2OutputImageTokensNoBreakdown: + """ + Regression test: the OpenAI Images endpoints (/v1/images/generations and + /v1/images/edits) return usage with NO output token breakdown — litellm's + ImageUsage has no ``output_tokens_details`` field. Before the fix, the + generated-image OUTPUT tokens were priced at the text rate + (``output_cost_per_token`` = $10/1M for gpt-image-2) instead of the image rate + (``output_cost_per_image_token`` = $30/1M), a ~3x undercount on the dominant + cost component. + """ + + def test_gpt_image_2_output_priced_as_image_when_no_breakdown(self): + from litellm.llms.openai.image_generation.cost_calculator import ( + cost_calculator, + ) + + # Mirrors a real gpt-image-2 /v1/images/edits response: input breakdown is + # present, but there is no usable output token breakdown. + usage = ImageUsage( + input_tokens=3987, + output_tokens=5488, + total_tokens=9475, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=943, + image_tokens=3044, + ), + ) + + image_response = ImageResponse( + created=1234567890, + data=[ImageObject(b64_json="test")], + ) + image_response.usage = usage + image_response._hidden_params = {"custom_llm_provider": "openai"} + + cost = cost_calculator( + model="gpt-image-2", + image_response=image_response, + custom_llm_provider="openai", + ) + + # gpt-image-2 pricing: + # text input: 943 * $5/1M = 0.004715 + # image input: 3044 * $8/1M = 0.024352 + # image output: 5488 * $30/1M = 0.164640 (NOT text output $10/1M = 0.054880) + expected_cost = 943 * 5e-6 + 3044 * 8e-6 + 5488 * 3e-5 + assert abs(cost - expected_cost) < 1e-6, ( + f"Expected {expected_cost}, got {cost}. Generated image output tokens " + f"are likely being priced at the text output_cost_per_token rate." + ) + + def test_gpt_image_2_chat_usage_without_breakdown_is_costed_not_zero(self): + """A chat ``Usage`` with ``completion_tokens_details=None`` must still be + costed via ``generic_cost_per_token`` (output at the text rate) rather than + erroring or silently returning 0.0.""" + from litellm.llms.openai.image_generation.cost_calculator import ( + cost_calculator, + ) + + usage = Usage( + prompt_tokens=600, + completion_tokens=5000, + total_tokens=5600, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=100, + image_tokens=500, + ), + ) + + image_response = ImageResponse( + created=1234567890, + data=[ImageObject(b64_json="test")], + ) + image_response.usage = usage + image_response._hidden_params = {"custom_llm_provider": "openai"} + + cost = cost_calculator( + model="gpt-image-2", + image_response=image_response, + custom_llm_provider="openai", + ) + + # No output breakdown -> output priced at the text rate (output_cost_per_token): + # text in 100*$5/1M + image in 500*$8/1M + output 5000*$10/1M + expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 1e-5 + assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}" + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c2aa4a095d4..3e2150848a7 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -4901,3 +4901,107 @@ def test_is_deployment_blocked_static_helper_reflects_blocked_flag(): ) is True ) + + +class TestRouterRequestTimeoutPropagation: + """litellm_settings.request_timeout must act as an independent per-attempt timeout. + + Regression for LIT-2369: request_timeout was shadowed by router_settings.timeout, + so Bedrock (and other provider) calls fell back to the hardcoded 600s httpx + default instead of the configured value. + """ + + def _make_router(self, timeout=None, stream_timeout=None): + return litellm.Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "sk-test", + }, + } + ], + timeout=timeout, + stream_timeout=stream_timeout, + ) + + @pytest.fixture + def explicit_request_timeout(self): + original_value = litellm.request_timeout + original_flag = litellm.request_timeout_explicitly_set + litellm.request_timeout = 300 + litellm.request_timeout_explicitly_set = True + try: + yield 300 + finally: + litellm.request_timeout = original_value + litellm.request_timeout_explicitly_set = original_flag + + def test_request_timeout_stored_independently_when_both_set( + self, explicit_request_timeout + ): + router = self._make_router(timeout=330) + assert router.timeout == 330 + assert router.request_timeout == 300 + + def test_request_timeout_none_when_not_explicitly_configured(self): + original_value = litellm.request_timeout + original_flag = litellm.request_timeout_explicitly_set + litellm.request_timeout = litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS + litellm.request_timeout_explicitly_set = False + try: + router = self._make_router(timeout=330) + assert router.timeout == 330 + assert router.request_timeout is None + finally: + litellm.request_timeout = original_value + litellm.request_timeout_explicitly_set = original_flag + + def test_non_stream_prefers_request_timeout_over_router_timeout( + self, explicit_request_timeout + ): + router = self._make_router(timeout=330) + assert router._get_non_stream_timeout(kwargs={}, data={}) == 300 + + def test_stream_prefers_request_timeout_over_router_timeout( + self, explicit_request_timeout + ): + router = self._make_router(timeout=330) + # stream=True resolves through _get_stream_timeout; request_timeout must win. + assert router._get_timeout(kwargs={"stream": True}, data={}) == 300 + + def test_explicit_stream_timeout_still_wins_over_request_timeout( + self, explicit_request_timeout + ): + router = self._make_router(timeout=330, stream_timeout=45) + assert router._get_stream_timeout(kwargs={}, data={}) == 45 + + def test_non_stream_falls_through_to_router_timeout_without_request_timeout(self): + original_value = litellm.request_timeout + original_flag = litellm.request_timeout_explicitly_set + litellm.request_timeout = litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS + litellm.request_timeout_explicitly_set = False + try: + router = self._make_router(timeout=330) + assert router._get_non_stream_timeout(kwargs={}, data={}) == 330 + finally: + litellm.request_timeout = original_value + litellm.request_timeout_explicitly_set = original_flag + + def test_per_deployment_timeout_overrides_request_timeout( + self, explicit_request_timeout + ): + router = self._make_router(timeout=330) + assert router._get_non_stream_timeout(kwargs={}, data={"timeout": 120}) == 120 + + def test_per_request_timeout_overrides_request_timeout( + self, explicit_request_timeout + ): + router = self._make_router(timeout=330) + assert ( + router._get_non_stream_timeout( + kwargs={"timeout": 60}, data={"timeout": 120} + ) + == 60 + ) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index ee64f44d32c..d4ac9659f00 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -537,3 +537,147 @@ def test_inherit_builtin_cache_pricing_noop_for_unknown_backend(): ) assert model_info == {"input_cost_per_token": 0.000003} + + +def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields(): + """The shared-backend-key stripping in Router relies on + CustomPricingLiteLLMParams enumerating every per-deployment pricing field. + If a new pricing field is added to ModelInfoBase but not mirrored here, a + deployment override on that field leaks into the shared backend key and + every sibling deployment reads the wrong rate (LIT-3897). This guard fails + fast when the two drift apart. + """ + import typing + + from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfoBase + + pricing_markers = ("cost", "price", "uplift", "vector_size", "tiered_pricing") + builtin_pricing_fields = { + name + for name in typing.get_type_hints(ModelInfoBase) + if any(marker in name for marker in pricing_markers) + } + denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) + + uncovered = sorted(builtin_pricing_fields - denylisted_fields) + assert not uncovered, ( + "ModelInfoBase pricing fields missing from CustomPricingLiteLLMParams; " + f"these would leak into shared backend keys: {uncovered}" + ) + + +def test_tiered_pricing_override_isolated_from_sibling_via_model_info_lookup(): + """LIT-3897: a deployment that overrides a tiered pricing field + (input_cost_per_token_above_272k_tokens) must not pollute the shared + backend key, so a sibling sharing the same backend resolves its pricing + via litellm.get_model_info (the path /model/info uses) without seeing the + override. + """ + backend_model = "gemini/gemini-2.5-flash" + override = 0.000999 + + builtin_info = litellm.get_model_info(model=backend_model) + assert builtin_info.get("input_cost_per_token_above_272k_tokens") != override + + model_keys = { + "lit3897-tiered-custom": litellm.model_cost.get("lit3897-tiered-custom"), + "lit3897-tiered-sibling": litellm.model_cost.get("lit3897-tiered-sibling"), + backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), + } + try: + Router( + model_list=[ + { + "model_name": "custom-priced-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-tiered-1", + }, + "model_info": { + "id": "lit3897-tiered-custom", + "input_cost_per_token_above_272k_tokens": override, + "cache_read_input_token_cost_above_272k_tokens": override, + }, + }, + { + "model_name": "gemini-2.5-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-tiered-2", + }, + "model_info": {"id": "lit3897-tiered-sibling"}, + }, + ], + ) + + shared = litellm.get_model_info(model=backend_model) + assert shared.get("input_cost_per_token_above_272k_tokens") != override, ( + "Tiered override leaked into the shared backend key; siblings read " + "the wrong rate via /model/info" + ) + assert shared.get("cache_read_input_token_cost_above_272k_tokens") != override + + custom_entry = litellm.model_cost["lit3897-tiered-custom"] + assert custom_entry["input_cost_per_token_above_272k_tokens"] == override + assert custom_entry["cache_read_input_token_cost_above_272k_tokens"] == override + finally: + _restore_model_cost_entries(model_keys) + + +def test_custom_pricing_isolated_from_sibling_via_proxy_model_info_path(): + """LIT-3897 end to end through the proxy resolution helper: the override + deployment reports its custom input rate while the sibling keeps the + canonical gemini rate when /model/info resolves each deployment. Mirrors the + ticket config where the override is set on litellm_params. + """ + from litellm.proxy.proxy_server import _get_proxy_model_info + + backend_model = "gemini/gemini-2.5-flash" + override_input = 5e-05 + override_output = 1e-04 + + builtin_info = litellm.get_model_info(model=backend_model) + builtin_input = builtin_info["input_cost_per_token"] + assert builtin_input != override_input + + model_keys = { + "lit3897-proxy-custom": litellm.model_cost.get("lit3897-proxy-custom"), + "lit3897-proxy-sibling": litellm.model_cost.get("lit3897-proxy-sibling"), + backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), + } + try: + router = Router( + model_list=[ + { + "model_name": "custom-priced-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-proxy-1", + "input_cost_per_token": override_input, + "output_cost_per_token": override_output, + }, + "model_info": {"id": "lit3897-proxy-custom"}, + }, + { + "model_name": "gemini-2.5-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-proxy-2", + }, + "model_info": {"id": "lit3897-proxy-sibling"}, + }, + ], + ) + + resolved = { + m["model_name"]: _get_proxy_model_info(model=copy.deepcopy(m))[ + "model_info" + ]["input_cost_per_token"] + for m in router.model_list + } + + assert resolved["custom-priced-flash"] == override_input + assert resolved["gemini-2.5-flash"] == builtin_input + assert resolved["gemini-2.5-flash"] != resolved["custom-priced-flash"] + finally: + _restore_model_cost_entries(model_keys) diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index 154ba579e4e..af2372616a6 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -4,8 +4,9 @@ GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params """ import pytest -from unittest.mock import MagicMock, patch +from unittest.mock import patch +import litellm from litellm import Router @@ -188,3 +189,133 @@ class TestPerDeploymentNumRetries: # Verify num_retries was converted from string to int assert exc.num_retries == 6 + + +class TestNumRetriesNoneGuard: + """ + Regression tests for the num_retries=None TypeError in async_function_with_retries. + + When num_retries reaches async_function_with_retries as None - e.g. a caller passes + num_retries=None explicitly (dict.get() does not fall back on an existing None value), + an auto_router/complexity_router path does not propagate it, or + Router.update_settings(num_retries=None) is used - AND the underlying call fails with a + retryable error, the comparison `if num_retries > 0:` raised: + + TypeError: '>' not supported between instances of 'NoneType' and 'int' + + This masked the real upstream error (rate limit / connection / 5xx) behind a TypeError. + Related issues: #23316, #25889, #23699, #28126. + """ + + @staticmethod + def _mock_router(num_retries=2): + return Router( + model_list=[ + { + "model_name": "mock-model", + "litellm_params": { + "model": "gpt-4o-mini", + "mock_response": "ok", + }, + } + ], + num_retries=num_retries, + ) + + def test_update_kwargs_normalises_explicit_none_to_router_default(self): + """ + _update_kwargs_before_fallbacks must normalise an explicit num_retries=None to + the router default (not leave it as None), while preserving an explicit 0. + """ + router = self._mock_router(num_retries=4) + + # explicit None -> router default + kwargs = {"num_retries": None} + router._update_kwargs_before_fallbacks(model="mock-model", kwargs=kwargs) + assert kwargs["num_retries"] == 4 + + # explicit 0 is preserved (retries stay disabled) + kwargs = {"num_retries": 0} + router._update_kwargs_before_fallbacks(model="mock-model", kwargs=kwargs) + assert kwargs["num_retries"] == 0 + + # absent -> router default (unchanged behaviour) + kwargs = {} + router._update_kwargs_before_fallbacks(model="mock-model", kwargs=kwargs) + assert kwargs["num_retries"] == 4 + + # explicit None with router default also None -> 0 (mirrors the downstream guard) + router.num_retries = None # simulate update_settings(num_retries=None) (#28126) + kwargs = {"num_retries": None} + router._update_kwargs_before_fallbacks(model="mock-model", kwargs=kwargs) + assert kwargs["num_retries"] == 0 + + @pytest.mark.asyncio + async def test_acompletion_num_retries_none_does_not_raise_typeerror(self): + """ + Per-request num_retries=None + a retryable error must NOT raise TypeError. + The router falls back to its configured num_retries and retries the (transient) + error, so the request succeeds. + """ + router = self._mock_router(num_retries=2) + with patch("asyncio.sleep", return_value=None): + response = await router.acompletion( + model="mock-model", + messages=[{"role": "user", "content": "hi"}], + num_retries=None, # the trigger + mock_testing_rate_limit_error=True, # retryable error path + ) + assert response.choices[0].message.content == "ok" + + @pytest.mark.asyncio + async def test_async_function_with_retries_none_falls_back_to_zero(self): + """ + When both the per-request value AND the router-level setting are None + (e.g. after Router.update_settings(num_retries=None), #28126), num_retries must + fall back to 0 and the real retryable error must surface - not a TypeError. + """ + router = self._mock_router(num_retries=0) + router.num_retries = None # simulate update_settings(num_retries=None) + + async def failing_fn(*args, **kwargs): + raise litellm.RateLimitError( + message="boom", model="mock-model", llm_provider="openai" + ) + + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.RateLimitError): + await router.async_function_with_retries( + original_function=failing_fn, + model="mock-model", + messages=[{"role": "user", "content": "hi"}], + num_retries=None, + ) + + @pytest.mark.asyncio + async def test_async_function_with_retries_none_falls_back_to_router_default(self): + """ + A None per-request num_retries falls back to the router-level setting, so retries + still happen (original_function is invoked more than once) before the real error + is raised - proving None did not silently disable retries or crash. + """ + router = self._mock_router(num_retries=3) + calls = {"n": 0} + + async def failing_fn(*args, **kwargs): + calls["n"] += 1 + raise litellm.InternalServerError( + message="boom", model="mock-model", llm_provider="openai" + ) + + with patch("asyncio.sleep", return_value=None): + with pytest.raises(litellm.InternalServerError): + await router.async_function_with_retries( + original_function=failing_fn, + model="mock-model", + messages=[{"role": "user", "content": "hi"}], + metadata={}, # populated by acompletion in the real path; log_retry needs it + num_retries=None, + ) + + # 1 initial attempt + at least 1 retry -> proves None fell back to a positive int + assert calls["n"] >= 2 diff --git a/tests/test_litellm/test_type_check_gate.py b/tests/test_litellm/test_type_check_gate.py index 18374c5db4b..e99ad0a4f41 100644 --- a/tests/test_litellm/test_type_check_gate.py +++ b/tests/test_litellm/test_type_check_gate.py @@ -56,29 +56,58 @@ def test_paths_outside_repo_are_skipped(): def test_at_or_under_ceiling_passes(): budget = {"no-any-return": {"baseline": 5, "slack": 0}} - assert gate.evaluate({"no-any-return": 5}, budget) == [] + assert gate.evaluate({"no-any-return": 5}, {}, budget) == [] def test_one_more_error_than_ceiling_fails(): budget = {"no-any-return": {"baseline": 5, "slack": 0}} - assert gate.evaluate({"no-any-return": 6}, budget) == [ - gate.Breach("no-any-return", 6, 5) + assert gate.evaluate({"no-any-return": 6}, {}, budget) == [ + gate.Breach("no-any-return", 6, 5, 6) ] def test_slack_absorbs_small_increase_then_fails_past_it(): budget = {"arg-type": {"baseline": 5, "slack": 5}} - assert gate.evaluate({"arg-type": 10}, budget) == [] - assert gate.evaluate({"arg-type": 11}, budget) == [gate.Breach("arg-type", 11, 10)] + assert gate.evaluate({"arg-type": 10}, {}, budget) == [] + assert gate.evaluate({"arg-type": 11}, {}, budget) == [ + gate.Breach("arg-type", 11, 10, 11) + ] def test_unbudgeted_new_code_uses_default_slack(): - assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK}, {}) == [] - assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK + 1}, {}) == [ - gate.Breach("brand-new", gate.DEFAULT_SLACK + 1, gate.DEFAULT_SLACK) + assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK}, {}, {}) == [] + assert gate.evaluate({"brand-new": gate.DEFAULT_SLACK + 1}, {}, {}) == [ + gate.Breach( + "brand-new", + gate.DEFAULT_SLACK + 1, + gate.DEFAULT_SLACK, + gate.DEFAULT_SLACK + 1, + ) ] +def test_drift_already_over_cap_in_base_is_not_blamed_on_a_flat_change(): + # The bystander case: a rule sits over its ceiling because two earlier PRs + # summed past it. A PR that branches off that base and adds nothing must pass + # -- total > cap but total == base, so the `> base` guard spares it. + budget = {"arg-type": {"baseline": 5, "slack": 5}} + assert gate.evaluate({"arg-type": 12}, {"arg-type": 12}, budget) == [] + + +def test_change_that_grows_an_over_cap_rule_is_blamed_for_only_what_it_added(): + # Over cap AND above base: blamed, and `added` is the delta vs base, not the + # whole overage, so the message points at this change's contribution. + budget = {"arg-type": {"baseline": 5, "slack": 5}} + assert gate.evaluate({"arg-type": 14}, {"arg-type": 12}, budget) == [ + gate.Breach("arg-type", 14, 10, 2) + ] + + +def test_reducing_an_over_cap_rule_below_base_passes(): + budget = {"arg-type": {"baseline": 5, "slack": 5}} + assert gate.evaluate({"arg-type": 11}, {"arg-type": 12}, budget) == [] + + def test_no_output_against_a_nonempty_budget_is_a_vacuous_run(): # A crashed type checker emits nothing; the gate must not certify it as clean. budget = {"no-untyped-def": {"baseline": 4888, "slack": 10}} diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d94a86d8e55..cf7afc1af68 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -865,6 +865,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_service_tier": {"type": "boolean"}, "supports_preset": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, + "supports_speed": {"type": "boolean"}, "bedrock_output_config_effort_ceiling": { "type": "string", "enum": ["low", "medium", "high", "max", "xhigh"], diff --git a/tests/test_litellm/types/test_completion.py b/tests/test_litellm/types/test_completion.py index f24b00df3fc..cd51913c5dd 100644 --- a/tests/test_litellm/types/test_completion.py +++ b/tests/test_litellm/types/test_completion.py @@ -8,9 +8,16 @@ Usage: pytest tests/test_litellm/types/test_completion.py -v """ +import dataclasses from typing import List -from litellm.types.completion import CompletionRequest, ChatCompletionMessageParam +import pytest + +from litellm.types.completion import ( + ChatCompletionMessageParam, + CompletionRequest, + _CompletionDispatchContext, +) def test_completion_request_messages_type_validation(): @@ -146,3 +153,55 @@ def test_completion_request_with_all_params(): assert request.presence_penalty == 0.0 assert request.stream is False assert request.n == 1 + + +def _build_dispatch_context() -> _CompletionDispatchContext: + return _CompletionDispatchContext( + _azure_detection_model="gpt-4o", + acompletion=False, + api_base=None, + api_key=None, + api_version=None, + client=None, + custom_llm_provider="openai", + custom_prompt_dict={}, + extra_headers=None, + headers={}, + hf_model_name=None, + kwargs={}, + litellm_params={}, + logger_fn=None, + logging=None, # type: ignore[arg-type] + max_retries=None, + max_tokens=None, + messages=[], + metadata=None, + model="gpt-4o", + model_response=None, # type: ignore[arg-type] + optional_params={}, + organization=None, + provider_config=None, + shared_session=None, + stream=None, + temperature=None, + text_completion=False, + timeout=None, + top_p=None, + ) + + +def test_dispatch_context_is_frozen(): + """A helper must not be able to re-route the call by rebinding a dispatch + input mid-flight; this pins the frozen invariant the dispatch shape relies on.""" + ctx = _build_dispatch_context() + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.model = "claude-haiku-4-5" # type: ignore[misc] + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.custom_llm_provider = "anthropic" # type: ignore[misc] + + +def test_dispatch_context_uses_slots(): + """slots=True keeps the per-call context lightweight (no per-instance __dict__).""" + ctx = _build_dispatch_context() + assert not hasattr(ctx, "__dict__") + assert hasattr(type(ctx), "__slots__") diff --git a/ui/litellm-dashboard/e2e_tests/globalSetup.ts b/ui/litellm-dashboard/e2e_tests/globalSetup.ts index 661155b761f..ef892870268 100644 --- a/ui/litellm-dashboard/e2e_tests/globalSetup.ts +++ b/ui/litellm-dashboard/e2e_tests/globalSetup.ts @@ -39,6 +39,11 @@ async function globalSetup() { if (await dismiss.isVisible({ timeout: 1_500 }).catch(() => false)) { await dismiss.click(); } + // The login flow stores a post-login return URL in the litellm_return_url + // cookie. If the snapshot captures it before the app consumes it, every + // test inheriting this storageState gets yanked to that stale URL the + // first time it mounts a page (the e2e suite's main flake source). + await page.context().clearCookies({ name: "litellm_return_url" }); await page.context().storageState({ path: storagePath }); } catch (e) { fs.mkdirSync("test-results", { recursive: true }); diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 3beae6526e2..2afc145d15b 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -6882,16 +6882,16 @@ } }, "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" @@ -7248,9 +7248,9 @@ } }, "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -8149,10 +8149,20 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -13726,9 +13736,9 @@ } }, "node_modules/ws": { - "version": "8.20.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.1.tgz", - "integrity": "sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "devOptional": true, "license": "MIT", "engines": { diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index c0899be8639..23da0bcd636 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -83,11 +83,11 @@ }, "overrides": { "prismjs": "1.30.0", - "js-yaml": "4.1.1", + "js-yaml": "4.2.0", "glob": "13.0.0", "minimatch": "10.2.4", "lodash": "4.18.1", - "ws": "8.20.1", + "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", "postcss": "8.5.13", diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 5d171139ab5..56999cd1a83 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -935,16 +935,6 @@ print(response.choices[0].message.content)`}
Connection Details
-
- URL: -
- {selectedMcpServer.url} - copyToClipboard(selectedMcpServer.url)} - className="cursor-pointer text-gray-500 hover:text-blue-500 flex-shrink-0" - /> -
-
{selectedMcpServer.command && (
Command: diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts index 856b2726c6d..802555b9d0b 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts @@ -1036,6 +1036,18 @@ describe("EntityUsageExport utils", () => { failed_requests: 2, total_tokens: 500, }, + api_key_breakdown: { + key1: { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 48, + failed_requests: 2, + total_tokens: 500, + }, + metadata: { team_id: "team-1" }, + }, + }, }, "gpt-3.5-turbo": { metrics: { @@ -1045,6 +1057,18 @@ describe("EntityUsageExport utils", () => { failed_requests: 3, total_tokens: 500, }, + api_key_breakdown: { + key2: { + metrics: { + spend: 5.5, + api_requests: 50, + successful_requests: 47, + failed_requests: 3, + total_tokens: 500, + }, + metadata: { team_id: "team-1" }, + }, + }, }, }, }, @@ -1118,6 +1142,18 @@ describe("EntityUsageExport utils", () => { failed_requests: 5, total_tokens: 1000, }, + api_key_breakdown: { + key1: { + metrics: { + spend: 10.5, + api_requests: 100, + successful_requests: 95, + failed_requests: 5, + total_tokens: 1000, + }, + metadata: { team_id: "team-1" }, + }, + }, }, }, }, @@ -1134,12 +1170,229 @@ describe("EntityUsageExport utils", () => { ); }); - it("should aggregate model metrics from api key breakdown", () => { + it("should attribute each model only its own per-key spend", () => { const result = generateDailyWithModelsData(mockSpendDataWithModels, "Team"); const gpt4Entry = result.find((r) => r.Model === "gpt-4"); - expect(gpt4Entry).toBeDefined(); - expect(gpt4Entry?.Requests).toBeGreaterThan(0); + const gpt35Entry = result.find((r) => r.Model === "gpt-3.5-turbo"); + + expect(gpt4Entry?.["Spend ($)"]).toBe("5.0000"); + expect(gpt4Entry?.Requests).toBe(50); + expect(gpt4Entry?.["Total Tokens"]).toBe(500); + + expect(gpt35Entry?.["Spend ($)"]).toBe("5.5000"); + expect(gpt35Entry?.Requests).toBe(50); + expect(gpt35Entry?.["Total Tokens"]).toBe(500); + }); + + it("should not duplicate a user's spend across every model (regression for LIT overcount)", () => { + // One user, one key, that key used two models. The entity-level api_key_breakdown + // carries the key's total (8.0) across both models; each model's api_key_breakdown + // carries only that model's share (3.0 + 5.0). The per-model rows must sum back to + // the user-day total, not repeat the total once per model. + const data: EntitySpendData = { + results: [ + { + date: "2025-02-14", + breakdown: { + entities: { + user1: { + metrics: { + spend: 8.0, + api_requests: 80, + successful_requests: 78, + failed_requests: 2, + total_tokens: 800, + prompt_tokens: 500, + completion_tokens: 300, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 8.0, + api_requests: 80, + successful_requests: 78, + failed_requests: 2, + total_tokens: 800, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + }, + models: { + "claude-3-haiku": { + metrics: { + spend: 3.0, + api_requests: 30, + successful_requests: 29, + failed_requests: 1, + total_tokens: 300, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 3.0, + api_requests: 30, + successful_requests: 29, + failed_requests: 1, + total_tokens: 300, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + "claude-sonnet-4-5": { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 49, + failed_requests: 1, + total_tokens: 500, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 49, + failed_requests: 1, + total_tokens: 500, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + }, + }, + }, + ], + metadata: { + total_spend: 8.0, + total_api_requests: 80, + total_successful_requests: 78, + total_failed_requests: 2, + total_tokens: 800, + }, + }; + + const result = generateDailyWithModelsData(data, "User"); + + expect(result).toHaveLength(2); + + const haiku = result.find((r) => r.Model === "claude-3-haiku"); + const sonnet = result.find((r) => r.Model === "claude-sonnet-4-5"); + + expect(haiku?.["Spend ($)"]).toBe("3.0000"); + expect(sonnet?.["Spend ($)"]).toBe("5.0000"); + + const totalSpend = result.reduce((sum, r) => sum + parseFloat(r["Spend ($)"].replace(/,/g, "")), 0); + const totalRequests = result.reduce((sum, r) => sum + r.Requests, 0); + const totalTokens = result.reduce((sum, r) => sum + r["Total Tokens"], 0); + + expect(totalSpend).toBeCloseTo(8.0, 4); + expect(totalRequests).toBe(80); + expect(totalTokens).toBe(800); + }); + + it("should omit models the user never called instead of fanning out", () => { + // A second key (key2) belongs to a different user and is the only caller of + // gpt-3.5-turbo. user1 only used key1 -> gpt-4. user1 must get exactly one row. + const data: EntitySpendData = { + results: [ + { + date: "2025-02-14", + breakdown: { + entities: { + user1: { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 48, + failed_requests: 2, + total_tokens: 500, + prompt_tokens: 300, + completion_tokens: 200, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 48, + failed_requests: 2, + total_tokens: 500, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + }, + models: { + "gpt-4": { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 48, + failed_requests: 2, + total_tokens: 500, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 5.0, + api_requests: 50, + successful_requests: 48, + failed_requests: 2, + total_tokens: 500, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + "gpt-3.5-turbo": { + metrics: { + spend: 9.0, + api_requests: 90, + successful_requests: 90, + failed_requests: 0, + total_tokens: 900, + }, + api_key_breakdown: { + key2: { + metrics: { + spend: 9.0, + api_requests: 90, + successful_requests: 90, + failed_requests: 0, + total_tokens: 900, + }, + metadata: { team_id: "team-2" }, + }, + }, + }, + }, + }, + }, + ], + metadata: { + total_spend: 14.0, + total_api_requests: 140, + total_successful_requests: 138, + total_failed_requests: 2, + total_tokens: 1400, + }, + }; + + const result = generateDailyWithModelsData(data, "User"); + + expect(result).toHaveLength(1); + expect(result[0].Model).toBe("gpt-4"); + expect(result[0]["Spend ($)"]).toBe("5.0000"); }); it("should use team alias when available", () => { @@ -1312,6 +1565,18 @@ describe("EntityUsageExport utils", () => { failed_requests: 5, total_tokens: 1000, }, + api_key_breakdown: { + key1: { + metrics: { + spend: 10.5, + api_requests: 100, + successful_requests: 95, + failed_requests: 5, + total_tokens: 1000, + }, + metadata: { team_id: "team-1" }, + }, + }, }, }, }, @@ -1595,7 +1860,43 @@ describe("EntityUsageExport utils", () => { metadata: { team_id: "team-1", key_alias: "staging-key" }, }, }, - models: { "gpt-4": { metrics: { spend: 35, api_requests: 350, total_tokens: 3500 } } }, + models: { + "gpt-4": { + metrics: { spend: 35.8, api_requests: 350, total_tokens: 3500 }, + api_key_breakdown: { + key1: { + metrics: { + spend: 10.5, + api_requests: 100, + successful_requests: 95, + failed_requests: 5, + total_tokens: 1000, + }, + metadata: { team_id: "team-1" }, + }, + key1b: { + metrics: { + spend: 5, + api_requests: 50, + successful_requests: 48, + failed_requests: 2, + total_tokens: 500, + }, + metadata: { team_id: "team-1" }, + }, + key2: { + metrics: { + spend: 20.3, + api_requests: 200, + successful_requests: 195, + failed_requests: 5, + total_tokens: 2000, + }, + metadata: { team_id: "team-2" }, + }, + }, + }, + }, }, })), }; @@ -1699,6 +2000,13 @@ describe("EntityUsageExport utils", () => { const result = generateDailyWithModelsData(aggregatedSpendData, "Team"); expect(result.length).toBeGreaterThan(0); expect(result[0]).toHaveProperty("Model"); + + // team-1 = key1 (10.5) + key1b (5) on gpt-4; team-2 = key2 (20.3) on gpt-4. + // Spend must aggregate per team-key, not repeat the model total per team. + const team1 = result.find((r) => r["Team ID"] === "team-1"); + const team2 = result.find((r) => r["Team ID"] === "team-2"); + expect(team1?.["Spend ($)"]).toBe("15.5000"); + expect(team2?.["Spend ($)"]).toBe("20.3000"); }); }); }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts index 2cab3143b9e..cec8af2dc74 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts @@ -237,9 +237,13 @@ export const generateDailyWithModelsData = ( } Object.entries(day.breakdown.models || {}).forEach(([model, modelData]: [string, any]) => { - const apiKeyBreakdown = entityData.api_key_breakdown || {}; + const entityApiKeys = entityData.api_key_breakdown || {}; + const modelApiKeys = modelData.api_key_breakdown || {}; + + Object.keys(entityApiKeys).forEach((apiKey) => { + const keyMetrics = modelApiKeys[apiKey]?.metrics; + if (!keyMetrics) return; - Object.entries(apiKeyBreakdown).forEach(([apiKey, apiKeyData]: [string, any]) => { if (!dailyEntityModels[entity][model]) { dailyEntityModels[entity][model] = { spend: 0, @@ -249,11 +253,11 @@ export const generateDailyWithModelsData = ( tokens: 0, }; } - dailyEntityModels[entity][model].spend += apiKeyData.metrics.spend || 0; - dailyEntityModels[entity][model].requests += apiKeyData.metrics.api_requests || 0; - dailyEntityModels[entity][model].successful += apiKeyData.metrics.successful_requests || 0; - dailyEntityModels[entity][model].failed += apiKeyData.metrics.failed_requests || 0; - dailyEntityModels[entity][model].tokens += apiKeyData.metrics.total_tokens || 0; + dailyEntityModels[entity][model].spend += keyMetrics.spend || 0; + dailyEntityModels[entity][model].requests += keyMetrics.api_requests || 0; + dailyEntityModels[entity][model].successful += keyMetrics.successful_requests || 0; + dailyEntityModels[entity][model].failed += keyMetrics.failed_requests || 0; + dailyEntityModels[entity][model].tokens += keyMetrics.total_tokens || 0; }); }); }); diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index d777ba1b0dc..0b6e5786aaf 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -1097,3 +1097,52 @@ describe("OldTeams - delete team warning copy", () => { ); }); }); + +describe("OldTeams - LIT-2530 organization stays optional for proxy admin with a single org", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockTeamInfoView.mockClear(); + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4"]); + vi.mocked(fetchMCPAccessGroups).mockResolvedValue([]); + vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [] }); + vi.mocked(teamListCall).mockResolvedValue({ teams: [], total: 0, page: 1, page_size: 100, total_pages: 1 }); + vi.mocked(teamCreateCall).mockResolvedValue({ + team_id: "new-team-1", + team_alias: "No Org Team", + models: ["gpt-4"], + organization_id: null, + keys: [], + members_with_roles: [], + spend: 0, + }); + mockUseOrganizations.mockReturnValue({ + data: [{ organization_id: "org-1", organization_alias: "Org 1", models: [], members: [] }], + }); + }); + + it("creates a team with no organization when exactly one organization exists", async () => { + renderWithQueryClient(); + + const createButton = screen.getAllByRole("button", { name: /create team/i })[0]; + act(() => { + fireEvent.click(createButton); + }); + + await waitFor(() => { + expect(screen.getByLabelText(/team name/i)).toBeInTheDocument(); + }); + + fireEvent.change(screen.getByLabelText(/team name/i), { target: { value: "No Org Team" } }); + fireEvent.change(screen.getByTestId("create-team-models-select"), { target: { value: "gpt-4" } }); + + const submitButtons = screen.getAllByRole("button", { name: /create team/i }); + fireEvent.click(submitButtons[submitButtons.length - 1]); + + await waitFor(() => { + expect(teamCreateCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ team_alias: "No Org Team", organization_id: null }), + ); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index adfec4bdf6a..be9015e3730 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -262,14 +262,15 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser useEffect(() => { if (isTeamModalVisible) { const adminOrgs = getAdminOrganizations(userRole, userID, organizations); + const isOrgAdmin = userRole !== "Admin"; - // If there's exactly one organization the user is admin for, preselect it - if (adminOrgs.length === 1) { + // Org admins must scope a team to an org, so with exactly one we preselect it. + // Proxy admins can create org-less teams, so the field stays optional regardless of org count. + if (isOrgAdmin && adminOrgs.length === 1) { const org = adminOrgs[0]; form.setFieldValue("organization_id", org.organization_id); setCurrentOrgForCreateTeam(org); } else { - // Reset the organization selection for multiple orgs form.setFieldValue("organization_id", currentOrg?.organization_id || null); setCurrentOrgForCreateTeam(currentOrg); } @@ -1132,7 +1133,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser : [] } help={ - isSingleOrg + isOrgAdmin && isSingleOrg ? "You can only create teams within this organization" : isOrgAdmin ? "required" @@ -1142,7 +1143,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser onChange(Array.from(e.target.selectedOptions, (o) => (o as HTMLOptionElement).value))} + > + {children} + + ); + Select.displayName = "MockSelect"; + Select.Option = ({ value, disabled, label }: any) => ( + + ); + Select.Option.displayName = "MockSelectOption"; + return { ...actual, Select }; +}); + +import { useMCPAccessGroups } from "@/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups"; +import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; +import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; + +const mockUseMCPServers = vi.mocked(useMCPServers); +const mockUseMCPAccessGroups = vi.mocked(useMCPAccessGroups); +const mockUseMCPToolsets = vi.mocked(useMCPToolsets); + +describe("MCPServerSelector no-mcp-servers option", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseMCPServers.mockReturnValue({ + data: [{ server_id: "srv-1", server_name: "Server One" }], + isLoading: false, + } as any); + mockUseMCPAccessGroups.mockReturnValue({ data: [], isLoading: false } as any); + mockUseMCPToolsets.mockReturnValue({ data: [], isLoading: false } as any); + }); + + const optionByValue = (value: string) => + Array.from(screen.getByTestId("mcp-select").querySelectorAll("option")).find( + (o) => (o as HTMLOptionElement).value === value, + ) as HTMLOptionElement | undefined; + + it("hides the No MCP Servers option by default", () => { + renderWithProviders( + , + ); + expect(optionByValue(NO_MCP_SERVERS_SENTINEL)).toBeUndefined(); + }); + + it("emits an exclusive sentinel when No MCP Servers is selected", async () => { + const onChange = vi.fn(); + renderWithProviders( + , + ); + expect(optionByValue(NO_MCP_SERVERS_SENTINEL)).toBeDefined(); + + await userEvent.selectOptions(screen.getByTestId("mcp-select"), [NO_MCP_SERVERS_SENTINEL]); + + expect(onChange).toHaveBeenCalledWith({ servers: [NO_MCP_SERVERS_SENTINEL], accessGroups: [], toolsets: [] }); + }); + + it("disables real server options while the sentinel is selected", () => { + renderWithProviders( + , + ); + expect(optionByValue("srv-1")?.disabled).toBe(true); + expect(optionByValue(NO_MCP_SERVERS_SENTINEL)?.disabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx index fc4b20517cf..bbda761938e 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx @@ -3,6 +3,7 @@ import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers" import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; import { Select } from "antd"; import React from "react"; +import { NO_MCP_SERVERS_SENTINEL } from "@/components/mcp_tools/constants"; interface MCPServerSelectorProps { onChange: (selected: { servers: string[]; accessGroups: string[]; toolsets: string[] }) => void; @@ -16,6 +17,7 @@ interface MCPServerSelectorProps { placeholder?: string; disabled?: boolean; teamId?: string | null; + allowNoMcpServers?: boolean; } const TOOLSET_PREFIX = "toolset:"; @@ -28,6 +30,7 @@ const MCPServerSelector: React.FC = ({ placeholder = "Select MCP servers", disabled = false, teamId, + allowNoMcpServers = false, }) => { const { data: mcpServers = [], isLoading: serversLoading } = useMCPServers(teamId); const { data: accessGroups = [], isLoading: groupsLoading } = useMCPAccessGroups(); @@ -77,8 +80,15 @@ const MCPServerSelector: React.FC = ({ ...(value?.toolsets || []).map((id) => `${TOOLSET_PREFIX}${id}`), ]; + const hasNoMcpServersSelected = allowNoMcpServers && selectedValues.includes(NO_MCP_SERVERS_SENTINEL); + // Handle selection const handleChange = (selected: string[]) => { + // "No MCP Servers" is exclusive: picking it clears everything else. + if (allowNoMcpServers && selected.includes(NO_MCP_SERVERS_SENTINEL)) { + onChange({ servers: [NO_MCP_SERVERS_SENTINEL], accessGroups: [], toolsets: [] }); + return; + } const toolsetsSelected = selected .filter((v) => v.startsWith(TOOLSET_PREFIX)) .map((v) => v.slice(TOOLSET_PREFIX.length)); @@ -102,12 +112,21 @@ const MCPServerSelector: React.FC = ({ style={{ width: "100%" }} disabled={disabled} filterOption={(input, option) => { + if (option?.value === NO_MCP_SERVERS_SENTINEL) return true; const searchText = options.find((opt) => opt.value === option?.value)?.searchText || ""; return searchText.toLowerCase().includes(input.toLowerCase()); }} > + {allowNoMcpServers && ( + +
+ No MCP Servers + Block all +
+
+ )} {options.map((opt) => ( - +
= ({ team, teams, data, addKey, autoOp accessToken={accessToken} teamId={selectedCreateKeyTeam?.team_id ?? null} placeholder="Select MCP servers or access groups (optional)" + allowNoMcpServers /> @@ -1419,7 +1421,9 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp
s !== NO_MCP_SERVERS_SENTINEL)} toolPermissions={form.getFieldValue("mcp_tool_permissions") || {}} onChange={(toolPerms) => form.setFieldsValue({ mcp_tool_permissions: toolPerms })} /> diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx index b02886ad3fb..f2a2a3d5901 100644 --- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx +++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx @@ -4,6 +4,7 @@ import { ServerIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/ import { Tooltip } from "antd"; import { fetchMCPServers, fetchMCPToolsets } from "../networking"; import { MCPServer, MCPToolset } from "../mcp_tools/types"; +import { NO_MCP_SERVERS_SENTINEL } from "../mcp_tools/constants"; interface MCPServerPermissionsProps { mcpServers: string[]; @@ -94,9 +95,13 @@ export function MCPServerPermissions({ return serverId; }; + const blocksAllMcpServers = mcpServers.includes(NO_MCP_SERVERS_SENTINEL); + // Merge servers and access groups into one list const mergedItems = [ - ...mcpServers.map((server) => ({ type: "server", value: server })), + ...mcpServers + .filter((server) => server !== NO_MCP_SERVERS_SENTINEL) + .map((server) => ({ type: "server", value: server })), ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group })), ]; const totalCount = mergedItems.length + mcpToolsets.length; @@ -106,12 +111,19 @@ export function MCPServerPermissions({
MCP Servers - - {totalCount} + + {blocksAllMcpServers ? "Blocked" : totalCount}
- {totalCount > 0 ? ( + {blocksAllMcpServers ? ( +
+ + + No MCP servers — this key is blocked from all MCP servers, including its team's servers + +
+ ) : totalCount > 0 ? (
{mergedItems.map((item, index) => { const toolsForServer = item.type === "server" ? mcpToolPermissions[item.value] : undefined; diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index a73cf699ac8..acc77d1263b 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -62,6 +62,22 @@ describe("provider_info_helpers", () => { expect(result.logo).toBe(providerLogoMap[Providers.Groq]); }); + it("should map bedrock_mantle slug to Bedrock Mantle display name and logo", () => { + const result = getProviderLogoAndName("bedrock_mantle"); + expect(result.displayName).toBe(Providers.BedrockMantle); + expect(result.logo).toBe(providerLogoMap[Providers.BedrockMantle]); + }); + + it("should resolve the BedrockMantle enum key to the Bedrock Mantle logo", () => { + // The Add Model dropdown passes the provider_map key ("BedrockMantle"), + // not the slug ("bedrock_mantle"). Unlike "Bedrock", the key does not + // lowercase-match its slug, so without the enum-key fallback this would + // render a blank fallback logo for a Bedrock variant (LIT-3885). + const result = getProviderLogoAndName("BedrockMantle"); + expect(result.displayName).toBe(Providers.BedrockMantle); + expect(result.logo).toBe(providerLogoMap[Providers.BedrockMantle]); + }); + it("should handle provider values case-insensitively", () => { const result = getProviderLogoAndName("OPENAI"); expect(result.displayName).toBe(Providers.OpenAI); @@ -306,6 +322,23 @@ describe("provider_info_helpers", () => { expect(result).not.toContain("openai-model"); }); + it("should return only bedrock_mantle models when called with 'BedrockMantle' provider key", () => { + // Selecting "Amazon Bedrock Mantle" in the dropdown must populate the + // model field with the Mantle models and exclude the regular Bedrock + // ones, so onboarding a gpt-oss model is a one-click flow (LIT-3885). + const modelMap = { + "bedrock_mantle/openai.gpt-oss-120b": { litellm_provider: "bedrock_mantle" }, + "bedrock_mantle/openai.gpt-5.5": { litellm_provider: "bedrock_mantle" }, + "bedrock-base": { litellm_provider: "bedrock" }, + "bedrock-converse-model": { litellm_provider: "bedrock_converse" }, + }; + const result = getProviderModels("BedrockMantle" as Providers, modelMap); + expect(result).toContain("bedrock_mantle/openai.gpt-oss-120b"); + expect(result).toContain("bedrock_mantle/openai.gpt-5.5"); + expect(result).not.toContain("bedrock-base"); + expect(result).not.toContain("bedrock-converse-model"); + }); + it("should include fireworks_ai-embedding-models when called with 'FireworksAI' provider key", () => { const modelMap = { "fireworks-base": { litellm_provider: "fireworks_ai" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 727eebfe951..9151ce6ba9a 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -70,9 +70,9 @@ export enum Providers { OOBABOOGA = "Oobabooga", OpenAI = "OpenAI", OPENAI_LIKE = "Openai Like", - OpenAI_Compatible = "OpenAI-Compatible Endpoints (Together AI, etc.)", + OpenAI_Compatible = "OpenAI-Compatible Chat Completions (Together AI, vLLM, etc.)", OpenAI_Text = "OpenAI Text Completion", - OpenAI_Text_Compatible = "OpenAI-Compatible Text Completion Models (Together AI, etc.)", + OpenAI_Text_Compatible = "OpenAI-Compatible Completions (legacy /v1/completions)", Openrouter = "Openrouter", Oracle = "Oracle Cloud Infrastructure (OCI)", OVHCLOUD = "Ovhcloud", @@ -318,10 +318,12 @@ export const getProviderLogoAndName = (providerValue: string): { logo: string; d return { logo, displayName }; } - // Find the enum key by matching provider_map values - const enumKey = Object.keys(provider_map).find( - (key) => provider_map[key].toLowerCase() === providerValue.toLowerCase(), - ); + // Resolve by the litellm provider slug (e.g. "bedrock_mantle"); fall back to + // the enum key (e.g. "BedrockMantle") for callers like the Add Model dropdown + // that pass the key instead of the slug. + const enumKey = + Object.keys(provider_map).find((key) => provider_map[key].toLowerCase() === providerValue.toLowerCase()) ?? + Object.keys(provider_map).find((key) => key.toLowerCase() === providerValue.toLowerCase()); if (!enumKey) { return { logo: "", displayName: providerValue }; diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index 9979fce5ebd..9d1e31804ef 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeAll, beforeEach } from "vitest"; -import { render, screen, waitFor } from "@testing-library/react"; -import PublicModelHub from "./public_model_hub"; +import { render, screen, waitFor, fireEvent } from "@testing-library/react"; +import { flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import PublicModelHub, { publicMCPHubColumns, MCPServerData } from "./public_model_hub"; vi.mock("next/navigation", () => ({ useRouter: vi.fn(() => ({ @@ -186,3 +187,80 @@ describe("PublicModelHub", () => { }); }); }); + +const PUBLIC_SERVER_URL = "https://mcp.exa.ai/mcp"; + +const mockMcpServer: MCPServerData = { + server_id: "server-1", + name: "exa_test", + server_name: "exa_test", + url: PUBLIC_SERVER_URL, + transport: "http", + auth_type: "none", + mcp_info: { server_name: "exa_test", description: "Fast, intelligent web search and web crawling" }, +}; + +function PublicMcpTestTable({ data }: { data: MCPServerData[] }) { + const columns = publicMCPHubColumns(vi.fn()); + const table = useReactTable({ data, columns, getCoreRowModel: getCoreRowModel() }); + + return ( + + + {table.getHeaderGroups().map((hg) => ( + + {hg.headers.map((h) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
{flexRender(h.column.columnDef.header, h.getContext())}
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ ); +} + +describe("publicMCPHubColumns", () => { + it("keeps the non-sensitive columns", () => { + render(); + expect(screen.getByText("Server Name")).toBeInTheDocument(); + expect(screen.getByText("Transport")).toBeInTheDocument(); + expect(screen.getByText("Auth Type")).toBeInTheDocument(); + }); + + it("does not expose a URL column header", () => { + render(); + expect(screen.queryByText("URL")).not.toBeInTheDocument(); + expect(publicMCPHubColumns(vi.fn()).some((c) => c.header === "URL")).toBe(false); + }); + + it("does not render the server url anywhere in the table", () => { + render(); + expect(screen.queryByText(PUBLIC_SERVER_URL)).not.toBeInTheDocument(); + }); +}); + +describe("public hub MCP details modal", () => { + it("does not show the upstream url when a server is opened", async () => { + const networkingModule = await import("./networking"); + vi.mocked(networkingModule.mcpHubPublicServersCall).mockResolvedValue([mockMcpServer]); + + render(); + + fireEvent.click(await screen.findByRole("tab", { name: /MCP Hub/i })); + fireEvent.click(await screen.findByRole("button", { name: "exa_test" })); + + // "Server Overview" only exists inside the opened MCP details modal, + // so finding it proves the modal rendered and the url assertion is not vacuous. + await screen.findByText("Server Overview"); + expect(screen.queryByText(PUBLIC_SERVER_URL)).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 62d8f2644ec..c91b783b047 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -74,12 +74,11 @@ interface AgentCard { [key: string]: any; } -interface MCPServerData { +export interface MCPServerData { server_id: string; name: string; alias?: string | null; server_name: string; - url: string; transport: string; spec_path?: string | null; auth_type: string; @@ -96,6 +95,73 @@ interface PublicModelHubProps { isEmbedded?: boolean; // When true, hides navbar and adjusts layout for embedding in dashboard } +export const publicMCPHubColumns = (showMcpModal: (server: MCPServerData) => void): ColumnDef[] => [ + { + header: "Server Name", + accessorKey: "server_name", + enableSorting: true, + cell: ({ row }) => ( +
+ + + +
+ ), + size: 150, + }, + { + header: "Description", + accessorKey: "mcp_info.description", + enableSorting: false, + cell: ({ row }) => { + const description = String(row.original.mcp_info?.description ?? "-"); + const truncated = description.length > 80 ? description.substring(0, 80) + "..." : description; + return ( + + {truncated} + + ); + }, + size: 250, + }, + { + header: "Transport", + accessorKey: "transport", + enableSorting: true, + cell: ({ row }) => { + const transport = row.original.transport; + return ( + + {transport} + + ); + }, + size: 100, + }, + { + header: "Auth Type", + accessorKey: "auth_type", + enableSorting: true, + cell: ({ row }) => { + const authType = row.original.auth_type; + const color = authType === "none" ? "gray" : "green"; + return ( + + {authType} + + ); + }, + size: 100, + }, +]; + const PublicModelHub: React.FC = ({ accessToken, isEmbedded = false }) => { const [modelHubData, setModelHubData] = useState(null); const [agentHubData, setAgentHubData] = useState(null); @@ -889,94 +955,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded }, ]; - const publicMCPHubColumns = (): ColumnDef[] => [ - { - header: "Server Name", - accessorKey: "server_name", - enableSorting: true, - cell: ({ row }) => ( -
- - - -
- ), - size: 150, - }, - { - header: "Description", - accessorKey: "mcp_info.description", - enableSorting: false, - cell: ({ row }) => { - const description = String(row.original.mcp_info?.description ?? "-"); - const truncated = description.length > 80 ? description.substring(0, 80) + "..." : description; - return ( - - {truncated} - - ); - }, - size: 250, - }, - { - header: "URL", - accessorKey: "url", - enableSorting: false, - cell: ({ row }) => { - const url = row.original.url ?? ""; - const truncated = url.length > 40 ? url.substring(0, 40) + "..." : url; - return ( - -
- {truncated} - copyToClipboard(url)} - className="cursor-pointer text-gray-500 hover:text-blue-500 w-3 h-3" - /> -
-
- ); - }, - size: 200, - }, - { - header: "Transport", - accessorKey: "transport", - enableSorting: true, - cell: ({ row }) => { - const transport = row.original.transport; - return ( - - {transport} - - ); - }, - size: 100, - }, - { - header: "Auth Type", - accessorKey: "auth_type", - enableSorting: true, - cell: ({ row }) => { - const authType = row.original.auth_type; - const color = authType === "none" ? "gray" : "green"; - return ( - - {authType} - - ); - }, - size: 100, - }, - ]; - return (
@@ -1286,7 +1264,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Description: {selectedMcpServer.mcp_info?.description || "-"}
-
diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index fe0fa1ab0c2..2f2d0097455 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -19,6 +19,7 @@ import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata import { BudgetWindowEntry, BudgetWindowsEditor } from "../key_team_helpers/BudgetWindowsEditor"; import { KeyResponse } from "../key_team_helpers/key_list"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; +import { NO_MCP_SERVERS_SENTINEL } from "../mcp_tools/constants"; import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; import NotificationsManager from "../molecules/notifications_manager"; import { getPromptsList, modelAvailableCall, tagListCall } from "../networking"; @@ -618,6 +619,7 @@ export function KeyEditView({ value={form.getFieldValue("mcp_servers_and_groups")} accessToken={accessToken || ""} placeholder="Select MCP servers or access groups (optional)" + allowNoMcpServers /> @@ -637,7 +639,9 @@ export function KeyEditView({
s !== NO_MCP_SERVERS_SENTINEL, + )} toolPermissions={form.getFieldValue("mcp_tool_permissions") || {}} onChange={(toolPerms) => form.setFieldsValue({ mcp_tool_permissions: toolPerms })} /> diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.budget_display.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.budget_display.test.tsx index 77abde3d870..5407d37fcf1 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.budget_display.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.budget_display.test.tsx @@ -1,7 +1,7 @@ import { renderWithProviders } from "../../../tests/test-utils"; import { screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { KeyResponse } from "../key_team_helpers/key_list"; +import { KeyResponse, Team } from "../key_team_helpers/key_list"; import KeyInfoView from "./key_info_view"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useTeams from "@/app/(dashboard)/hooks/useTeams"; @@ -103,8 +103,26 @@ const baseAuthorized = { userEmail: null, disabledPersonalKeyCreation: null, showSSOBanner: false, + isLoading: false, + isAuthorized: true, }; +const makeTeam = (overrides: Partial): Team => ({ + team_id: "team-default", + team_alias: "Default Team", + models: [], + max_budget: null, + budget_duration: null, + tpm_limit: null, + rpm_limit: null, + organization_id: "", + created_at: "2026-01-01T00:00:00Z", + keys: [], + members_with_roles: [], + spend: 0, + ...overrides, +}); + describe("KeyInfoView overview budget display (LIT-2845)", () => { beforeEach(() => { vi.mocked(useTeams).mockReturnValue({ teams: [], setTeams: vi.fn() }); @@ -151,7 +169,64 @@ describe("KeyInfoView overview budget display (LIT-2845)", () => { it("renders 'Unlimited' when max_budget is null", async () => { renderWithProviders( {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + await waitFor(() => { + expect(screen.getByText(/of Unlimited/)).toBeInTheDocument(); + }); + }); + + it("renders team budget with alias and duration when key has no own budget but team has one", async () => { + vi.mocked(useTeams).mockReturnValue({ + teams: [makeTeam({ team_id: "team-123", team_alias: "Test Budget", max_budget: 1200, budget_duration: "30d" })], + setTeams: vi.fn(), + }); + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + await waitFor(() => { + expect(screen.getByText(/of \$1,200\.00 \(Team: Test Budget \/ 30d\)/)).toBeInTheDocument(); + }); + }); + + it("renders team budget without duration when team has no budget_duration", async () => { + vi.mocked(useTeams).mockReturnValue({ + teams: [makeTeam({ team_id: "team-456", team_alias: "No Duration Team", max_budget: 500 })], + setTeams: vi.fn(), + }); + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + await waitFor(() => { + expect(screen.getByText(/of \$500\.00 \(Team: No Duration Team\)/)).toBeInTheDocument(); + }); + }); + + it("renders 'Unlimited' when key has no budget and team also has no budget", async () => { + vi.mocked(useTeams).mockReturnValue({ + teams: [makeTeam({ team_id: "team-789", team_alias: "Free Team" })], + setTeams: vi.fn(), + }); + renderWithProviders( + {}} keyId={"test-key-id"} onKeyDataUpdate={() => {}} diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 018880b70aa..4244ae2d794 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -411,6 +411,15 @@ export default function KeyInfoView({ }); }; + const parentTeam = currentKeyData.team_id ? teamsData?.find((team) => team.team_id === currentKeyData.team_id) : null; + + const budgetDisplay = + currentKeyData.max_budget !== null + ? `$${formatNumberWithCommas(currentKeyData.max_budget, 2)}` + : parentTeam?.max_budget != null + ? `$${formatNumberWithCommas(parentTeam.max_budget, 2)} (Team: ${parentTeam.team_alias || parentTeam.team_id}${parentTeam.budget_duration ? ` / ${parentTeam.budget_duration}` : ""})` + : "Unlimited"; + return (
Spend
${formatNumberWithCommas(currentKeyData.spend, 4)} - - of{" "} - {currentKeyData.max_budget !== null - ? `$${formatNumberWithCommas(currentKeyData.max_budget, 2)}` - : "Unlimited"} - + of {budgetDisplay}
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx index b73dcafcdc3..e7295ed7a72 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/PrettyMessagesView.test.tsx @@ -27,6 +27,17 @@ describe("PrettyMessagesView", () => { expect(screen.getByText("Hi there!")).toBeInTheDocument(); }); + it("renders input when request is a bare messages array (cold storage payload)", () => { + const request = [{ role: "user", content: "Write me a poem" }]; + const response = { + choices: [{ message: { role: "assistant", content: "A quiet moment." } }], + }; + + render(); + expect(screen.getByText("Write me a poem")).toBeInTheDocument(); + expect(screen.getByText("A quiet moment.")).toBeInTheDocument(); + }); + it("should render the realtime pretty view for realtime API responses", () => { const request = {}; const response = { diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts index 32ae294b1ee..09b8f551c1d 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts @@ -39,18 +39,24 @@ export const ROLE_STYLES: Record = { * Parse request messages and response message from log data */ export const parseMessages = (request: any, response: any): ParsedMessages => { - // Parse request messages + // Parse request messages. `request` is either the raw request body + // ({ messages: [...] }) or, when prompts come from cold storage, the bare + // messages array itself. const requestMessages: ParsedMessage[] = []; - if (request?.messages && Array.isArray(request.messages)) { - request.messages.forEach((msg: any) => { - requestMessages.push({ - role: msg.role || "user", - content: parseMessageContent(msg.content), - toolCallId: msg.tool_call_id, - }); + const requestMessageList = Array.isArray(request) + ? request + : Array.isArray(request?.messages) + ? request.messages + : []; + + requestMessageList.forEach((msg: any) => { + requestMessages.push({ + role: msg.role || "user", + content: parseMessageContent(msg.content), + toolCallId: msg.tool_call_id, }); - } + }); // Parse response message let responseMessage: ParsedMessage | null = null; diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index 5a253dd1122..1265b8449de 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -327,7 +327,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] }, }, { - header: "Key Name", + header: "Key Alias", accessorKey: "metadata.user_api_key_alias", cell: (info: any) => ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 024d5ecf2a2..6fae14ee6ec 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25112,6 +25112,8 @@ export interface components { } | null; /** Adaptive Router Default Model */ adaptive_router_default_model?: string | null; + /** Annotation Cost Per Page */ + annotation_cost_per_page?: number | null; /** Api Base */ api_base?: string | null; /** Api Key */ @@ -25156,8 +25158,12 @@ export interface components { cache_read_input_token_cost_above_200k_tokens?: number | null; /** Cache Read Input Token Cost Above 200K Tokens Priority */ cache_read_input_token_cost_above_200k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens */ + cache_read_input_token_cost_above_272k_tokens?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 512K Tokens */ + cache_read_input_token_cost_above_512k_tokens?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -25194,6 +25200,8 @@ export interface components { input_cost_per_image?: number | null; /** Input Cost Per Image Above 128K Tokens */ input_cost_per_image_above_128k_tokens?: number | null; + /** Input Cost Per Image Token */ + input_cost_per_image_token?: number | null; /** Input Cost Per Pixel */ input_cost_per_pixel?: number | null; /** Input Cost Per Query */ @@ -25208,8 +25216,12 @@ export interface components { input_cost_per_token_above_200k_tokens?: number | null; /** Input Cost Per Token Above 200K Tokens Priority */ input_cost_per_token_above_200k_tokens_priority?: number | null; + /** Input Cost Per Token Above 272K Tokens */ + input_cost_per_token_above_272k_tokens?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ input_cost_per_token_above_272k_tokens_priority?: number | null; + /** Input Cost Per Token Above 512K Tokens */ + input_cost_per_token_above_512k_tokens?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -25255,6 +25267,10 @@ export interface components { model_info?: { [key: string]: unknown; } | null; + /** Ocr Cost Per Credit */ + ocr_cost_per_credit?: number | null; + /** Ocr Cost Per Page */ + ocr_cost_per_page?: number | null; /** Organization */ organization?: string | null; /** Output Cost Per Audio Per Second */ @@ -25285,8 +25301,12 @@ export interface components { output_cost_per_token_above_200k_tokens?: number | null; /** Output Cost Per Token Above 200K Tokens Priority */ output_cost_per_token_above_200k_tokens_priority?: number | null; + /** Output Cost Per Token Above 272K Tokens */ + output_cost_per_token_above_272k_tokens?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ output_cost_per_token_above_272k_tokens_priority?: number | null; + /** Output Cost Per Token Above 512K Tokens */ + output_cost_per_token_above_512k_tokens?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */ @@ -25295,6 +25315,8 @@ export interface components { output_cost_per_token_priority?: number | null; /** Output Cost Per Video Per Second */ output_cost_per_video_per_second?: number | null; + /** Output Vector Size */ + output_vector_size?: number | null; /** Quality Router Config */ quality_router_config?: { [key: string]: unknown; @@ -25303,6 +25325,10 @@ export interface components { quality_router_default_model?: string | null; /** Region Name */ region_name?: string | null; + /** Regional Processing Uplift Multiplier Eu */ + regional_processing_uplift_multiplier_eu?: number | null; + /** Regional Processing Uplift Multiplier Us */ + regional_processing_uplift_multiplier_us?: number | null; /** Rpm */ rpm?: number | null; /** S3 Bucket Name */ @@ -26776,8 +26802,6 @@ export interface components { * @enum {string} */ transport: "sse" | "http" | "stdio"; - /** Url */ - url?: string | null; }; /** * MCPSemanticFilterSettings @@ -32796,6 +32820,8 @@ export interface components { } | null; /** Adaptive Router Default Model */ adaptive_router_default_model?: string | null; + /** Annotation Cost Per Page */ + annotation_cost_per_page?: number | null; /** Api Base */ api_base?: string | null; /** Api Key */ @@ -32840,8 +32866,12 @@ export interface components { cache_read_input_token_cost_above_200k_tokens?: number | null; /** Cache Read Input Token Cost Above 200K Tokens Priority */ cache_read_input_token_cost_above_200k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens */ + cache_read_input_token_cost_above_272k_tokens?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 512K Tokens */ + cache_read_input_token_cost_above_512k_tokens?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -32878,6 +32908,8 @@ export interface components { input_cost_per_image?: number | null; /** Input Cost Per Image Above 128K Tokens */ input_cost_per_image_above_128k_tokens?: number | null; + /** Input Cost Per Image Token */ + input_cost_per_image_token?: number | null; /** Input Cost Per Pixel */ input_cost_per_pixel?: number | null; /** Input Cost Per Query */ @@ -32892,8 +32924,12 @@ export interface components { input_cost_per_token_above_200k_tokens?: number | null; /** Input Cost Per Token Above 200K Tokens Priority */ input_cost_per_token_above_200k_tokens_priority?: number | null; + /** Input Cost Per Token Above 272K Tokens */ + input_cost_per_token_above_272k_tokens?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ input_cost_per_token_above_272k_tokens_priority?: number | null; + /** Input Cost Per Token Above 512K Tokens */ + input_cost_per_token_above_512k_tokens?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -32939,6 +32975,10 @@ export interface components { model_info?: { [key: string]: unknown; } | null; + /** Ocr Cost Per Credit */ + ocr_cost_per_credit?: number | null; + /** Ocr Cost Per Page */ + ocr_cost_per_page?: number | null; /** Organization */ organization?: string | null; /** Output Cost Per Audio Per Second */ @@ -32969,8 +33009,12 @@ export interface components { output_cost_per_token_above_200k_tokens?: number | null; /** Output Cost Per Token Above 200K Tokens Priority */ output_cost_per_token_above_200k_tokens_priority?: number | null; + /** Output Cost Per Token Above 272K Tokens */ + output_cost_per_token_above_272k_tokens?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ output_cost_per_token_above_272k_tokens_priority?: number | null; + /** Output Cost Per Token Above 512K Tokens */ + output_cost_per_token_above_512k_tokens?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */ @@ -32979,6 +33023,8 @@ export interface components { output_cost_per_token_priority?: number | null; /** Output Cost Per Video Per Second */ output_cost_per_video_per_second?: number | null; + /** Output Vector Size */ + output_vector_size?: number | null; /** Quality Router Config */ quality_router_config?: { [key: string]: unknown; @@ -32987,6 +33033,10 @@ export interface components { quality_router_default_model?: string | null; /** Region Name */ region_name?: string | null; + /** Regional Processing Uplift Multiplier Eu */ + regional_processing_uplift_multiplier_eu?: number | null; + /** Regional Processing Uplift Multiplier Us */ + regional_processing_uplift_multiplier_us?: number | null; /** Rpm */ rpm?: number | null; /** S3 Bucket Name */ diff --git a/uv.lock b/uv.lock index f3ca7b4e626..cac1696bf34 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-06-17T22:13:32.966924Z" +exclude-newer = "2026-06-20T23:16:25.061268Z" exclude-newer-span = "P3D" [manifest] @@ -19,7 +19,7 @@ members = [ "litellm-proxy-extras", ] constraints = [ - { name = "aiohttp", specifier = ">=3.13.5,<3.14" }, + { name = "aiohttp", specifier = ">=3.14.1,<4.0" }, { name = "tornado", specifier = ">=6.5.6" }, ] @@ -71,7 +71,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -81,78 +81,88 @@ dependencies = [ { name = "frozenlist" }, { name = "multidict" }, { name = "propcache" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/85/cebc47ee74d8b408749073a1a46c6fcba13d170dc8af7e61996c6c9394ac/aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b", size = 750547, upload-time = "2026-03-31T21:56:30.024Z" }, - { url = "https://files.pythonhosted.org/packages/05/98/afd308e35b9d3d8c9ec54c0918f1d722c86dc17ddfec272fcdbcce5a3124/aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5", size = 503535, upload-time = "2026-03-31T21:56:31.935Z" }, - { url = "https://files.pythonhosted.org/packages/6f/4d/926c183e06b09d5270a309eb50fbde7b09782bfd305dec1e800f329834fb/aiohttp-3.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f546a4dc1e6a5edbb9fd1fd6ad18134550e096a5a43f4ad74acfbd834fc6670", size = 497830, upload-time = "2026-03-31T21:56:33.654Z" }, - { url = "https://files.pythonhosted.org/packages/e4/d6/f47d1c690f115a5c2a5e8938cce4a232a5be9aac5c5fb2647efcbbbda333/aiohttp-3.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c86969d012e51b8e415a8c6ce96f7857d6a87d6207303ab02d5d11ef0cad2274", size = 1682474, upload-time = "2026-03-31T21:56:35.513Z" }, - { url = "https://files.pythonhosted.org/packages/01/44/056fd37b1bb52eac760303e5196acc74d9d546631b035704ae5927f7b4ac/aiohttp-3.13.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b6f6cd1560c5fa427e3b6074bb24d2c64e225afbb7165008903bd42e4e33e28a", size = 1655259, upload-time = "2026-03-31T21:56:37.843Z" }, - { url = "https://files.pythonhosted.org/packages/91/9f/78eb1a20c1c28ae02f6a3c0f4d7b0dcc66abce5290cadd53d78ce3084175/aiohttp-3.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:636bc362f0c5bbc7372bc3ae49737f9e3030dbce469f0f422c8f38079780363d", size = 1736204, upload-time = "2026-03-31T21:56:39.822Z" }, - { url = "https://files.pythonhosted.org/packages/de/6c/d20d7de23f0b52b8c1d9e2033b2db1ac4dacbb470bb74c56de0f5f86bb4f/aiohttp-3.13.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a7cbeb06d1070f1d14895eeeed4dac5913b22d7b456f2eb969f11f4b3993796", size = 1826198, upload-time = "2026-03-31T21:56:41.378Z" }, - { url = "https://files.pythonhosted.org/packages/2f/86/a6f3ff1fd795f49545a7c74b2c92f62729135d73e7e4055bf74da5a26c82/aiohttp-3.13.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bca9ef7517fd7874a1a08970ae88f497bf5c984610caa0bf40bd7e8450852b95", size = 1681329, upload-time = "2026-03-31T21:56:43.374Z" }, - { url = "https://files.pythonhosted.org/packages/fb/68/84cd3dab6b7b4f3e6fe9459a961acb142aaab846417f6e8905110d7027e5/aiohttp-3.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:019a67772e034a0e6b9b17c13d0a8fe56ad9fb150fc724b7f3ffd3724288d9e5", size = 1560023, upload-time = "2026-03-31T21:56:45.031Z" }, - { url = "https://files.pythonhosted.org/packages/41/2c/db61b64b0249e30f954a65ab4cb4970ced57544b1de2e3c98ee5dc24165f/aiohttp-3.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f34ecee82858e41dd217734f0c41a532bd066bcaab636ad830f03a30b2a96f2a", size = 1652372, upload-time = "2026-03-31T21:56:47.075Z" }, - { url = "https://files.pythonhosted.org/packages/25/6f/e96988a6c982d047810c772e28c43c64c300c943b0ed5c1c0c4ce1e1027c/aiohttp-3.13.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4eac02d9af4813ee289cd63a361576da36dba57f5a1ab36377bc2600db0cbb73", size = 1662031, upload-time = "2026-03-31T21:56:48.835Z" }, - { url = "https://files.pythonhosted.org/packages/b7/26/a56feace81f3d347b4052403a9d03754a0ab23f7940780dada0849a38c92/aiohttp-3.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4beac52e9fe46d6abf98b0176a88154b742e878fdf209d2248e99fcdf73cd297", size = 1708118, upload-time = "2026-03-31T21:56:50.833Z" }, - { url = "https://files.pythonhosted.org/packages/78/6e/b6173a8ff03d01d5e1a694bc06764b5dad1df2d4ed8f0ceec12bb3277936/aiohttp-3.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c180f480207a9b2475f2b8d8bd7204e47aec952d084b2a2be58a782ffcf96074", size = 1548667, upload-time = "2026-03-31T21:56:52.81Z" }, - { url = "https://files.pythonhosted.org/packages/16/13/13296ffe2c132d888b3fe2c195c8b9c0c24c89c3fa5cc2c44464dc23b22e/aiohttp-3.13.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2837fb92951564d6339cedae4a7231692aa9f73cbc4fb2e04263b96844e03b4e", size = 1724490, upload-time = "2026-03-31T21:56:54.541Z" }, - { url = "https://files.pythonhosted.org/packages/7a/b4/1f1c287f4a79782ef36e5a6e62954c85343bc30470d862d30bd5f26c9fa2/aiohttp-3.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9010032a0b9710f58012a1e9c222528763d860ba2ee1422c03473eab47703e7", size = 1667109, upload-time = "2026-03-31T21:56:56.21Z" }, - { url = "https://files.pythonhosted.org/packages/ef/42/8461a2aaf60a8f4ea4549a4056be36b904b0eb03d97ca9a8a2604681a500/aiohttp-3.13.5-cp310-cp310-win32.whl", hash = "sha256:7c4b6668b2b2b9027f209ddf647f2a4407784b5d88b8be4efcc72036f365baf9", size = 439478, upload-time = "2026-03-31T21:56:58.292Z" }, - { url = "https://files.pythonhosted.org/packages/e5/71/06956304cb5ee439dfe8d86e1b2e70088bd88ed1ced1f42fb29e5d855f0e/aiohttp-3.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:cd3db5927bf9167d5a6157ddb2f036f6b6b0ad001ac82355d43e97a4bde76d76", size = 462047, upload-time = "2026-03-31T21:57:00.257Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f5/a20c4ac64aeaef1679e25c9983573618ff765d7aa829fa2b84ae7573169e/aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6", size = 757513, upload-time = "2026-03-31T21:57:02.146Z" }, - { url = "https://files.pythonhosted.org/packages/75/0a/39fa6c6b179b53fcb3e4b3d2b6d6cad0180854eda17060c7218540102bef/aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d", size = 506748, upload-time = "2026-03-31T21:57:04.275Z" }, - { url = "https://files.pythonhosted.org/packages/87/ec/e38ce072e724fd7add6243613f8d1810da084f54175353d25ccf9f9c7e5a/aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c", size = 501673, upload-time = "2026-03-31T21:57:06.208Z" }, - { url = "https://files.pythonhosted.org/packages/ba/ba/3bc7525d7e2beaa11b309a70d48b0d3cfc3c2089ec6a7d0820d59c657053/aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb", size = 1763757, upload-time = "2026-03-31T21:57:07.882Z" }, - { url = "https://files.pythonhosted.org/packages/5e/ab/e87744cf18f1bd78263aba24924d4953b41086bd3a31d22452378e9028a0/aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6", size = 1720152, upload-time = "2026-03-31T21:57:09.946Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f3/ed17a6f2d742af17b50bae2d152315ed1b164b07a5fd5cc1754d99e4dfa5/aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13", size = 1818010, upload-time = "2026-03-31T21:57:12.157Z" }, - { url = "https://files.pythonhosted.org/packages/53/06/ecbc63dc937192e2a5cb46df4d3edb21deb8225535818802f210a6ea5816/aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174", size = 1907251, upload-time = "2026-03-31T21:57:14.023Z" }, - { url = "https://files.pythonhosted.org/packages/7e/a5/0521aa32c1ddf3aa1e71dcc466be0b7db2771907a13f18cddaa45967d97b/aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc", size = 1759969, upload-time = "2026-03-31T21:57:16.146Z" }, - { url = "https://files.pythonhosted.org/packages/f6/78/a38f8c9105199dd3b9706745865a8a59d0041b6be0ca0cc4b2ccf1bab374/aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6", size = 1616871, upload-time = "2026-03-31T21:57:17.856Z" }, - { url = "https://files.pythonhosted.org/packages/6f/41/27392a61ead8ab38072105c71aa44ff891e71653fe53d576a7067da2b4e8/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49", size = 1739844, upload-time = "2026-03-31T21:57:19.679Z" }, - { url = "https://files.pythonhosted.org/packages/6e/55/5564e7ae26d94f3214250009a0b1c65a0c6af4bf88924ccb6fdab901de28/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8", size = 1731969, upload-time = "2026-03-31T21:57:22.006Z" }, - { url = "https://files.pythonhosted.org/packages/6d/c5/705a3929149865fc941bcbdd1047b238e4a72bcb215a9b16b9d7a2e8d992/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d", size = 1795193, upload-time = "2026-03-31T21:57:24.256Z" }, - { url = "https://files.pythonhosted.org/packages/a6/19/edabed62f718d02cff7231ca0db4ef1c72504235bc467f7b67adb1679f48/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c", size = 1606477, upload-time = "2026-03-31T21:57:26.364Z" }, - { url = "https://files.pythonhosted.org/packages/de/fc/76f80ef008675637d88d0b21584596dc27410a990b0918cb1e5776545b5b/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac", size = 1813198, upload-time = "2026-03-31T21:57:28.316Z" }, - { url = "https://files.pythonhosted.org/packages/e5/67/5b3ac26b80adb20ea541c487f73730dc8fa107d632c998f25bbbab98fcda/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3", size = 1752321, upload-time = "2026-03-31T21:57:30.549Z" }, - { url = "https://files.pythonhosted.org/packages/88/06/e4a2e49255ea23fa4feeb5ab092d90240d927c15e47b5b5c48dff5a9ce29/aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06", size = 439069, upload-time = "2026-03-31T21:57:32.388Z" }, - { url = "https://files.pythonhosted.org/packages/c0/43/8c7163a596dab4f8be12c190cf467a1e07e4734cf90eebb39f7f5d53fc6a/aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8", size = 462859, upload-time = "2026-03-31T21:57:34.455Z" }, - { url = "https://files.pythonhosted.org/packages/be/6f/353954c29e7dcce7cf00280a02c75f30e133c00793c7a2ed3776d7b2f426/aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9", size = 748876, upload-time = "2026-03-31T21:57:36.319Z" }, - { url = "https://files.pythonhosted.org/packages/f5/1b/428a7c64687b3b2e9cd293186695affc0e1e54a445d0361743b231f11066/aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416", size = 499557, upload-time = "2026-03-31T21:57:38.236Z" }, - { url = "https://files.pythonhosted.org/packages/29/47/7be41556bfbb6917069d6a6634bb7dd5e163ba445b783a90d40f5ac7e3a7/aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2", size = 500258, upload-time = "2026-03-31T21:57:39.923Z" }, - { url = "https://files.pythonhosted.org/packages/67/84/c9ecc5828cb0b3695856c07c0a6817a99d51e2473400f705275a2b3d9239/aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4", size = 1749199, upload-time = "2026-03-31T21:57:41.938Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d3/3c6d610e66b495657622edb6ae7c7fd31b2e9086b4ec50b47897ad6042a9/aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9", size = 1721013, upload-time = "2026-03-31T21:57:43.904Z" }, - { url = "https://files.pythonhosted.org/packages/49/a0/24409c12217456df0bae7babe3b014e460b0b38a8e60753d6cb339f6556d/aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5", size = 1781501, upload-time = "2026-03-31T21:57:46.285Z" }, - { url = "https://files.pythonhosted.org/packages/98/9d/b65ec649adc5bccc008b0957a9a9c691070aeac4e41cea18559fef49958b/aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e", size = 1878981, upload-time = "2026-03-31T21:57:48.734Z" }, - { url = "https://files.pythonhosted.org/packages/57/d8/8d44036d7eb7b6a8ec4c5494ea0c8c8b94fbc0ed3991c1a7adf230df03bf/aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1", size = 1767934, upload-time = "2026-03-31T21:57:51.171Z" }, - { url = "https://files.pythonhosted.org/packages/31/04/d3f8211f273356f158e3464e9e45484d3fb8c4ce5eb2f6fe9405c3273983/aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286", size = 1566671, upload-time = "2026-03-31T21:57:53.326Z" }, - { url = "https://files.pythonhosted.org/packages/41/db/073e4ebe00b78e2dfcacff734291651729a62953b48933d765dc513bf798/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9", size = 1705219, upload-time = "2026-03-31T21:57:55.385Z" }, - { url = "https://files.pythonhosted.org/packages/48/45/7dfba71a2f9fd97b15c95c06819de7eb38113d2cdb6319669195a7d64270/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88", size = 1743049, upload-time = "2026-03-31T21:57:57.341Z" }, - { url = "https://files.pythonhosted.org/packages/18/71/901db0061e0f717d226386a7f471bb59b19566f2cae5f0d93874b017271f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3", size = 1749557, upload-time = "2026-03-31T21:57:59.626Z" }, - { url = "https://files.pythonhosted.org/packages/08/d5/41eebd16066e59cd43728fe74bce953d7402f2b4ddfdfef2c0e9f17ca274/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b", size = 1558931, upload-time = "2026-03-31T21:58:01.972Z" }, - { url = "https://files.pythonhosted.org/packages/30/e6/4a799798bf05740e66c3a1161079bda7a3dd8e22ca392481d7a7f9af82a6/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe", size = 1774125, upload-time = "2026-03-31T21:58:04.007Z" }, - { url = "https://files.pythonhosted.org/packages/84/63/7749337c90f92bc2cb18f9560d67aa6258c7060d1397d21529b8004fcf6f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14", size = 1732427, upload-time = "2026-03-31T21:58:06.337Z" }, - { url = "https://files.pythonhosted.org/packages/98/de/cf2f44ff98d307e72fb97d5f5bbae3bfcb442f0ea9790c0bf5c5c2331404/aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3", size = 433534, upload-time = "2026-03-31T21:58:08.712Z" }, - { url = "https://files.pythonhosted.org/packages/aa/ca/eadf6f9c8fa5e31d40993e3db153fb5ed0b11008ad5d9de98a95045bed84/aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1", size = 460446, upload-time = "2026-03-31T21:58:10.945Z" }, - { url = "https://files.pythonhosted.org/packages/78/e9/d76bf503005709e390122d34e15256b88f7008e246c4bdbe915cd4f1adce/aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61", size = 742930, upload-time = "2026-03-31T21:58:13.155Z" }, - { url = "https://files.pythonhosted.org/packages/57/00/4b7b70223deaebd9bb85984d01a764b0d7bd6526fcdc73cca83bcbe7243e/aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832", size = 496927, upload-time = "2026-03-31T21:58:15.073Z" }, - { url = "https://files.pythonhosted.org/packages/9c/f5/0fb20fb49f8efdcdce6cd8127604ad2c503e754a8f139f5e02b01626523f/aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9", size = 497141, upload-time = "2026-03-31T21:58:17.009Z" }, - { url = "https://files.pythonhosted.org/packages/3b/86/b7c870053e36a94e8951b803cb5b909bfbc9b90ca941527f5fcafbf6b0fa/aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090", size = 1732476, upload-time = "2026-03-31T21:58:18.925Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e5/4e161f84f98d80c03a238671b4136e6530453d65262867d989bbe78244d0/aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b", size = 1706507, upload-time = "2026-03-31T21:58:21.094Z" }, - { url = "https://files.pythonhosted.org/packages/d4/56/ea11a9f01518bd5a2a2fcee869d248c4b8a0cfa0bb13401574fa31adf4d4/aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a", size = 1773465, upload-time = "2026-03-31T21:58:23.159Z" }, - { url = "https://files.pythonhosted.org/packages/eb/40/333ca27fb74b0383f17c90570c748f7582501507307350a79d9f9f3c6eb1/aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8", size = 1873523, upload-time = "2026-03-31T21:58:25.59Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d2/e2f77eef1acb7111405433c707dc735e63f67a56e176e72e9e7a2cd3f493/aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665", size = 1754113, upload-time = "2026-03-31T21:58:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/fb/56/3f653d7f53c89669301ec9e42c95233e2a0c0a6dd051269e6e678db4fdb0/aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540", size = 1562351, upload-time = "2026-03-31T21:58:29.918Z" }, - { url = "https://files.pythonhosted.org/packages/ec/a6/9b3e91eb8ae791cce4ee736da02211c85c6f835f1bdfac0594a8a3b7018c/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb", size = 1693205, upload-time = "2026-03-31T21:58:32.214Z" }, - { url = "https://files.pythonhosted.org/packages/98/fc/bfb437a99a2fcebd6b6eaec609571954de2ed424f01c352f4b5504371dd3/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46", size = 1730618, upload-time = "2026-03-31T21:58:34.728Z" }, - { url = "https://files.pythonhosted.org/packages/e4/b6/c8534862126191a034f68153194c389addc285a0f1347d85096d349bbc15/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8", size = 1745185, upload-time = "2026-03-31T21:58:36.909Z" }, - { url = "https://files.pythonhosted.org/packages/0b/93/4ca8ee2ef5236e2707e0fd5fecb10ce214aee1ff4ab307af9c558bda3b37/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d", size = 1557311, upload-time = "2026-03-31T21:58:39.38Z" }, - { url = "https://files.pythonhosted.org/packages/57/ae/76177b15f18c5f5d094f19901d284025db28eccc5ae374d1d254181d33f4/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6", size = 1773147, upload-time = "2026-03-31T21:58:41.476Z" }, - { url = "https://files.pythonhosted.org/packages/01/a4/62f05a0a98d88af59d93b7fcac564e5f18f513cb7471696ac286db970d6a/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c", size = 1730356, upload-time = "2026-03-31T21:58:44.049Z" }, - { url = "https://files.pythonhosted.org/packages/e4/85/fc8601f59dfa8c9523808281f2da571f8b4699685f9809a228adcc90838d/aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc", size = 432637, upload-time = "2026-03-31T21:58:46.167Z" }, - { url = "https://files.pythonhosted.org/packages/c0/1b/ac685a8882896acf0f6b31d689e3792199cfe7aba37969fa91da63a7fa27/aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83", size = 458896, upload-time = "2026-03-31T21:58:48.119Z" }, + { url = "https://files.pythonhosted.org/packages/6d/67/58ded4b3f2e10f94972d8928050c85330e249a31dd45a0e5f3c0e9c3fa05/aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e", size = 766140, upload-time = "2026-06-07T21:05:37.471Z" }, + { url = "https://files.pythonhosted.org/packages/18/68/4ae5b4e08943f316594bb68da89957d3baf5760588fa09509594bd777e4b/aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491", size = 519430, upload-time = "2026-06-07T21:05:40.751Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c1/316c8f3549dbe5245f92bfd523ec6f32dd4d98cafe21df3f6a19b1184c75/aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce", size = 514406, upload-time = "2026-06-07T21:05:42.111Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ee/fb0ac28684e8d753b83c8a4eebc19a5846912aa0a4daaabb6a9936363840/aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3", size = 1703649, upload-time = "2026-06-07T21:05:43.427Z" }, + { url = "https://files.pythonhosted.org/packages/3b/57/aa2beab673331f111885db8a7b69dfe3ab0e53e446a0ace18ca694b4dc58/aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505", size = 1675126, upload-time = "2026-06-07T21:05:44.897Z" }, + { url = "https://files.pythonhosted.org/packages/47/ea/dad128abe365e79be03b16ed464198ac73e0d257e8260c6f7d6f31cbef26/aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521", size = 1771558, upload-time = "2026-06-07T21:05:46.405Z" }, + { url = "https://files.pythonhosted.org/packages/63/f3/b5b4e10327cb85d34d24232c6b71b64602f190b3ccb238a043ac6b187dac/aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd", size = 1856631, upload-time = "2026-06-07T21:05:47.844Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb", size = 1714139, upload-time = "2026-06-07T21:05:49.982Z" }, + { url = "https://files.pythonhosted.org/packages/29/c4/93067c85a0373492ce8e577435203c5947c454af074ac48ed4f3a1b9dd4a/aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42", size = 1588321, upload-time = "2026-06-07T21:05:51.431Z" }, + { url = "https://files.pythonhosted.org/packages/c4/39/9ff91aaf02af8b7b8222a987466da539f154c3e01732c22b5f5a20a8ee66/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b", size = 1670375, upload-time = "2026-06-07T21:05:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e4/77452a3676b8d99ac1375f77691d6bf65ea6e9f4b201b82ef77c916dc767/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192", size = 1690933, upload-time = "2026-06-07T21:05:54.902Z" }, + { url = "https://files.pythonhosted.org/packages/7d/84/b0059a7c7fc05ea23f3bc1596ba91c12f79588b9450564a24cac37536d0a/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05", size = 1740798, upload-time = "2026-06-07T21:05:56.458Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3a/e2a513ecbfc362591caa51a7f7e011b3bfc8938b388ae44cd95560d36999/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe", size = 1576412, upload-time = "2026-06-07T21:05:57.953Z" }, + { url = "https://files.pythonhosted.org/packages/a1/10/08f1654f538f93d36dcac66310a06eefce4641cdafca83f9f0a5317be254/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d", size = 1750199, upload-time = "2026-06-07T21:05:59.488Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/d91b70c57d8b8e9611e4a2e52238ca3698d3dc1c2efe25b7a9bf594ac584/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966", size = 1699356, upload-time = "2026-06-07T21:06:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f1/15340176f35ff61b95dbe34020bcf43f9e624a2d7bbac934715ff97d2033/aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6", size = 458939, upload-time = "2026-06-07T21:06:02.86Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/a2f1ec5b37f903109e43ae2862268cfe4a67a60c1b2cf43169fcdff5995f/aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df", size = 482583, upload-time = "2026-06-07T21:06:04.666Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7a/7b56f6732ef79530afaa72aa335d41b67c8d79b946995f0b11ad72985435/aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c", size = 453470, upload-time = "2026-06-07T21:06:06.322Z" }, + { url = "https://files.pythonhosted.org/packages/26/dd/bf526e6f0a1120dd6f2df2e97bacfe4d358f13d17a0ff5847301a1375a51/aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2", size = 765225, upload-time = "2026-06-07T21:06:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e1/a2872aa55495a70f61310d411541c6ee23812d9a884e000c716e1bc3edbf/aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f", size = 518743, upload-time = "2026-06-07T21:06:09.749Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e7/c60c7b209e509cc787de3cea0550a518538cfc08003e1c1e14c1c63fff71/aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8", size = 514139, upload-time = "2026-06-07T21:06:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8d/614ace2f579702c9840ab1e1447fd8509e35b0b904f7196418fa2f57b25d/aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04", size = 1784088, upload-time = "2026-06-07T21:06:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/726e90f99542bf292f81a96a12cc4847deb86f3ccf62c6f4014a201f4d33/aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8", size = 1737835, upload-time = "2026-06-07T21:06:14.564Z" }, + { url = "https://files.pythonhosted.org/packages/0b/4b/d176d5c4db9d33dacf0543102ea59503bc1d528af4cfd0b719949ca49389/aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6", size = 1842801, upload-time = "2026-06-07T21:06:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d6/5a99b563690ea0cbed912ae94a2ce33993a5709a651a3a4fe761e7dd973a/aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af", size = 1929992, upload-time = "2026-06-07T21:06:17.947Z" }, + { url = "https://files.pythonhosted.org/packages/76/7f/a987b14a3859094b3cea3f4825219c3e5536242564af6e3f9c2f6c994eb2/aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730", size = 1786989, upload-time = "2026-06-07T21:06:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/f1/1a/420e5c85a3e73349372ed22ce0b6af86bfa6ce16a4b20a64a2e94608c781/aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621", size = 1640129, upload-time = "2026-06-07T21:06:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/a7/80/18a592ed3be0a402cc03670bd72ee1f8563ddbe1d8d5542dbf868f274136/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee", size = 1756576, upload-time = "2026-06-07T21:06:24.8Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0b/8b3d5713373858ff71a617daf6e3b0e81ad63e79d09a3cf2f6b6b983939c/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573", size = 1754668, upload-time = "2026-06-07T21:06:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/9f/49/fd564575cf225821d7ba5a117cb8bc27213d8a7e1811162afb43ae077039/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7", size = 1817019, upload-time = "2026-06-07T21:06:28.297Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1b/e850c9ae6fc91356552ae668bb6c51e93fa29c8aef13398a10b56678557f/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf", size = 1631638, upload-time = "2026-06-07T21:06:30.242Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/3c337ba72451a89806ace6f75bddc92bafc5b8d53d90115a512858024b63/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85", size = 1835660, upload-time = "2026-06-07T21:06:31.943Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9c/9c18cf367a0498212d9ba7daf990b504a5e8ae064cda4b504e2647c89c03/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3", size = 1775698, upload-time = "2026-06-07T21:06:33.72Z" }, + { url = "https://files.pythonhosted.org/packages/b5/63/a251a9d2a6cb45065b2ddc0bde2b3dd10108740a9a42f632c66405a761a2/aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126", size = 458386, upload-time = "2026-06-07T21:06:35.279Z" }, + { url = "https://files.pythonhosted.org/packages/17/ca/69274c51dcd6e8947d77b2806cf47a4a15f2c846e2cbeb1882547d3da283/aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5", size = 483406, upload-time = "2026-06-07T21:06:36.824Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8a/c25904f77690c3688ec140f87591ef11a0cfe36bf3d5c0f1f38056fb62b3/aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b", size = 452987, upload-time = "2026-06-07T21:06:38.371Z" }, + { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, + { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, + { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, + { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, + { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, + { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, + { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, + { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, + { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, + { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, + { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, + { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, + { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, + { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, + { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, + { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, ] [[package]] @@ -1176,48 +1186,48 @@ wheels = [ [[package]] name = "cryptography" -version = "46.0.7" +version = "48.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652, upload-time = "2026-04-08T01:57:54.692Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/5d/4a8f770695d73be252331e60e526291e3df0c9b27556a90a6b47bccca4c2/cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4", size = 7179869, upload-time = "2026-04-08T01:56:17.157Z" }, - { url = "https://files.pythonhosted.org/packages/5f/45/6d80dc379b0bbc1f9d1e429f42e4cb9e1d319c7a8201beffd967c516ea01/cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325", size = 4275492, upload-time = "2026-04-08T01:56:19.36Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9a/1765afe9f572e239c3469f2cb429f3ba7b31878c893b246b4b2994ffe2fe/cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308", size = 4426670, upload-time = "2026-04-08T01:56:21.415Z" }, - { url = "https://files.pythonhosted.org/packages/8f/3e/af9246aaf23cd4ee060699adab1e47ced3f5f7e7a8ffdd339f817b446462/cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77", size = 4280275, upload-time = "2026-04-08T01:56:23.539Z" }, - { url = "https://files.pythonhosted.org/packages/0f/54/6bbbfc5efe86f9d71041827b793c24811a017c6ac0fd12883e4caa86b8ed/cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1", size = 4928402, upload-time = "2026-04-08T01:56:25.624Z" }, - { url = "https://files.pythonhosted.org/packages/2d/cf/054b9d8220f81509939599c8bdbc0c408dbd2bdd41688616a20731371fe0/cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef", size = 4459985, upload-time = "2026-04-08T01:56:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/f9/46/4e4e9c6040fb01c7467d47217d2f882daddeb8828f7df800cb806d8a2288/cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de", size = 3990652, upload-time = "2026-04-08T01:56:29.095Z" }, - { url = "https://files.pythonhosted.org/packages/36/5f/313586c3be5a2fbe87e4c9a254207b860155a8e1f3cca99f9910008e7d08/cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83", size = 4279805, upload-time = "2026-04-08T01:56:30.928Z" }, - { url = "https://files.pythonhosted.org/packages/69/33/60dfc4595f334a2082749673386a4d05e4f0cf4df8248e63b2c3437585f2/cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb", size = 4892883, upload-time = "2026-04-08T01:56:32.614Z" }, - { url = "https://files.pythonhosted.org/packages/c7/0b/333ddab4270c4f5b972f980adef4faa66951a4aaf646ca067af597f15563/cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b", size = 4459756, upload-time = "2026-04-08T01:56:34.306Z" }, - { url = "https://files.pythonhosted.org/packages/d2/14/633913398b43b75f1234834170947957c6b623d1701ffc7a9600da907e89/cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85", size = 4410244, upload-time = "2026-04-08T01:56:35.977Z" }, - { url = "https://files.pythonhosted.org/packages/10/f2/19ceb3b3dc14009373432af0c13f46aa08e3ce334ec6eff13492e1812ccd/cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e", size = 4674868, upload-time = "2026-04-08T01:56:38.034Z" }, - { url = "https://files.pythonhosted.org/packages/1a/bb/a5c213c19ee94b15dfccc48f363738633a493812687f5567addbcbba9f6f/cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457", size = 3026504, upload-time = "2026-04-08T01:56:39.666Z" }, - { url = "https://files.pythonhosted.org/packages/2b/02/7788f9fefa1d060ca68717c3901ae7fffa21ee087a90b7f23c7a603c32ae/cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b", size = 3488363, upload-time = "2026-04-08T01:56:41.893Z" }, - { url = "https://files.pythonhosted.org/packages/a7/7f/cd42fc3614386bc0c12f0cb3c4ae1fc2bbca5c9662dfed031514911d513d/cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4", size = 7165618, upload-time = "2026-04-08T01:57:10.645Z" }, - { url = "https://files.pythonhosted.org/packages/a5/d0/36a49f0262d2319139d2829f773f1b97ef8aef7f97e6e5bd21455e5a8fb5/cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7", size = 4270628, upload-time = "2026-04-08T01:57:12.885Z" }, - { url = "https://files.pythonhosted.org/packages/8a/6c/1a42450f464dda6ffbe578a911f773e54dd48c10f9895a23a7e88b3e7db5/cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832", size = 4415405, upload-time = "2026-04-08T01:57:14.923Z" }, - { url = "https://files.pythonhosted.org/packages/9a/92/4ed714dbe93a066dc1f4b4581a464d2d7dbec9046f7c8b7016f5286329e2/cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163", size = 4272715, upload-time = "2026-04-08T01:57:16.638Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e6/a26b84096eddd51494bba19111f8fffe976f6a09f132706f8f1bf03f51f7/cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2", size = 4918400, upload-time = "2026-04-08T01:57:19.021Z" }, - { url = "https://files.pythonhosted.org/packages/c7/08/ffd537b605568a148543ac3c2b239708ae0bd635064bab41359252ef88ed/cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067", size = 4450634, upload-time = "2026-04-08T01:57:21.185Z" }, - { url = "https://files.pythonhosted.org/packages/16/01/0cd51dd86ab5b9befe0d031e276510491976c3a80e9f6e31810cce46c4ad/cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0", size = 3985233, upload-time = "2026-04-08T01:57:22.862Z" }, - { url = "https://files.pythonhosted.org/packages/92/49/819d6ed3a7d9349c2939f81b500a738cb733ab62fbecdbc1e38e83d45e12/cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba", size = 4271955, upload-time = "2026-04-08T01:57:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/80/07/ad9b3c56ebb95ed2473d46df0847357e01583f4c52a85754d1a55e29e4d0/cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006", size = 4879888, upload-time = "2026-04-08T01:57:26.88Z" }, - { url = "https://files.pythonhosted.org/packages/b8/c7/201d3d58f30c4c2bdbe9b03844c291feb77c20511cc3586daf7edc12a47b/cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0", size = 4449961, upload-time = "2026-04-08T01:57:29.068Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ef/649750cbf96f3033c3c976e112265c33906f8e462291a33d77f90356548c/cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85", size = 4401696, upload-time = "2026-04-08T01:57:31.029Z" }, - { url = "https://files.pythonhosted.org/packages/41/52/a8908dcb1a389a459a29008c29966c1d552588d4ae6d43f3a1a4512e0ebe/cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e", size = 4664256, upload-time = "2026-04-08T01:57:33.144Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fa/f0ab06238e899cc3fb332623f337a7364f36f4bb3f2534c2bb95a35b132c/cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246", size = 3013001, upload-time = "2026-04-08T01:57:34.933Z" }, - { url = "https://files.pythonhosted.org/packages/d2/f1/00ce3bde3ca542d1acd8f8cfa38e446840945aa6363f9b74746394b14127/cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3", size = 3472985, upload-time = "2026-04-08T01:57:36.714Z" }, - { url = "https://files.pythonhosted.org/packages/63/0c/dca8abb64e7ca4f6b2978769f6fea5ad06686a190cec381f0a796fdcaaba/cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f", size = 3476879, upload-time = "2026-04-08T01:57:38.664Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ea/075aac6a84b7c271578d81a2f9968acb6e273002408729f2ddff517fed4a/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15", size = 4219700, upload-time = "2026-04-08T01:57:40.625Z" }, - { url = "https://files.pythonhosted.org/packages/6c/7b/1c55db7242b5e5612b29fc7a630e91ee7a6e3c8e7bf5406d22e206875fbd/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455", size = 4385982, upload-time = "2026-04-08T01:57:42.725Z" }, - { url = "https://files.pythonhosted.org/packages/cb/da/9870eec4b69c63ef5925bf7d8342b7e13bc2ee3d47791461c4e49ca212f4/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65", size = 4219115, upload-time = "2026-04-08T01:57:44.939Z" }, - { url = "https://files.pythonhosted.org/packages/f4/72/05aa5832b82dd341969e9a734d1812a6aadb088d9eb6f0430fc337cc5a8f/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968", size = 4385479, upload-time = "2026-04-08T01:57:46.86Z" }, - { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829, upload-time = "2026-04-08T01:57:48.874Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, + { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, + { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, + { url = "https://files.pythonhosted.org/packages/89/c6/24266ac10c47f6cd2a865f4446062b466da1d1f10b27189eac00e61bf0c9/cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", size = 5300085, upload-time = "2026-06-09T22:31:58.703Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bb/cc4b78784f97efc8c5874c2a9743708d172be6663024b34a0467885ae0c8/cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", size = 4746137, upload-time = "2026-06-09T22:31:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/1f/52/0c44de3f5267f8fbe8e835138017522a333436166e406f0db9b9e6e3033f/cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", size = 4333867, upload-time = "2026-06-09T22:32:28.096Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/772d7adbfa931537bc401640b7cac9976bff689bda187833e5d63b428e49/cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", size = 4701805, upload-time = "2026-06-09T22:31:38.284Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a3/b06844f303873493c963caf581c04df31c7035e0c1b0f02c4814d319ec80/cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", size = 5258461, upload-time = "2026-06-09T22:31:04.187Z" }, + { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, + { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, + { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, + { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, + { url = "https://files.pythonhosted.org/packages/e5/0b/aa68b221dde92d09cb29a024ede17550ee21e77a404e59fc093c82bb51e1/cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", size = 5289970, upload-time = "2026-06-09T22:31:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/78/13/fba657f958d2af66ea959a4ba01212632089249d34af1ae48054136344d7/cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", size = 4731873, upload-time = "2026-06-09T22:31:22.253Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4c/9a964756d24a26b3e34dfcb16f961b89838786e6700b635b0d1e3adff4b6/cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", size = 4330804, upload-time = "2026-06-09T22:31:36.56Z" }, + { url = "https://files.pythonhosted.org/packages/4b/0f/a10f3a6eb12950a10e3a874070283aa2dd5875b2bfd15fad8a3e17b3f13e/cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46", size = 4696217, upload-time = "2026-06-09T22:31:13.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6f/5cd12f951165ea73ef85266775d97e4c763b2474ccfd816dd69d3a18d6f8/cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", size = 5245252, upload-time = "2026-06-09T22:32:02.193Z" }, + { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, + { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, + { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, + { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, + { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, + { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, ] [[package]] @@ -1489,6 +1499,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "expression" +version = "5.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/c7/bb061623b5815566bda69f5e9d156e38a97ebb383b8db3d2dedb26415466/expression-5.6.0.tar.gz", hash = "sha256:454f6fe138347194a43c7f878d958efe9b84b9cc770e462010c7a52e18058065", size = 59147, upload-time = "2025-02-19T09:37:37.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/a2/656b8bebe495117342a8676ccabf52b3885ce11a856c8dfe1fbbdc250d2d/expression-5.6.0-py3-none-any.whl", hash = "sha256:f5c62e38186c9287e088dee9cf3939b0bbde21cb4c59571872154a53d33dd7c0", size = 69673, upload-time = "2025-02-19T09:37:35.476Z" }, +] + [[package]] name = "fakeredis" version = "2.34.1" @@ -2943,16 +2965,16 @@ wheels = [ [[package]] name = "langchain" -version = "1.2.10" +version = "1.3.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/22/a4d4ac98fc2e393537130bbfba0d71a8113e6f884d96f935923e247397fe/langchain-1.2.10.tar.gz", hash = "sha256:bdcd7218d9c79a413cf15e106e4eb94408ac0963df9333ccd095b9ed43bf3be7", size = 570071, upload-time = "2026-02-10T14:56:49.74Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/7c/651d0dc4913a7a892156c03dd343b99cfe19ee729e6911ab1f4fe7567b8b/langchain-1.3.9.tar.gz", hash = "sha256:9b14ef0db9ef314299ded858b22ca2a40b8f1b05c8c9cb6b82d53a53075fef00", size = 631514, upload-time = "2026-06-12T16:53:27.083Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/06/c3394327f815fade875724c0f6cff529777c96a1e17fea066deb997f8cf5/langchain-1.2.10-py3-none-any.whl", hash = "sha256:e07a377204451fffaed88276b8193e894893b1003e25c5bca6539288ccca3698", size = 111738, upload-time = "2026-02-10T14:56:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/3481619d21b9bdfbfda8680fba5cfc6cfe926789b8eaaad95353078cfa20/langchain-1.3.9-py3-none-any.whl", hash = "sha256:4af49ad1095799e4408b489fb79d4b8b49292453618b202d8a697fca59bb6871", size = 132873, upload-time = "2026-06-12T16:53:25.489Z" }, ] [[package]] @@ -3030,7 +3052,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.4.0" +version = "1.4.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -3043,9 +3065,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/59/de/679a53472c25860837e32c0442c962fa86e95317a36460e2c9d5c91b17c2/langchain_core-1.4.0.tar.gz", hash = "sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f", size = 920260, upload-time = "2026-05-11T18:42:35.919Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/e3/bea6d0080acf183332f24dcd74c208aee5857cf8f783c3fb0bd86027d8fb/langchain_core-1.4.8.tar.gz", hash = "sha256:5bf1f8411077c904182ad8f975943d36adcbf579c4e017b3a118b719229ebf9a", size = 957974, upload-time = "2026-06-18T19:39:23.636Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/1a/86c38c27b81913a1c6c12448cab55defb5a1097c7dc9a4cea83f55477a2d/langchain_core-1.4.0-py3-none-any.whl", hash = "sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c", size = 548120, upload-time = "2026-05-11T18:42:33.992Z" }, + { url = "https://files.pythonhosted.org/packages/13/d6/bdf6f0481cc57ef300d6b1eb48cf1400c0409be715d6eb3cabadd1142a09/langchain_core-1.4.8-py3-none-any.whl", hash = "sha256:d84c28b05e3ba8d4271d0827aad5b592ccdaaf986e76768c23503f0a2045e8aa", size = 557416, upload-time = "2026-06-18T19:39:21.902Z" }, ] [[package]] @@ -3078,14 +3100,14 @@ wheels = [ [[package]] name = "langchain-protocol" -version = "0.0.15" +version = "0.0.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4f/24/9777489d6fbbee64af0c8f96d4f840239c408cf694f3394672807dafc490/langchain_protocol-0.0.15.tar.gz", hash = "sha256:9ab2d11ee73944754f10e037e717098d3a6796f0e58afa9cadda6154e7655ade", size = 5862, upload-time = "2026-05-01T22:30:04.748Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/59/b5959aea96faa9146e2e49a7a22882b3528c62efafe9a6a95beab30c2305/langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6", size = 6150, upload-time = "2026-06-18T17:08:26.959Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/7a/9c97a7b9cbe4c5dc6a44cdb1545450c28f0c8ce89b9c1f0ee7fbad896263/langchain_protocol-0.0.15-py3-none-any.whl", hash = "sha256:461eb794358f83d5e42635a5797799ffec7b4702314e34edf73ac21e75d3ef79", size = 6982, upload-time = "2026-05-01T22:30:03.877Z" }, + { url = "https://files.pythonhosted.org/packages/99/2e/d82db9eec13ad0f72e7aaad5c4bc730ab111934fdc83c85523206eb9b0a0/langchain_protocol-0.0.18-py3-none-any.whl", hash = "sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a", size = 7221, upload-time = "2026-06-18T17:08:25.996Z" }, ] [[package]] @@ -3121,7 +3143,7 @@ wheels = [ [[package]] name = "langgraph" -version = "1.0.10" +version = "1.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -3131,9 +3153,9 @@ dependencies = [ { name = "pydantic" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/92/14df6fefba28c10caf1cb05aa5b8c7bf005838fe32a86d903b6c7cc4018d/langgraph-1.0.10.tar.gz", hash = "sha256:73bd10ee14a8020f31ef07e9cd4c1a70c35cc07b9c2b9cd637509a10d9d51e29", size = 511644, upload-time = "2026-02-27T21:04:38.743Z" } +sdist = { url = "https://files.pythonhosted.org/packages/02/7a/ea09b05bb0cbddfa43bd34fc581357e87fc3f21a751cc0d419688c3106da/langgraph-1.2.6.tar.gz", hash = "sha256:f9b45a34f13930c94d96cdb76277447ad2cc70ec2d18cd2764d7fdadb36cdc1b", size = 714400, upload-time = "2026-06-18T20:58:21.514Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/60/260e0c04620a37ba8916b712766c341cc5fc685dabc6948c899494bbc2ae/langgraph-1.0.10-py3-none-any.whl", hash = "sha256:7c298bef4f6ea292fcf9824d6088fe41a6727e2904ad6066f240c4095af12247", size = 160920, upload-time = "2026-02-27T21:04:35.932Z" }, + { url = "https://files.pythonhosted.org/packages/89/32/772db1b00a9fe42f50320d1aa20caefb76e621eff1f7218b9918093d631d/langgraph-1.2.6-py3-none-any.whl", hash = "sha256:1cf94d3ca124f84f77ce408fa1b06c3dee680a8aafffe364a8fd5d7d03eb8695", size = 246132, upload-time = "2026-06-18T20:58:20.335Z" }, ] [[package]] @@ -3151,33 +3173,36 @@ wheels = [ [[package]] name = "langgraph-prebuilt" -version = "1.0.8" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph-checkpoint" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0d/06/dd61a5c2dce009d1b03b1d56f2a85b3127659fdddf5b3be5d8f1d60820fb/langgraph_prebuilt-1.0.8.tar.gz", hash = "sha256:0cd3cf5473ced8a6cd687cc5294e08d3de57529d8dd14fdc6ae4899549efcf69", size = 164442, upload-time = "2026-02-19T18:14:39.083Z" } +sdist = { url = "https://files.pythonhosted.org/packages/29/66/ed9b93f56bc17ef22d551892f0ac2b225a97fe0fcf23a511b857f70d590b/langgraph_prebuilt-1.1.0.tar.gz", hash = "sha256:3c579cf6eed2d17f9c157c2d0fcaddcd8688524e7022d3b22b37a3bf4589d528", size = 178833, upload-time = "2026-05-12T03:37:49.332Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/41/ec966424ad3f2ed3996d24079d3342c8cd6c0bd0653c12b2a917a685ec6c/langgraph_prebuilt-1.0.8-py3-none-any.whl", hash = "sha256:d16a731e591ba4470f3e313a319c7eee7dbc40895bcf15c821f985a3522a7ce0", size = 35648, upload-time = "2026-02-19T18:14:37.611Z" }, + { url = "https://files.pythonhosted.org/packages/e9/43/3fe1a700b8490ed02679cdbbc8c915eb23a092faf496c9c1118abcd10be3/langgraph_prebuilt-1.1.0-py3-none-any.whl", hash = "sha256:51e311747d755b751d5c6b39b0c1446124d3a7643d2515017e6714b323508fc9", size = 41043, upload-time = "2026-05-12T03:37:48.007Z" }, ] [[package]] name = "langgraph-sdk" -version = "0.3.14" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, + { name = "langchain-core" }, + { name = "langchain-protocol" }, { name = "orjson" }, + { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/134046c20bc4a4a15d410d1d21c9e298a3e9923777b4cc867b8669bc636b/langgraph_sdk-0.3.14.tar.gz", hash = "sha256:acd1674c538e97f3cdaa610f6dd7e34bc9bad30167f0ccc482dcd563325e81f5", size = 198162, upload-time = "2026-05-05T18:40:03.524Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/2b/bd8ac26d4e97f6df88ef05ce5b6a38945a3903e1025d926f4752aa88aa97/langgraph_sdk-0.4.2.tar.gz", hash = "sha256:b88f0f5f6328ac0680d6790614a905b2bcfa257f2276dba4e38f0e86db0aa738", size = 348327, upload-time = "2026-06-01T17:51:19.856Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/96/1c9f9fbfe756ddd850a2585e7f1949d8ebb97fdaa7a5eff8f45ed1314670/langgraph_sdk-0.3.14-py3-none-any.whl", hash = "sha256:68935bf6f4924eda92617a9e5dfb4f4281197508c648cb9d62ff083907607f9d", size = 97028, upload-time = "2026-05-05T18:40:02.099Z" }, + { url = "https://files.pythonhosted.org/packages/a0/05/aac507337cceae773c2cc9ab91eb6301963af7aeeb55b4217a00e15aff17/langgraph_sdk-0.4.2-py3-none-any.whl", hash = "sha256:75fa5096c1177ce39c847096a8fe3745ffd480ddb412995f836e9f5f884c43dd", size = 160521, upload-time = "2026-06-01T17:51:18.849Z" }, ] [[package]] name = "langsmith" -version = "0.8.3" +version = "0.8.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -3187,12 +3212,13 @@ dependencies = [ { name = "requests" }, { name = "requests-toolbelt" }, { name = "uuid-utils" }, + { name = "websockets" }, { name = "xxhash" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/8a/1e8ea5e8bab2a65fa95bd36229ef38e8723ec46e430e20ca2d953487a7f1/langsmith-0.8.3.tar.gz", hash = "sha256:767ff7a8d136ed42926bf99059ac631dc6883542d6e3104b32e71c7625e1fa05", size = 4460330, upload-time = "2026-05-07T19:56:56.18Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/d9/a6681aa9847bbbc5ec21abe20a5e233b94e5edcfe39624db607ac7e8ccb4/langsmith-0.8.18.tar.gz", hash = "sha256:32dde9c0e67e053e0fb738921fc8ced768af7b8fa83d7a0e3fd63597cf8776dd", size = 4526988, upload-time = "2026-06-19T13:12:17.123Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/a9/51e644c1f1dbc3dd7d22dfd6412eab206d538c81e024e4f287373544bdcb/langsmith-0.8.3-py3-none-any.whl", hash = "sha256:b2e40e308222fa0beb2dccee3b4b30bfee9062d7a4f20a3e3e93df3c51a08ab4", size = 399048, upload-time = "2026-05-07T19:56:53.994Z" }, + { url = "https://files.pythonhosted.org/packages/03/70/0e0cc80a3b064c8d6c8d697c3125ed86e39d5a7393ec6dc8b07cb1cf13c4/langsmith-0.8.18-py3-none-any.whl", hash = "sha256:3940183349993faef48e6c7d08e4822ee9cefd906b362d0e3c2d650314d2f282", size = 508108, upload-time = "2026-06-19T13:12:15.348Z" }, ] [[package]] @@ -3245,7 +3271,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.90.0" +version = "1.91.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -3297,6 +3323,7 @@ proxy = [ { name = "backoff" }, { name = "boto3" }, { name = "cryptography" }, + { name = "expression" }, { name = "fastapi" }, { name = "fastapi-sso" }, { name = "granian" }, @@ -3456,10 +3483,11 @@ requires-dist = [ { name = "backoff", marker = "extra == 'proxy'", specifier = ">=2.2.1,<3.0" }, { name = "boto3", marker = "extra == 'proxy'", specifier = ">=1.43.1,<2.0" }, { name = "click", specifier = ">=8.0.0,<9.0" }, - { name = "cryptography", marker = "extra == 'proxy'", specifier = ">=46.0.7,<47.0" }, + { name = "cryptography", marker = "extra == 'proxy'", specifier = ">=48.0.1,<49.0" }, { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = ">=2.19.0,<3.0" }, { name = "detect-secrets", marker = "extra == 'proxy-runtime'", specifier = ">=1.5.0,<2.0" }, { name = "diskcache", marker = "extra == 'caching'", specifier = ">=5.6.3,<6.0" }, + { name = "expression", marker = "extra == 'proxy'", specifier = ">=5.6.0,<6.0" }, { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, @@ -3537,11 +3565,11 @@ ci = [ { name = "detect-secrets", specifier = "==1.5.0" }, { name = "google-generativeai", specifier = "==0.8.6" }, { name = "jsonlines", specifier = "==4.0.0" }, - { name = "langchain", specifier = "==1.2.10" }, + { name = "langchain", specifier = "==1.3.9" }, { name = "langchain-mcp-adapters", specifier = "==0.2.1" }, { name = "langchain-openai", specifier = "==1.1.14" }, - { name = "langgraph", specifier = "==1.0.10" }, - { name = "langgraph-prebuilt", specifier = "==1.0.8" }, + { name = "langgraph", specifier = ">=1.2.4,<1.3.0" }, + { name = "langgraph-prebuilt", specifier = ">=1.1.0,<1.3.0" }, { name = "logfire", specifier = "==4.6.0" }, { name = "lunary", marker = "python_full_version == '3.10.*'", specifier = "==1.4.36" }, { name = "lunary", marker = "python_full_version >= '3.11'", specifier = "==1.4.37" }, @@ -3590,7 +3618,7 @@ dev = [ { name = "types-redis", specifier = "==4.6.0.20241004" }, { name = "types-requests", specifier = "==2.32.4.20260107" }, { name = "types-setuptools", specifier = "==75.8.0.20250225" }, - { name = "vcrpy", specifier = "==8.1.1" }, + { name = "vcrpy", specifier = "==8.2.1" }, ] healthcheck = [ { name = "httpx", specifier = "==0.28.1" }, @@ -3952,7 +3980,7 @@ wheels = [ [[package]] name = "mlflow" -version = "3.11.1" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -3979,14 +4007,14 @@ dependencies = [ { name = "sqlalchemy" }, { name = "waitress", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/34/e328c073cd32c186fb242a957e5bade82433c06bc45b7d1695bf4d02f166/mlflow-3.11.1.tar.gz", hash = "sha256:84e54c4be91b5b2a19039a2673fe688b1d7307ceddacc08af51f8df05b19ee56", size = 9797469, upload-time = "2026-04-07T14:26:58.463Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/0b/3404a057daceffe9ce18cd08868648a1e9b817270177bdf8a764576b988b/mlflow-3.14.0.tar.gz", hash = "sha256:5a1f818fa003035c724162096ce3ded7bc7bc47a1cae595df6173961983f4718", size = 11792369, upload-time = "2026-06-17T07:57:44.712Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/62/96826c340354638dfedcbdbcd35d67754566bd45f6592300e0c215c80e30/mlflow-3.11.1-py3-none-any.whl", hash = "sha256:8f6bf1238ac04f97664c229dd480380c5c254a78bdb3c0e433e3a0397508b1af", size = 10479141, upload-time = "2026-04-07T14:26:55.709Z" }, + { url = "https://files.pythonhosted.org/packages/de/b9/76dcdef7f7f856b36f18cfcd752c2717d9847812a0aaa36d50a7baed569d/mlflow-3.14.0-py3-none-any.whl", hash = "sha256:dbf77f7cdb5b5c0ec59b4671c61730b1b914b4dff7a2892e267a547cb5454f56", size = 12564161, upload-time = "2026-06-17T07:57:42.348Z" }, ] [[package]] name = "mlflow-skinny" -version = "3.11.1" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -4006,17 +4034,18 @@ dependencies = [ { name = "pyyaml" }, { name = "requests" }, { name = "sqlparse" }, + { name = "starlette" }, { name = "typing-extensions" }, { name = "uvicorn" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/77/fe2027ddad9e52ed1ac360fbc262169e6366f6678632e350cbd0d901bb9b/mlflow_skinny-3.11.1.tar.gz", hash = "sha256:86ce63491349f6713afc8a4ef0bf77a8314d0e79e03753cb150d6c860a0b0475", size = 2642799, upload-time = "2026-04-07T14:26:43.818Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/4f/a054cd8860590e4e942aee1aab3c94307878159f945fa844acc9ea787721/mlflow_skinny-3.14.0.tar.gz", hash = "sha256:e50f4506422c7737157ae6643c165122af7898345f2e828fa93c4f10128653cf", size = 2901772, upload-time = "2026-06-17T07:57:44.252Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/a7/e61ec397b34dc3c9e91572f45e41617f429d5c524d38a4e1aa2316ee1b5e/mlflow_skinny-3.11.1-py3-none-any.whl", hash = "sha256:82ffd5f6980320b4ac19f741e7a754faa1d01707e632b002ea68e04fd25a0535", size = 3171551, upload-time = "2026-04-07T14:26:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/58/e7/b80f76ce689b9d6f21cdb84abb2b02148a1149e63a6428dd2c629cefd061/mlflow_skinny-3.14.0-py3-none-any.whl", hash = "sha256:a4880e086365871ef9d78e727a34ea5fb1ce615689579998d48e8c65ee1665a9", size = 3462788, upload-time = "2026-06-17T07:57:42.583Z" }, ] [[package]] name = "mlflow-tracing" -version = "3.11.1" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -4028,9 +4057,9 @@ dependencies = [ { name = "protobuf" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1b/77/73af163432f3c66e2d213045250972e504a6683c76f63dd1abfba441a16a/mlflow_tracing-3.11.1.tar.gz", hash = "sha256:cb63cee16385d081467ec5bee4807fe1af59ddfdf04be4c79e7a7813b1002193", size = 1314550, upload-time = "2026-04-07T14:26:32.785Z" } +sdist = { url = "https://files.pythonhosted.org/packages/04/34/ff5e72919b4eec8fe65e6fc843978a1a512194e6fafbd1761deca48269ad/mlflow_tracing-3.14.0.tar.gz", hash = "sha256:c2f701e001d35964f23fbbdfdda36c818a76c157b912ae83781199fd714be09a", size = 1429017, upload-time = "2026-06-17T07:58:00.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/ab/d980c84e7df4224ab8db2457afbe135b430f371ca081a37cf89f8ef18ca1/mlflow_tracing-3.11.1-py3-none-any.whl", hash = "sha256:fa82df64dacf8293b714ae666440fe7c1902c6470c024df389bb91e9de3106d9", size = 1575790, upload-time = "2026-04-07T14:26:30.804Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4a/4658a9e514c8f079e40b608661844b9beb21c7530e6ee1e7f830cf81541e/mlflow_tracing-3.14.0-py3-none-any.whl", hash = "sha256:854488dd18068f15e2a56f1cc7b8868c611d09ea39068d0a691a3f07e0048cae", size = 1703863, upload-time = "2026-06-17T07:57:58.687Z" }, ] [[package]] @@ -5881,16 +5910,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.1" +version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] [[package]] @@ -6006,14 +6035,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.13.1" +version = "6.13.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/d9/9d12fa0d9660d03320725ff686c961b645a4218940a82296e1272d9e1ff0/pypdf-6.13.1.tar.gz", hash = "sha256:4841d8a4c1589e5833915dc0c7ddfacff80a2e0bcbeb5d1e681fecaa1674b03a", size = 6477811, upload-time = "2026-06-08T11:01:49.344Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/18/9947cc201af9ccf76720fd3347bf4f70eb882ce3fcf4cb05f7443e4cf871/pypdf-6.13.3.tar.gz", hash = "sha256:f3cb822769725f1bac658c406cfc9460399043f3750c2d3e4650e0a85eacabd7", size = 6484063, upload-time = "2026-06-17T15:22:00.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/dd/8f03e0a5788a5d1feb4550617c3e6db5e9099eaee248a3e482ddaeacbbb0/pypdf-6.13.1-py3-none-any.whl", hash = "sha256:e555e4ce3f561ef069307622f1374136ba964ca6ca24f24158701decaf83ed9b", size = 346259, upload-time = "2026-06-08T11:01:47.741Z" }, + { url = "https://files.pythonhosted.org/packages/94/56/2967e621598987905fb8cdfadd8f8de6b5c68c9351f0523c4df8409f28f1/pypdf-6.13.3-py3-none-any.whl", hash = "sha256:c6e3f86afb625791510b02ad5480e94b63970bb957df75d44657c282ecc52224", size = 347288, upload-time = "2026-06-17T15:21:59.512Z" }, ] [[package]] @@ -6220,11 +6249,11 @@ wheels = [ [[package]] name = "python-multipart" -version = "0.0.27" +version = "0.0.32" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/9b/f23807317a113dc36e74e75eb265a02dd1a4d9082abc3c1064acd22997c4/python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602", size = 44043, upload-time = "2026-04-27T10:51:26.649Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/78/4126abcbdbd3c559d43e0db7f7b9173fc6befe45d39a2856cc0b8ec2a5a6/python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645", size = 29254, upload-time = "2026-04-27T10:51:24.997Z" }, + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] [[package]] @@ -7344,15 +7373,15 @@ wheels = [ [[package]] name = "starlette" -version = "1.1.0" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/66/4d20cdf39a8d6a51e663b7038e3b828ff211d3891a43a713fe7e4643f3a8/starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f", size = 2660060, upload-time = "2026-05-23T16:55:41.376Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899, upload-time = "2026-05-23T16:55:39.201Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] @@ -8024,15 +8053,15 @@ wheels = [ [[package]] name = "vcrpy" -version = "8.1.1" +version = "8.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b3/07/bcfd5ebd7cb308026ab78a353e091bd699593358be49197d39d004e5ad83/vcrpy-8.1.1.tar.gz", hash = "sha256:58e3053e33b423f3594031cb758c3f4d1df931307f1e67928e30cf352df7709f", size = 85770, upload-time = "2026-01-04T19:22:03.886Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/db/08183b845b0040bb877dad2bd7e4e0976fc232bb3476d7ee369c6c4f8b5a/vcrpy-8.2.1.tar.gz", hash = "sha256:d73a6e4eb6dae8148e659764b7a00e68cc51ba29ba9e6a85e1f0790ad96b97df", size = 90511, upload-time = "2026-06-16T13:20:52.906Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/d7/f79b05a5d728f8786876a7d75dfb0c5cae27e428081b2d60152fb52f155f/vcrpy-8.1.1-py3-none-any.whl", hash = "sha256:2d16f31ad56493efb6165182dd99767207031b0da3f68b18f975545ede8ac4b9", size = 42445, upload-time = "2026-01-04T19:22:02.532Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/0e812ab83f5289404c674f3461ba783250b967d34b5ab034d361236ec042/vcrpy-8.2.1-py3-none-any.whl", hash = "sha256:7ce58c9e2792b246f79d6f4b3e9660676cc6f853be17e1547305b4437ab1ff85", size = 44925, upload-time = "2026-06-16T13:20:51.734Z" }, ] [[package]]