Merge pull request #31140 from BerriAI/litellm_internal_staging

chore(ci): promote internal staging to main

.github/workflows/osv-scan.yml

@@ -7,11 +7,6 @@ on:
       - litellm_internal_staging
       - litellm_oss_branch
       - "litellm_**"
-    paths:
-      - uv.lock
-      - ui/litellm-dashboard/package-lock.json
-      - osv-scanner.toml
-      - .github/workflows/osv-scan.yml
   schedule:
     - cron: "23 6 * * *"
   workflow_dispatch:

.github/workflows/test-linting.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -87,9 +87,11 @@ jobs:
         run: |
           uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
 
-      - name: Run basedpyright type checking
+      - name: Check basedpyright budget (delta vs base)
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py
+          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
 
       - name: Check for circular imports
         run: |

.github/workflows/test-rust.yml

@@ -0,0 +1,65 @@
+name: LiteLLM Rust
+
+on:
+  push:
+    paths:
+      - "litellm-rust/**"
+      - ".github/workflows/test-rust.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+    paths:
+      - "litellm-rust/**"
+      - ".github/workflows/test-rust.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  rust-checks:
+    name: rustfmt, clippy, test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: litellm-rust
+    env:
+      CARGO_TERM_COLOR: always
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Rust
+        run: |
+          rustup toolchain install stable --profile minimal --component clippy,rustfmt
+          rustup default stable
+
+      - name: Cache Cargo registry and target
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            litellm-rust/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
+
+      - name: Check Rust formatting
+        run: cargo fmt --check
+
+      - name: Run Clippy
+        run: cargo clippy --workspace --all-targets --locked -- -D warnings
+
+      - name: Run Rust tests
+        run: cargo test --workspace --locked

.github/workflows/test-unit-misc.yml

@@ -32,7 +32,9 @@ jobs:
         tests/test_litellm/repositories
         tests/test_litellm/images
         tests/test_litellm/interactions
+        tests/test_litellm/ocr
         tests/test_litellm/passthrough
+        tests/test_litellm/sandbox
         tests/test_litellm/vector_stores
         tests/test_litellm/test_*.py
       workers: 2

Makefile

@@ -125,7 +125,8 @@ lint-ruff-FULL-dev: install-dev
 	else echo "No changed .py files to check."; fi
 
 lint-basedpyright: install-dev
-	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py
+	git fetch origin litellm_internal_staging
+	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
 
 lint-basedpyright-budget-update: install-dev
 	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update

backend/routes/allowlist.py

@@ -120,6 +120,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/robots.txt",
     # Health (k8s probes)
     "/health",
+    # Plugin system
+    "/api/plugins",
+    "/plugin-proxy/",
 )
 
 BACKEND_EXACT_PATHS: frozenset[str] = frozenset(

basedpyright-code-budget.json

@@ -121,7 +121,7 @@
   },
   "reportReturnType": {
     "baseline": 126,
-    "slack": 13
+    "slack": 100
   },
   "reportTypedDictNotRequiredAccess": {
     "baseline": 20,
@@ -157,7 +157,7 @@
   },
   "reportUnnecessaryComparison": {
     "baseline": 683,
-    "slack": 10
+    "slack": 100
   },
   "reportUnnecessaryContains": {
     "baseline": 4,

docs/plugin_architecture.md

@@ -0,0 +1,141 @@
+# LiteLLM Plugin Architecture
+
+Plugins let external services appear as selectable modes in the litellm UI sidebar alongside the AI Gateway.
+
+---
+
+## Quick start
+
+### 1. Configure the plugin
+
+Add a `plugins` block to your litellm `config.yaml`:
+
+```yaml
+general_settings:
+  master_key: sk-...
+  plugins:
+    - name: my-plugin            # unique identifier (no spaces)
+      display_name: My Plugin    # shown in the UI dropdown
+      url: "https://my-plugin.example.com"
+      plugin_key: "sk-..."       # plugin's own auth credential
+```
+
+`plugin_key` is injected as `Authorization: Bearer <plugin_key>` on every
+request proxied through `/plugin-proxy/my-plugin/*`.  The caller's litellm
+credential is stripped before forwarding so the plugin never receives a live
+litellm API key.
+
+### 2. Implement two endpoints on your service
+
+| Endpoint | Method | Purpose |
+|---|---|---|
+| `GET /api/plugin-manifest` | public | Returns plugin metadata for the UI |
+| `POST /api/plugin-auth` | public | Decrypts the identity claim for seamless sign-in |
+
+#### `GET /api/plugin-manifest`
+
+```json
+{
+  "name": "my-plugin",
+  "display_name": "My Plugin",
+  "version": "1.0.0",
+  "nav_items": [
+    { "key": "home",    "label": "Home",    "icon": "HomeOutlined",    "path": "/" },
+    { "key": "reports", "label": "Reports", "icon": "BarChartOutlined", "path": "/reports" }
+  ],
+  "capabilities": ["reports", "data"]
+}
+```
+
+#### `POST /api/plugin-auth`
+
+Receives `{ "session_claim": "<fernet-ciphertext>" }`.
+
+The proxy never shares `LITELLM_SALT_KEY` with your plugin.  Each plugin is
+provisioned with its own dedicated key, derived as
+`HMAC-SHA256(LITELLM_SALT_KEY, plugin_name)`.  Compute it once on the proxy
+host and hand the result to your plugin as a secret (e.g. `PLUGIN_AUTH_KEY`):
+
+```bash
+python -c 'import base64,hmac,hashlib,os; \
+print(base64.urlsafe_b64encode(hmac.new(os.environ["LITELLM_SALT_KEY"].encode(), b"my-plugin", hashlib.sha256).digest()).decode())'
+```
+
+A compromised plugin holding only this scoped key cannot recover
+`LITELLM_SALT_KEY` or decrypt any other litellm secret.
+
+Decrypt and validate the claim with that key:
+
+```python
+import json, os, time
+from cryptography.fernet import Fernet
+
+_CLAIM_TTL_SECONDS = 30
+
+def plugin_auth(session_claim: str) -> dict:
+    cipher = Fernet(os.environ["PLUGIN_AUTH_KEY"].encode())
+    claim = json.loads(cipher.decrypt(session_claim.encode(), ttl=_CLAIM_TTL_SECONDS))
+    if claim.get("plugin") != "my-plugin":
+        raise ValueError("claim audience mismatch")
+    if int(claim.get("exp", 0)) < int(time.time()):
+        raise ValueError("claim expired")
+    return claim
+```
+
+The claim is `{ "plugin", "user_id", "user_role", "exp" }`; it carries no
+litellm bearer token.  Establish the plugin's own session from `user_id` /
+`user_role` and authenticate API calls back to litellm through the
+`/plugin-proxy/my-plugin/*` reverse proxy, which injects `plugin_key` for you.
+
+---
+
+## How iframe auth works
+
+```
+litellm UI
+  ├─ GET /api/plugins/auth-token        -> { session_claim }
+  └─ postMessage({ type:"litellm-auth", session_claim }, pluginOrigin)
+       │
+       ▼
+Plugin iframe browser
+  └─ POST /api/plugin-auth { session_claim }
+       │
+       ▼
+Plugin server
+  ├─ decrypt(session_claim, PLUGIN_AUTH_KEY) -> { user_id, user_role, exp }
+  └─ establish plugin session  ->  stored in sessionStorage
+```
+
+No litellm bearer token ever leaves the proxy; the claim only conveys the
+caller's identity and expires after 30 seconds.  A postMessage intercept
+yields ciphertext that is useless without the plugin's scoped key.
+
+---
+
+## Proxy routes
+
+- `GET /api/plugins` — list registered plugins (`name`, `display_name`, `url`). `plugin_key` is **never** returned; it stays server-side. Requires an authenticated caller.
+- `GET /api/plugins/auth-token?plugin_name=<name>` — short-lived encrypted identity claim for the named plugin. Requires `LITELLM_SALT_KEY` to be set (503 otherwise) and the plugin to be registered (404 otherwise).
+- `ANY /plugin-proxy/{name}/{path}` — authenticated reverse proxy to the plugin backend. Restricted to `proxy_admin`.
+
+---
+
+## Reverse proxy behaviour
+
+When an admin (or server-to-server caller) hits `/plugin-proxy/<name>/<path>`, the proxy authenticates the caller locally, then rewrites the request before forwarding it to the plugin's `url`:
+
+- **Every litellm credential header is stripped** — `Authorization`, `x-api-key`, `API-Key`, `x-goog-api-key`, `Ocp-Apim-Subscription-Key`, `x-litellm-api-key`, any configured `litellm_key_header_name`, plus `Cookie`. The plugin can never be handed the caller's live litellm key.
+- **`plugin_key` is injected** as `Authorization: Bearer <plugin_key>` — the only credential the plugin receives.
+- **Caller identity is forwarded** as `x-litellm-user-id` and `x-litellm-user-role` so the plugin can run its own authorization. These are informational, not credentials.
+- **Responses are sandboxed** — `Content-Security-Policy: sandbox` and `X-Content-Type-Options: nosniff` are set so plugin-controlled bytes served from the litellm origin cannot execute against the dashboard.
+
+---
+
+## Security checklist
+
+- [ ] `LITELLM_SALT_KEY` is set on the proxy and never shared with the plugin
+- [ ] The plugin holds only its derived `HMAC(LITELLM_SALT_KEY, plugin_name)` key, provisioned as a dedicated secret
+- [ ] `plugin_key` is a dedicated credential scoped to the plugin (not your litellm master key)
+- [ ] Plugin's `POST /api/plugin-auth` enforces the claim's `plugin` audience and `exp` (30s TTL)
+- [ ] Plugin treats `x-litellm-user-id` / `x-litellm-user-role` as identity hints, not as proof of authentication
+- [ ] Plugin service URL uses HTTPS in production

litellm-rust/.gitignore

@@ -0,0 +1 @@
+/target/

litellm-rust/ADDING_A_PROVIDER.md

@@ -0,0 +1,9 @@
+# Adding a provider / route to litellm-rust
+
+Three layers, same for every route (see `ocr` and `realtime` as references):
+
+1. **Transform contract (pure)** — `crates/core/src/<route>/transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth.
+2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
+3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
+
+**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.

litellm-rust/CLAUDE.md

@@ -0,0 +1,88 @@
+# CLAUDE.md
+
+This file defines the rules for Rust work in LiteLLM.
+
+## Core Boundary
+
+The `core` and `providers` crates describe work; hosts execute work.
+
+Route-level Rust structure mirrors LiteLLM's Python responsibilities:
+- `core/src/<route>/` owns the route contract, shared types, and provider
+  template traits. For OCR, this means `core/src/ocr`.
+- `providers/src/<provider>/<route>/transformation.rs` owns the
+  provider-specific transform. For Mistral OCR, this means
+  `providers/src/mistral/ocr/transformation.rs`.
+- Future network execution belongs in a host/transport layer such as
+  `llm_http_handler`, not inside `core` or `providers`.
+
+Allowed in `core` and `providers`:
+- Pure request transforms
+- Pure response transforms
+- Pure stream chunk normalization
+- Shared data types and validation errors
+- Deterministic token/cost helper logic
+
+Not allowed in `core` or `providers`:
+- Network calls
+- Environment variable or secret reads
+- Filesystem access
+- Database or cache access
+- Provider SDK signing or auth flows
+- Logging callbacks, spend writes, or custom callbacks
+- Global mutable runtime state
+
+Python owns rollout state and fallback while Rust is being introduced. Rust
+paths must be off by default until parity tests prove equivalence with Python.
+
+## Production Bar
+
+Rust code in this workspace is held to a strict parity and robustness bar from
+the first PR:
+
+- Correctness parity is proven with tests. Do not rely on README claims or
+  manual inspection for a port that mirrors Python behavior.
+- Every provider transform must have unit tests for supported-parameter
+  filtering, request body shape, response normalization, missing/null fields,
+  and bad-input errors.
+- When Rust is exposed through Python, add Python tests that prove disabled,
+  enabled, and unavailable-bridge fallback behavior.
+- Avoid panics on user/provider input. Return typed errors and let the host map
+  them to Python exceptions or HTTP responses.
+- OCR handles documents that often contain personal data. Do not log document
+  contents, base64 payloads, provider response bodies, or secrets.
+- Error messages must be useful but data-minimized. Truncate or sanitize any
+  upstream body before it crosses a host boundary.
+- Treat empty or whitespace-only credentials, URLs, and config values as absent
+  at the host/config resolution layer.
+- Preserve Python output shape intentionally. If a field is always serialized as
+  `null` for Python parity, leave a short comment explaining that parity choice.
+
+## Host I/O Rules
+
+These rules apply when adding future crates or modules that execute network I/O,
+such as `ai-gateway`, router hosts, or standalone servers:
+
+- Set connect and full-request timeouts. No unbounded waits.
+- Reuse HTTP clients; do not construct clients per request.
+- Prefer rustls TLS for portable Python wheels and Linux images unless there is
+  a documented reason not to.
+- Add request IDs and structured tracing at the host layer, without logging OCR
+  document contents or secrets.
+- Do not echo raw upstream response bodies to callers. Sanitize and bound them.
+- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
+  impossible by construction and documented.
+
+## Checks
+
+Run these before pushing Rust changes. The same checks run in GitHub Actions
+for changes under `litellm-rust/`.
+
+```bash
+cd litellm-rust
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```
+
+When a Rust path is exposed through Python, add Python parity tests that compare
+the existing Python output with the Rust-backed output.

litellm-rust/Cargo.toml

@@ -0,0 +1,24 @@
+[workspace]
+members = [
+    "crates/core",
+    "crates/providers",
+    "crates/python-bridge",
+]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+license = "MIT"
+repository = "https://github.com/BerriAI/litellm"
+
+[workspace.dependencies]
+litellm-core = { path = "crates/core" }
+litellm-providers = { path = "crates/providers" }
+pyo3 = "0.23.5"
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "2.0"
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }
+tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
+futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }

litellm-rust/README.md

@@ -0,0 +1,34 @@
+# LiteLLM Rust
+
+This workspace contains the staged Rust implementation for LiteLLM.
+
+Rust starts as a pure transform core used by the existing Python host. Python
+continues to own auth, configuration, network I/O, retries, routing, logging,
+callbacks, spend tracking, and customer plugins until each Rust path has parity
+coverage and production evidence.
+
+## Layout
+
+```text
+crates/
+  core/           Route contracts, shared pure types, errors, and templates.
+    src/ocr/
+  providers/      Provider-specific pure transforms.
+    src/mistral/ocr/transformation.rs
+  python-bridge/  PyO3 bridge for Python LiteLLM.
+```
+
+The folder shape should follow the Python provider tree:
+`providers/src/<provider>/<route>/transformation.rs`. The bridge should expose
+one function per top-level route, starting with `ocr(payload)`.
+
+## Checks
+
+Run these before pushing Rust changes. GitHub Actions runs the same checks for
+changes under `litellm-rust/`.
+
+```bash
+cargo fmt --check
+cargo clippy --workspace --all-targets -- -D warnings
+cargo test --workspace
+```

litellm-rust/crates/core/CLAUDE.md

@@ -0,0 +1,47 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/core`.
+
+## Responsibility
+
+`core` owns shared data types, typed errors, and deterministic helper contracts.
+It must stay pure and host-independent.
+
+Allowed:
+- Shared request/response structs.
+- Typed errors with stable, non-sensitive messages.
+- Deterministic validation helpers.
+- Serialization helpers that intentionally mirror Python output shape.
+- Route templates that match Python base config responsibilities, such as
+  `ocr::transformation::OcrProviderConfig`.
+
+Not allowed:
+- Network, filesystem, database, cache, or environment access.
+- Secret reads or auth/header construction.
+- Logging callbacks, tracing spans, spend writes, or customer callbacks.
+- Provider-specific branching that belongs in `providers`.
+- Panics for user/provider-controlled input.
+
+## Typed Contracts (core rule)
+
+Trait and function boundaries MUST be strongly typed. No stringly-typed JSON
+(`&str` / `String` / `Vec<String>` / bare `serde_json::Value`) as a transform
+input or output. Parse wire bytes into typed structs/enums at the host edge;
+`core` and `providers` operate only on those types (e.g. `RealtimeEvent`,
+`RealtimeTransformResult`, `OcrRequestData`). A `type`-style discriminator is a
+typed field on a struct, not a raw string threaded through the API.
+
+## Structure
+
+Use route names directly under `src/`: `ocr`, future `messages`,
+`chat_completions`, `embeddings`, and similar top-level LiteLLM calls. Do not
+invent broad names like `engine` for route contracts.
+
+## Parity Rules
+
+- Every shared type used by a provider transform needs unit tests for
+  serialization shape.
+- If Python parity requires always emitting a `null` field instead of omitting
+  it, document that in code and pin it with a test.
+- Error enums should preserve enough detail for Python/HTTP hosts to map errors
+  consistently without exposing document contents or upstream bodies.

litellm-rust/crates/core/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "litellm-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true

litellm-rust/crates/core/src/error.rs

@@ -0,0 +1,33 @@
+use thiserror::Error;
+
+pub type CoreResult<T> = Result<T, CoreError>;
+
+#[derive(Debug, Error, PartialEq, Eq)]
+pub enum CoreError {
+    #[error("expected {expected}, got {actual}")]
+    InvalidType {
+        expected: &'static str,
+        actual: &'static str,
+    },
+    #[error("missing required field: {0}")]
+    MissingField(&'static str),
+    #[error("invalid response: {0}")]
+    InvalidResponse(String),
+    #[error("{0}")]
+    Auth(String),
+    #[error("OCR request failed with status {status}: {body}")]
+    Http { status: u16, body: String },
+    #[error("OCR network error: {0}")]
+    Network(String),
+}
+
+pub fn json_type_name(value: &serde_json::Value) -> &'static str {
+    match value {
+        serde_json::Value::Null => "null",
+        serde_json::Value::Bool(_) => "bool",
+        serde_json::Value::Number(_) => "number",
+        serde_json::Value::String(_) => "string",
+        serde_json::Value::Array(_) => "array",
+        serde_json::Value::Object(_) => "object",
+    }
+}

litellm-rust/crates/core/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod error;
+pub mod ocr;
+pub mod realtime;
+
+pub use error::{CoreError, CoreResult};

litellm-rust/crates/core/src/ocr/mod.rs

@@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;

litellm-rust/crates/core/src/ocr/transformation.rs

@@ -0,0 +1,32 @@
+use serde_json::{Map, Value};
+
+use crate::CoreResult;
+
+use super::types::{OcrRequestData, OcrResponseData};
+
+pub trait OcrProviderConfig {
+    fn supported_ocr_params(&self) -> &'static [&'static str];
+
+    fn map_ocr_params(&self, non_default_params: &Map<String, Value>) -> Map<String, Value> {
+        let mut mapped_params = Map::new();
+        for (param, value) in non_default_params {
+            if self.supported_ocr_params().contains(&param.as_str()) {
+                mapped_params.insert(param.clone(), value.clone());
+            }
+        }
+        mapped_params
+    }
+
+    fn transform_ocr_request(
+        &self,
+        model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<OcrRequestData>;
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> CoreResult<OcrResponseData>;
+}

litellm-rust/crates/core/src/ocr/types.rs

@@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrRequestData {
+    pub data: Value,
+    pub files: Option<Value>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct OcrResponseData {
+    pub pages: Vec<Value>,
+    pub model: String,
+    pub document_annotation: Option<Value>,
+    pub usage_info: Option<Value>,
+    pub object: String,
+}
+
+impl OcrResponseData {
+    pub fn into_json(self) -> Value {
+        serde_json::json!({
+            "pages": self.pages,
+            "model": self.model,
+            "document_annotation": self.document_annotation,
+            "usage_info": self.usage_info,
+            "object": self.object,
+        })
+    }
+}

litellm-rust/crates/core/src/realtime/mod.rs

@@ -0,0 +1,2 @@
+pub mod transformation;
+pub mod types;

litellm-rust/crates/core/src/realtime/transformation.rs

@@ -0,0 +1,22 @@
+use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult};
+use crate::CoreResult;
+
+pub trait RealtimeProviderConfig {
+    /// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`).
+    /// Pure string construction only — no network, no env.
+    fn complete_url(&self, api_base: Option<&str>, model: &str) -> String;
+
+    /// Transform a client → backend event before it is forwarded upstream.
+    fn transform_realtime_request(
+        &self,
+        event: &RealtimeEvent,
+        model: &str,
+    ) -> CoreResult<RealtimeTransformResult>;
+
+    /// Transform a backend → client event before it is forwarded downstream.
+    fn transform_realtime_response(
+        &self,
+        event: &RealtimeEvent,
+        model: &str,
+    ) -> CoreResult<RealtimeTransformResult>;
+}

litellm-rust/crates/core/src/realtime/types.rs

@@ -0,0 +1,60 @@
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+
+/// A single realtime event exchanged over the WebSocket.
+///
+/// The `type` discriminator is a typed field; the remaining fields are
+/// preserved losslessly in `data` so a transform can pass an event through, or
+/// inspect/modify specific fields, without enumerating every event variant.
+/// Wire (de)serialization happens at the host edge — `core`/`providers` operate
+/// only on this typed form.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct RealtimeEvent {
+    #[serde(rename = "type")]
+    pub event_type: String,
+    #[serde(flatten)]
+    pub data: Map<String, Value>,
+}
+
+/// One or more typed events produced by a realtime transform.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct RealtimeTransformResult {
+    pub events: Vec<RealtimeEvent>,
+}
+
+impl RealtimeTransformResult {
+    /// Forward a single event unchanged (the OpenAI baseline).
+    pub fn passthrough(event: RealtimeEvent) -> Self {
+        Self {
+            events: vec![event],
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn event(raw: &str) -> RealtimeEvent {
+        serde_json::from_str(raw).expect("valid event json")
+    }
+
+    #[test]
+    fn realtime_event_round_trips_type_and_extra_fields() {
+        let raw = r#"{"type":"response.output_text.delta","delta":"hi","response_id":"r1"}"#;
+        let parsed = event(raw);
+        assert_eq!(parsed.event_type, "response.output_text.delta");
+        assert_eq!(parsed.data.get("delta"), Some(&Value::String("hi".into())));
+        // Re-serializing yields a semantically-equal event (key order may differ).
+        let reparsed: RealtimeEvent =
+            serde_json::from_str(&serde_json::to_string(&parsed).unwrap()).unwrap();
+        assert_eq!(parsed, reparsed);
+    }
+
+    #[test]
+    fn passthrough_produces_single_element_vec() {
+        let parsed = event(r#"{"type":"session.update"}"#);
+        let result = RealtimeTransformResult::passthrough(parsed.clone());
+        assert_eq!(result.events, vec![parsed]);
+    }
+}

litellm-rust/crates/providers/CLAUDE.md

@@ -0,0 +1,53 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/providers`.
+
+## Responsibility
+
+`providers` owns provider-specific pure transforms. It mirrors the existing
+Python provider modules closely enough that parity review is mechanical.
+
+Provider files should map to the Python provider tree:
+
+```text
+providers/src/<provider>/<route>/transformation.rs
+```
+
+For example, Mistral OCR lives at
+`providers/src/mistral/ocr/transformation.rs`, matching
+`litellm/llms/mistral/ocr/transformation.py`.
+
+Allowed:
+- Provider request transforms.
+- Provider response normalization.
+- Supported-parameter filtering.
+- Provider-specific validation that does not require I/O or secrets.
+
+Not allowed:
+- HTTP clients or provider SDK calls.
+- Environment variable reads.
+- API key resolution or auth header construction.
+- Logging, callbacks, spend tracking, retries, routing, cooldowns, or fallbacks.
+- Panics on bad user/provider input.
+
+## Required Tests
+
+Every provider transform must include focused unit tests for:
+- Supported params matching the Python provider config.
+- Unknown params being dropped or transformed the same way as Python.
+- Request body shape matching Python output.
+- Response normalization with complete, missing, null, and extra fields.
+- Bad input returning typed errors.
+
+For OCR specifically, assume documents can contain personal data. Tests should
+prove transforms do not copy document contents into error messages.
+
+## Implementation Rules
+
+- Prefer static supported-parameter lists over allocating strings on every call.
+- Keep transforms deterministic and allocation-conscious, but choose clarity over
+  premature micro-optimization for tiny parameter lists.
+- Use typed errors from `core`; avoid stringly-typed error plumbing.
+- Add comments only when they explain Python-parity decisions or provider quirks.
+- Put route-level provider dispatch in a route file such as `providers/src/ocr.rs`.
+  Do not move provider-specific transform logic into the Python bridge.

litellm-rust/crates/providers/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "litellm-providers"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-core.workspace = true
+reqwest.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+tokio-tungstenite.workspace = true
+futures-util.workspace = true
+
+[dev-dependencies]
+serde_json.workspace = true

litellm-rust/crates/providers/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod mistral;
+pub mod ocr;
+pub mod openai;
+pub mod realtime;

litellm-rust/crates/providers/src/mistral/mod.rs

@@ -0,0 +1 @@
+pub mod ocr;

litellm-rust/crates/providers/src/mistral/ocr/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/providers/src/mistral/ocr/transformation.rs

@@ -0,0 +1,292 @@
+use litellm_core::error::{json_type_name, CoreError, CoreResult};
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::ocr::types::{OcrRequestData, OcrResponseData};
+use serde_json::{Map, Value};
+
+const SUPPORTED_OCR_PARAMS: &[&str] = &[
+    "pages",
+    "include_image_base64",
+    "image_limit",
+    "image_min_size",
+    "bbox_annotation_format",
+    "document_annotation_format",
+    "document_annotation_prompt",
+    "extract_header",
+    "extract_footer",
+    "table_format",
+    "confidence_scores_granularity",
+    "id",
+];
+
+/// Default Mistral API base, used when the caller does not override `api_base`.
+pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1";
+
+/// Environment variable holding the Mistral API key.
+pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY";
+
+/// Error message raised when no Mistral API key can be resolved.
+pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params";
+
+/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`.
+///
+/// Blank/whitespace `api_base` is treated as absent (guard at resolution time).
+pub fn complete_url(api_base: Option<&str>) -> String {
+    let base = api_base
+        .map(str::trim)
+        .filter(|base| !base.is_empty())
+        .unwrap_or(MISTRAL_DEFAULT_API_BASE)
+        .trim_end_matches('/');
+
+    if base.ends_with("/v1") {
+        format!("{base}/ocr")
+    } else {
+        format!("{base}/v1/ocr")
+    }
+}
+
+/// Resolve the Mistral API key from the explicit param or the environment.
+///
+/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth`
+/// when no usable key is available.
+///
+/// Note: the env fallback only reads the process environment. Secret-manager
+/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in
+/// via `api_key`; this fallback is a last resort for direct/standalone use.
+pub fn resolve_api_key(
+    api_key: Option<&str>,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> CoreResult<String> {
+    api_key
+        .map(str::trim)
+        .filter(|key| !key.is_empty())
+        .map(str::to_string)
+        .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty()))
+        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+pub struct MistralOcrConfig;
+
+pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig;
+
+impl OcrProviderConfig for MistralOcrConfig {
+    fn supported_ocr_params(&self) -> &'static [&'static str] {
+        SUPPORTED_OCR_PARAMS
+    }
+
+    fn transform_ocr_request(
+        &self,
+        model: &str,
+        document: Value,
+        optional_params: Map<String, Value>,
+    ) -> CoreResult<OcrRequestData> {
+        if !document.is_object() {
+            return Err(CoreError::InvalidType {
+                expected: "object",
+                actual: json_type_name(&document),
+            });
+        }
+
+        let mut data = Map::new();
+        data.insert("model".to_string(), Value::String(model.to_string()));
+        data.insert("document".to_string(), document);
+        for (param, value) in optional_params {
+            data.insert(param, value);
+        }
+
+        Ok(OcrRequestData {
+            data: Value::Object(data),
+            files: None,
+        })
+    }
+
+    fn transform_ocr_response(
+        &self,
+        model: &str,
+        response_json: Value,
+    ) -> CoreResult<OcrResponseData> {
+        let response_object = response_json
+            .as_object()
+            .ok_or_else(|| CoreError::InvalidType {
+                expected: "object",
+                actual: json_type_name(&response_json),
+            })?;
+
+        let pages = response_object
+            .get("pages")
+            .and_then(Value::as_array)
+            .cloned()
+            .unwrap_or_default();
+        let model = response_object
+            .get("model")
+            .and_then(Value::as_str)
+            .unwrap_or(model)
+            .to_string();
+        let document_annotation = response_object.get("document_annotation").cloned();
+        let usage_info = response_object.get("usage_info").cloned();
+
+        Ok(OcrResponseData {
+            pages,
+            model,
+            document_annotation,
+            usage_info,
+            object: "ocr".to_string(),
+        })
+    }
+}
+
+pub fn supported_ocr_params() -> &'static [&'static str] {
+    MISTRAL_OCR_CONFIG.supported_ocr_params()
+}
+
+pub fn map_ocr_params(non_default_params: &Map<String, Value>) -> Map<String, Value> {
+    MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params)
+}
+
+pub fn transform_ocr_request(
+    model: &str,
+    document: Value,
+    optional_params: Map<String, Value>,
+) -> CoreResult<OcrRequestData> {
+    MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params)
+}
+
+pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult<OcrResponseData> {
+    MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn supported_params_match_python_mistral_ocr_config() {
+        assert_eq!(
+            supported_ocr_params(),
+            &[
+                "pages",
+                "include_image_base64",
+                "image_limit",
+                "image_min_size",
+                "bbox_annotation_format",
+                "document_annotation_format",
+                "document_annotation_prompt",
+                "extract_header",
+                "extract_footer",
+                "table_format",
+                "confidence_scores_granularity",
+                "id",
+            ]
+        );
+    }
+
+    #[test]
+    fn map_ocr_params_drops_unknown_params() {
+        let params = json!({
+            "extract_header": true,
+            "unsupported_param": "value",
+            "pages": [0, 1]
+        });
+        let mapped = map_ocr_params(params.as_object().unwrap());
+
+        assert_eq!(mapped.get("extract_header"), Some(&json!(true)));
+        assert_eq!(mapped.get("pages"), Some(&json!([0, 1])));
+        assert!(!mapped.contains_key("unsupported_param"));
+    }
+
+    #[test]
+    fn transform_ocr_request_builds_mistral_body() {
+        let document = json!({
+            "type": "document_url",
+            "document_url": "https://example.com/doc.pdf"
+        });
+        let optional_params = json!({
+            "include_image_base64": true,
+            "table_format": "html"
+        })
+        .as_object()
+        .unwrap()
+        .clone();
+
+        let result = transform_ocr_request("mistral-ocr-latest", document.clone(), optional_params)
+            .expect("request should transform");
+
+        assert_eq!(
+            result.data,
+            json!({
+                "model": "mistral-ocr-latest",
+                "document": document,
+                "include_image_base64": true,
+                "table_format": "html"
+            })
+        );
+        assert_eq!(result.files, None);
+    }
+
+    #[test]
+    fn transform_ocr_request_rejects_non_object_document() {
+        let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new())
+            .expect_err("string document should be rejected");
+
+        assert_eq!(
+            err,
+            CoreError::InvalidType {
+                expected: "object",
+                actual: "string",
+            }
+        );
+    }
+
+    #[test]
+    fn transform_ocr_response_normalizes_mistral_json() {
+        let response = json!({
+            "pages": [{"index": 0, "markdown": "hello"}],
+            "model": "mistral-ocr-2505-completion",
+            "document_annotation": null,
+            "usage_info": {"pages_processed": 1}
+        });
+
+        let result = transform_ocr_response("mistral-ocr-latest", response)
+            .expect("response should transform");
+
+        assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]);
+        assert_eq!(result.model, "mistral-ocr-2505-completion");
+        assert_eq!(result.document_annotation, Some(Value::Null));
+        assert_eq!(result.usage_info, Some(json!({"pages_processed": 1})));
+        assert_eq!(result.object, "ocr");
+    }
+
+    #[test]
+    fn complete_url_defaults_and_dedupes_v1() {
+        assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr");
+        assert_eq!(complete_url(Some("   ")), "https://api.mistral.ai/v1/ocr");
+        assert_eq!(
+            complete_url(Some("https://proxy.internal")),
+            "https://proxy.internal/v1/ocr"
+        );
+        assert_eq!(
+            complete_url(Some("https://proxy.internal/v1/")),
+            "https://proxy.internal/v1/ocr"
+        );
+    }
+
+    #[test]
+    fn resolve_api_key_prefers_param_then_env() {
+        let no_env = |_: &str| None;
+        assert_eq!(
+            resolve_api_key(Some("sk-param"), &no_env).unwrap(),
+            "sk-param"
+        );
+
+        let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string());
+        assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env");
+        // Blank param falls through to the environment.
+        assert_eq!(resolve_api_key(Some("  "), &with_env).unwrap(), "sk-env");
+    }
+
+    #[test]
+    fn resolve_api_key_errors_when_absent() {
+        let err = resolve_api_key(None, &|_| None).expect_err("missing key should error");
+        assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string()));
+    }
+}

litellm-rust/crates/providers/src/ocr.rs

@@ -0,0 +1,127 @@
+//! End-to-end OCR orchestration.
+//!
+//! Owns the whole Mistral OCR call so the Python side stays a thin bridge:
+//! resolve the API key, build the URL + body via the pure transforms, POST it,
+//! and normalize the response. The HTTP client is built once and reused.
+
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use litellm_core::ocr::transformation::OcrProviderConfig;
+use litellm_core::CoreResult;
+use serde_json::{Map, Value};
+
+use crate::mistral::ocr::transformation as mistral;
+use crate::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
+
+/// OCR over large documents can take a while; bound it generously rather than
+/// hanging forever on an unresponsive upstream. The client-level limit is the
+/// outer ceiling; callers can tighten it per request via ``run_ocr``'s ``timeout``.
+const OCR_TIMEOUT_SECS: u64 = 600;
+
+/// Maximum upstream body characters retained in error messages. OCR responses
+/// can echo document contents and prompts; keep enough for debugging without
+/// forwarding sensitive payloads across the host boundary.
+const ERROR_BODY_MAX_CHARS: usize = 256;
+
+/// Process-wide blocking HTTP client (connection pool + TLS reused across calls).
+fn http_client() -> &'static reqwest::blocking::Client {
+    static CLIENT: OnceLock<reqwest::blocking::Client> = OnceLock::new();
+    CLIENT.get_or_init(|| {
+        reqwest::blocking::Client::builder()
+            .timeout(Duration::from_secs(OCR_TIMEOUT_SECS))
+            .build()
+            .expect("failed to build reqwest client")
+    })
+}
+
+fn truncate_error_body(body: &str) -> String {
+    if body.chars().count() <= ERROR_BODY_MAX_CHARS {
+        return body.to_string();
+    }
+    let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect();
+    format!("{truncated}... (truncated)")
+}
+
+/// Perform a Mistral OCR call end to end and return the normalized response as
+/// JSON (the shape the Python `OCRResponse` model expects).
+///
+/// Blocking: intended to be called with the GIL released from the Python bridge.
+pub fn run_ocr(
+    model: &str,
+    document: Value,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    optional_params: Map<String, Value>,
+    timeout: Option<Duration>,
+) -> CoreResult<Value> {
+    let config = &MISTRAL_OCR_CONFIG;
+
+    let api_key = mistral::resolve_api_key(api_key, &|key| std::env::var(key).ok())?;
+    let url = mistral::complete_url(api_base);
+    let filtered_params = config.map_ocr_params(&optional_params);
+    let body = config
+        .transform_ocr_request(model, document, filtered_params)?
+        .data;
+
+    let mut request = http_client().post(&url).bearer_auth(&api_key).json(&body);
+    if let Some(duration) = timeout {
+        request = request.timeout(duration);
+    }
+
+    let response = request
+        .send()
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    let status = response.status();
+    let text = response
+        .text()
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    if !status.is_success() {
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+
+    let response_json: Value = serde_json::from_str(&text)
+        .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?;
+
+    Ok(config
+        .transform_ocr_response(model, response_json)?
+        .into_json())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn truncate_error_body_passes_short_strings_through() {
+        let body = "Unauthorized";
+        assert_eq!(truncate_error_body(body), "Unauthorized");
+    }
+
+    #[test]
+    fn truncate_error_body_caps_long_payloads() {
+        let body = "x".repeat(ERROR_BODY_MAX_CHARS + 50);
+        let truncated = truncate_error_body(&body);
+
+        assert!(truncated.ends_with("... (truncated)"));
+        let prefix_chars = truncated
+            .strip_suffix("... (truncated)")
+            .expect("truncated marker present")
+            .chars()
+            .count();
+        assert_eq!(prefix_chars, ERROR_BODY_MAX_CHARS);
+    }
+
+    #[test]
+    fn truncate_error_body_does_not_split_multibyte_chars() {
+        let body = "é".repeat(ERROR_BODY_MAX_CHARS + 10);
+        let truncated = truncate_error_body(&body);
+        assert!(truncated.is_char_boundary(truncated.len()));
+    }
+}

litellm-rust/crates/providers/src/openai/mod.rs

@@ -0,0 +1 @@
+pub mod realtime;

litellm-rust/crates/providers/src/openai/realtime/mod.rs

@@ -0,0 +1 @@
+pub mod transformation;

litellm-rust/crates/providers/src/openai/realtime/transformation.rs

@@ -0,0 +1,189 @@
+use litellm_core::realtime::transformation::RealtimeProviderConfig;
+use litellm_core::realtime::types::{RealtimeEvent, RealtimeTransformResult};
+use litellm_core::CoreResult;
+
+/// Default OpenAI API base, used when the caller does not override `api_base`.
+pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com";
+
+/// Path appended to the resolved host base to reach the realtime endpoint.
+pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime";
+
+/// Percent-encode a query value, escaping any char outside the RFC 3986
+/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime
+/// model slugs have no special chars, but this stays correct for the rest.
+fn percent_encode(value: &str) -> String {
+    let mut encoded = String::with_capacity(value.len());
+    for byte in value.bytes() {
+        let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~');
+        if unreserved {
+            encoded.push(byte as char);
+        } else {
+            encoded.push('%');
+            encoded.push_str(&format!("{byte:02X}"));
+        }
+    }
+    encoded
+}
+
+/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`.
+///
+/// Blank/whitespace `api_base` is treated as absent (guard at resolution time),
+/// falling back to the default. The scheme is swapped to its WebSocket
+/// equivalent (`https://`→`wss://`, `http://`→`ws://`); bases already using
+/// `ws`/`wss` are left untouched. A bare host or unrecognized scheme defaults to
+/// secure `wss://` so we never hand a scheme-less URL to the connector (this is
+/// a deliberate hardening over Python's `_construct_url`, which would emit a
+/// scheme-less URL here). A trailing `/` is trimmed before the path and
+/// `?model=<encoded>` are appended.
+pub fn complete_url(api_base: Option<&str>, model: &str) -> String {
+    let base = api_base
+        .map(str::trim)
+        .filter(|base| !base.is_empty())
+        .unwrap_or(OPENAI_REALTIME_DEFAULT_API_BASE);
+
+    let base = if let Some(rest) = base.strip_prefix("https://") {
+        format!("wss://{rest}")
+    } else if let Some(rest) = base.strip_prefix("http://") {
+        format!("ws://{rest}")
+    } else if base.starts_with("wss://") || base.starts_with("ws://") {
+        base.to_string()
+    } else {
+        format!("wss://{base}")
+    };
+
+    let base = base.trim_end_matches('/');
+
+    format!(
+        "{base}{OPENAI_REALTIME_PATH}?model={}",
+        percent_encode(model)
+    )
+}
+
+pub struct OpenAiRealtimeConfig;
+
+pub const OPENAI_REALTIME_CONFIG: OpenAiRealtimeConfig = OpenAiRealtimeConfig;
+
+impl RealtimeProviderConfig for OpenAiRealtimeConfig {
+    fn complete_url(&self, api_base: Option<&str>, model: &str) -> String {
+        complete_url(api_base, model)
+    }
+
+    fn transform_realtime_request(
+        &self,
+        event: &RealtimeEvent,
+        _model: &str,
+    ) -> CoreResult<RealtimeTransformResult> {
+        Ok(RealtimeTransformResult::passthrough(event.clone()))
+    }
+
+    fn transform_realtime_response(
+        &self,
+        event: &RealtimeEvent,
+        _model: &str,
+    ) -> CoreResult<RealtimeTransformResult> {
+        Ok(RealtimeTransformResult::passthrough(event.clone()))
+    }
+}
+
+pub fn transform_realtime_request(
+    event: &RealtimeEvent,
+    model: &str,
+) -> CoreResult<RealtimeTransformResult> {
+    OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model)
+}
+
+pub fn transform_realtime_response(
+    event: &RealtimeEvent,
+    model: &str,
+) -> CoreResult<RealtimeTransformResult> {
+    OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn complete_url_defaults_to_openai_wss() {
+        assert_eq!(
+            complete_url(None, "gpt-4o-realtime-preview"),
+            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_blank_base_uses_default() {
+        assert_eq!(
+            complete_url(Some("   "), "gpt-4o-realtime-preview"),
+            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_swaps_http_to_ws() {
+        assert_eq!(
+            complete_url(Some("http://localhost:8080"), "gpt-4o-realtime-preview"),
+            "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_dedupes_trailing_slash() {
+        assert_eq!(
+            complete_url(Some("https://api.openai.com/"), "gpt-4o-realtime-preview"),
+            "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_custom_base() {
+        assert_eq!(
+            complete_url(Some("https://oai.azure.example"), "gpt-4o-realtime-preview"),
+            "wss://oai.azure.example/v1/realtime?model=gpt-4o-realtime-preview"
+        );
+    }
+
+    #[test]
+    fn complete_url_preserves_existing_wss_scheme() {
+        assert_eq!(
+            complete_url(Some("wss://api.openai.com"), "gpt-realtime"),
+            "wss://api.openai.com/v1/realtime?model=gpt-realtime"
+        );
+    }
+
+    #[test]
+    fn complete_url_bare_host_defaults_to_wss() {
+        assert_eq!(
+            complete_url(Some("api.openai.com"), "gpt-realtime"),
+            "wss://api.openai.com/v1/realtime?model=gpt-realtime"
+        );
+    }
+
+    #[test]
+    fn complete_url_percent_encodes_model_space() {
+        assert_eq!(
+            complete_url(None, "gpt 4o"),
+            "wss://api.openai.com/v1/realtime?model=gpt%204o"
+        );
+    }
+
+    #[test]
+    fn transform_realtime_request_passthrough_preserves_event() {
+        let event: RealtimeEvent =
+            serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#)
+                .expect("valid event");
+        let result =
+            transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible");
+        assert_eq!(result.events, vec![event]);
+    }
+
+    #[test]
+    fn transform_realtime_response_passthrough_preserves_event() {
+        let event: RealtimeEvent =
+            serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#)
+                .expect("valid event");
+        let result =
+            transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible");
+        assert_eq!(result.events, vec![event]);
+    }
+}

litellm-rust/crates/providers/src/realtime.rs

@@ -0,0 +1,193 @@
+//! End-to-end OpenAI realtime invocation.
+//!
+//! The host-facing entry point, mirroring `providers::ocr::run_ocr`: open the
+//! WebSocket to OpenAI, drive typed events through the pure
+//! `OPENAI_REALTIME_CONFIG` transforms, and collect the response events.
+//! Network, auth header, key resolution, and wire (de)serialization live here so
+//! the `transformation` module stays pure and typed.
+
+use std::time::Duration;
+
+use futures_util::{SinkExt, StreamExt};
+use litellm_core::error::CoreError;
+use litellm_core::realtime::transformation::RealtimeProviderConfig;
+use litellm_core::realtime::types::RealtimeEvent;
+use litellm_core::CoreResult;
+use tokio_tungstenite::connect_async;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION;
+use tokio_tungstenite::tungstenite::http::HeaderValue;
+use tokio_tungstenite::tungstenite::Message;
+
+use crate::openai::realtime::transformation::OPENAI_REALTIME_CONFIG;
+
+/// Environment variable holding the OpenAI API key (last-resort fallback).
+const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
+
+/// Default overall ceiling for a single realtime invocation.
+const DEFAULT_TIMEOUT_SECS: u64 = 60;
+
+const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable";
+
+/// Resolve the OpenAI API key from the explicit param or the environment.
+///
+/// Blank/whitespace values are treated as absent (guard at resolution time).
+fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
+    api_key
+        .map(str::trim)
+        .filter(|key| !key.is_empty())
+        .map(str::to_string)
+        .or_else(|| {
+            std::env::var(OPENAI_API_KEY_ENV)
+                .ok()
+                .filter(|key| !key.trim().is_empty())
+        })
+        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+/// True for events that end a realtime turn: a completed response or an error.
+fn is_terminal_event(event: &RealtimeEvent) -> bool {
+    event.event_type == "response.done" || event.event_type == "error"
+}
+
+/// Invoke the OpenAI realtime API end to end over a WebSocket.
+///
+/// Sends each `input_events` entry after passing it through
+/// `transform_realtime_request`, then collects backend events — each passed
+/// through `transform_realtime_response` — until a terminal event
+/// (`response.done` / `error`) arrives, the socket closes, or the `timeout`
+/// elapses. Returns the transformed backend events in arrival order.
+///
+/// Mirrors `run_ocr`: pure transforms come from `core`/`providers`; the network,
+/// auth header, key resolution, and JSON (de)serialization are owned here.
+pub async fn realtime(
+    model: &str,
+    input_events: Vec<RealtimeEvent>,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    timeout: Option<Duration>,
+) -> CoreResult<Vec<RealtimeEvent>> {
+    let config = &OPENAI_REALTIME_CONFIG;
+    let api_key = resolve_api_key(api_key)?;
+    let url = config.complete_url(api_base, model);
+
+    let mut request = url
+        .as_str()
+        .into_client_request()
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+    // GA realtime API: only Authorization is needed. The legacy
+    // `OpenAI-Beta: realtime=v1` header opts into the now-removed beta request
+    // shape and triggers `beta_api_shape_disabled`, so we do not send it.
+    request.headers_mut().insert(
+        AUTHORIZATION,
+        HeaderValue::from_str(&format!("Bearer {api_key}"))
+            .map_err(|err| CoreError::Auth(err.to_string()))?,
+    );
+
+    let (mut ws, _response) = connect_async(request)
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    for event in &input_events {
+        for outbound in config.transform_realtime_request(event, model)?.events {
+            let payload = serde_json::to_string(&outbound)
+                .map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
+            ws.send(Message::Text(payload))
+                .await
+                .map_err(|err| CoreError::Network(err.to_string()))?;
+        }
+    }
+
+    let deadline = timeout.unwrap_or_else(|| Duration::from_secs(DEFAULT_TIMEOUT_SECS));
+    let mut received: Vec<RealtimeEvent> = Vec::new();
+
+    let collect = async {
+        while let Some(message) = ws.next().await {
+            match message.map_err(|err| CoreError::Network(err.to_string()))? {
+                Message::Text(text) => {
+                    let event: RealtimeEvent = serde_json::from_str(text.as_str())
+                        .map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
+                    for outbound in config.transform_realtime_response(&event, model)?.events {
+                        let terminal = is_terminal_event(&outbound);
+                        received.push(outbound);
+                        if terminal {
+                            return Ok::<(), CoreError>(());
+                        }
+                    }
+                }
+                Message::Close(_) => return Ok(()),
+                _ => {}
+            }
+        }
+        Ok(())
+    };
+
+    tokio::time::timeout(deadline, collect)
+        .await
+        .map_err(|_| CoreError::Network("realtime call timed out".to_string()))??;
+
+    Ok(received)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn event(raw: &str) -> RealtimeEvent {
+        serde_json::from_str(raw).expect("valid event json")
+    }
+
+    #[test]
+    fn resolve_api_key_prefers_param_then_blank_falls_through() {
+        assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test");
+        // A blank param with no env set should error.
+        if std::env::var(OPENAI_API_KEY_ENV).is_err() {
+            assert!(resolve_api_key(Some("   ")).is_err());
+        }
+    }
+
+    #[test]
+    fn is_terminal_event_matches_done_and_error_only() {
+        assert!(is_terminal_event(&event(r#"{"type":"response.done"}"#)));
+        assert!(is_terminal_event(&event(r#"{"type":"error","error":{}}"#)));
+        assert!(!is_terminal_event(&event(
+            r#"{"type":"response.output_text.delta"}"#
+        )));
+    }
+
+    /// Live end-to-end check against OpenAI. Ignored by default (CI never runs
+    /// it); run explicitly with `OPENAI_API_KEY` set:
+    ///   `cargo test -p litellm-providers realtime_invokes_openai -- --ignored --nocapture`
+    #[tokio::test]
+    #[ignore = "hits the live OpenAI realtime API; needs OPENAI_API_KEY"]
+    async fn realtime_invokes_openai_and_responds() {
+        let key =
+            std::env::var(OPENAI_API_KEY_ENV).expect("set OPENAI_API_KEY to run this ignored test");
+
+        let response_create = event(
+            r#"{"type":"response.create","response":{"output_modalities":["text"],"instructions":"Respond with exactly: hello world"}}"#,
+        );
+
+        let events = realtime(
+            "gpt-realtime",
+            vec![response_create],
+            Some(&key),
+            None,
+            Some(Duration::from_secs(30)),
+        )
+        .await
+        .expect("realtime call should succeed");
+
+        let types: Vec<&str> = events.iter().map(|e| e.event_type.as_str()).collect();
+        eprintln!("received {} events: {:?}", events.len(), types);
+
+        assert!(
+            types.contains(&"response.done"),
+            "expected a response.done event, got: {types:?}"
+        );
+        assert!(
+            types.contains(&"response.output_text.delta"),
+            "expected streamed text output, got: {types:?}"
+        );
+    }
+}

litellm-rust/crates/python-bridge/CLAUDE.md

@@ -0,0 +1,36 @@
+# CLAUDE.md
+
+Rules for `litellm-rust/crates/python-bridge`.
+
+## Responsibility
+
+`python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms.
+Keep this crate thin. It adapts Python objects to Rust payloads and returns
+Python-compatible dictionaries.
+
+## Bridge Shape
+
+- Prefer one stable method per top-level LiteLLM route, for example
+  `ocr(payload)`.
+- Do not add one exported PyO3 function per provider helper unless there is a
+  measured reason.
+- Provider dispatch belongs in Rust route modules such as
+  `litellm_providers::ocr`, not in this PyO3 crate.
+- Python owns rollout state and fallback. Rust should return errors; Python
+  decides whether to raise or fall back.
+
+## Data Handling
+
+- OCR payloads can contain personal data and large base64 images. Do not log
+  payloads or provider responses.
+- Avoid copying large payloads more than needed. The current JSON round-trip is
+  acceptable for the first scaffold, but future performance work should evaluate
+  direct PyO3 conversion before expanding Rust coverage to image-heavy paths.
+- Do not expose raw Rust errors that include document contents or upstream
+  bodies.
+
+## Tests
+
+- `cargo test --workspace` must compile this crate.
+- Python tests must cover bridge disabled, bridge enabled, and module-missing
+  fallback behavior for every exposed route.

litellm-rust/crates/python-bridge/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "litellm-python-bridge"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[lib]
+name = "litellm_python_bridge"
+crate-type = ["cdylib"]
+
+[dependencies]
+litellm-core.workspace = true
+litellm-providers.workspace = true
+pyo3 = { workspace = true, features = ["extension-module"] }
+serde_json.workspace = true

litellm-rust/crates/python-bridge/src/gil.rs

@@ -0,0 +1,32 @@
+//! GIL accounting.
+//!
+//! A single chokepoint for releasing the GIL around blocking work. Every
+//! blocking call in the bridge goes through [`release_gil`] instead of calling
+//! `Python::allow_threads` directly, so the release count stays accurate and we
+//! have one place to extend later (timing histograms, per-call labels, etc.).
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use pyo3::prelude::*;
+
+/// Number of times the bridge has released the GIL since process start.
+static GIL_RELEASES: AtomicU64 = AtomicU64::new(0);
+
+/// Release the GIL around `f`, recording the release.
+///
+/// `f` must not touch any Python state — that is what makes releasing the GIL
+/// safe. Returning the value back to Python re-acquires the GIL at the call
+/// site, after `f` has finished.
+pub fn release_gil<T, F>(py: Python<'_>, f: F) -> T
+where
+    F: FnOnce() -> T + Send,
+    T: Send,
+{
+    GIL_RELEASES.fetch_add(1, Ordering::Relaxed);
+    py.allow_threads(f)
+}
+
+/// Total GIL releases performed by the bridge so far.
+pub fn release_count() -> u64 {
+    GIL_RELEASES.load(Ordering::Relaxed)
+}

litellm-rust/crates/python-bridge/src/lib.rs

@@ -0,0 +1,100 @@
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use litellm_providers::ocr::run_ocr;
+use pyo3::exceptions::{PyRuntimeError, PyValueError};
+use pyo3::prelude::*;
+use pyo3::types::{PyAny, PyDict};
+use serde_json::{Map, Value};
+
+mod gil;
+
+fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult<Value> {
+    let json = py.import("json")?;
+    let encoded: String = json.call_method1("dumps", (value,))?.extract()?;
+    serde_json::from_str(&encoded).map_err(|err| PyValueError::new_err(err.to_string()))
+}
+
+fn json_to_py(py: Python<'_>, value: Value) -> PyResult<Py<PyAny>> {
+    let json = py.import("json")?;
+    let encoded =
+        serde_json::to_string(&value).map_err(|err| PyValueError::new_err(err.to_string()))?;
+    Ok(json.call_method1("loads", (encoded,))?.unbind())
+}
+
+/// Map a core error to the closest Python exception. Caller-input problems
+/// (auth, bad types, missing fields) -> `ValueError`; everything else
+/// (network, upstream status, parse failures) -> `RuntimeError`.
+fn core_error_to_pyerr(err: CoreError) -> PyErr {
+    match err {
+        CoreError::Auth(message) => PyValueError::new_err(message),
+        CoreError::InvalidType { .. } | CoreError::MissingField(_) => {
+            PyValueError::new_err(err.to_string())
+        }
+        other => PyRuntimeError::new_err(other.to_string()),
+    }
+}
+
+/// Perform a Mistral OCR call end to end and return the response as a dict.
+#[pyfunction]
+#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None, timeout_seconds=None))]
+fn ocr(
+    py: Python<'_>,
+    model: String,
+    document: Py<PyAny>,
+    api_key: Option<String>,
+    api_base: Option<String>,
+    optional_params: Option<Py<PyAny>>,
+    timeout_seconds: Option<f64>,
+) -> PyResult<Py<PyAny>> {
+    let document = py_to_json(py, document.bind(py))?;
+
+    let optional_params = match optional_params {
+        Some(params) => match py_to_json(py, params.bind(py))? {
+            Value::Object(map) => map,
+            _ => return Err(PyValueError::new_err("optional_params must be a dict")),
+        },
+        None => Map::new(),
+    };
+
+    let timeout = timeout_seconds.and_then(|secs| {
+        if secs.is_finite() && secs > 0.0 {
+            Some(Duration::from_secs_f64(secs))
+        } else {
+            None
+        }
+    });
+
+    // Release the GIL during the blocking HTTP call (counted for observability).
+    let result = gil::release_gil(py, || {
+        run_ocr(
+            &model,
+            document,
+            api_key.as_deref(),
+            api_base.as_deref(),
+            optional_params,
+            timeout,
+        )
+    });
+
+    match result {
+        Ok(value) => json_to_py(py, value),
+        Err(err) => Err(core_error_to_pyerr(err)),
+    }
+}
+
+/// Bridge GIL accounting, e.g. `{"releases": 12}`. Lets the Python side observe
+/// how often the bridge has dropped the GIL for blocking work.
+#[pyfunction]
+fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {
+    let stats = PyDict::new(py);
+    stats.set_item("releases", gil::release_count())?;
+    Ok(stats.into_any().unbind())
+}
+
+#[pymodule]
+fn litellm_python_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    module.add_function(wrap_pyfunction!(ocr, module)?)?;
+    module.add_function(wrap_pyfunction!(gil_stats, module)?)?;
+    Ok(())
+}

litellm/__init__.py

@@ -80,6 +80,7 @@ from litellm.constants import (
     WANDB_MODELS,
     REPEATED_STREAMING_CHUNK_LIMIT,
     request_timeout,
+    request_timeout_explicitly_set as request_timeout_explicitly_set,
     open_ai_embedding_models,
     cohere_embedding_models,
     bedrock_embedding_models,
@@ -673,6 +674,7 @@ elevenlabs_models: Set = set()
 dashscope_models: Set = set()
 moonshot_models: Set = set()
 publicai_models: Set = set()
+darkbloom_models: Set = set()
 v0_models: Set = set()
 morph_models: Set = set()
 lambda_ai_models: Set = set()
@@ -927,6 +929,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
             moonshot_models.add(key)
         elif value.get("litellm_provider") == "publicai":
             publicai_models.add(key)
+        elif value.get("litellm_provider") == "darkbloom":
+            darkbloom_models.add(key)
         elif value.get("litellm_provider") == "v0":
             v0_models.add(key)
         elif value.get("litellm_provider") == "morph":
@@ -1075,6 +1079,7 @@ model_list = list(
     | dashscope_models
     | moonshot_models
     | publicai_models
+    | darkbloom_models
     | v0_models
     | morph_models
     | lambda_ai_models
@@ -1179,6 +1184,7 @@ models_by_provider: dict = {
     "modelscope": modelscope_models,
     "moonshot": moonshot_models,
     "publicai": publicai_models,
+    "darkbloom": darkbloom_models,
     "v0": v0_models,
     "morph": morph_models,
     "lambda_ai": lambda_ai_models,
@@ -1400,6 +1406,7 @@ from .skills.main import (
 )
 from .containers.main import *
 from .ocr.main import *
+from .ocr.rust_bridge import use_litellm_rust
 from .rag.main import *
 from .sandbox.main import *
 from .search.main import *
@@ -1922,9 +1929,6 @@ if TYPE_CHECKING:
     from .llms.fireworks_ai.completion.transformation import (
         FireworksAITextCompletionConfig as FireworksAITextCompletionConfig,
     )
-    from .llms.fireworks_ai.audio_transcription.transformation import (
-        FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig,
-    )
     from .llms.fireworks_ai.embed.fireworks_ai_transformation import (
         FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig,
     )

litellm/_lazy_imports_registry.py

@@ -260,7 +260,6 @@ LLM_CONFIG_NAMES = (
     "SambaNovaEmbeddingConfig",
     "FireworksAIConfig",
     "FireworksAITextCompletionConfig",
-    "FireworksAIAudioTranscriptionConfig",
     "FireworksAIEmbeddingConfig",
     "FriendliaiChatConfig",
     "JinaAIEmbeddingConfig",
@@ -1027,10 +1026,6 @@ _LLM_CONFIGS_IMPORT_MAP = {
         ".llms.fireworks_ai.completion.transformation",
         "FireworksAITextCompletionConfig",
     ),
-    "FireworksAIAudioTranscriptionConfig": (
-        ".llms.fireworks_ai.audio_transcription.transformation",
-        "FireworksAIAudioTranscriptionConfig",
-    ),
     "FireworksAIEmbeddingConfig": (
         ".llms.fireworks_ai.embed.fireworks_ai_transformation",
         "FireworksAIEmbeddingConfig",

litellm/constants.py

@@ -201,6 +201,18 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
 
 # Provider-specific API base URLs
 XAI_API_BASE = "https://api.x.ai/v1"
+OPEN_SANDBOX_API_BASE_ENV_VAR = "OPEN_SANDBOX_API_BASE"
+OPEN_SANDBOX_API_KEY_ENV_VAR = "OPEN_SANDBOX_API_KEY"
+OPEN_SANDBOX_DEFAULT_TEMPLATE = "opensandbox/code-interpreter:v1.1.0"
+_OPEN_SANDBOX_FALLBACK_ENTRYPOINT = "/opt/code-interpreter/code-interpreter.sh"
+OPEN_SANDBOX_DEFAULT_ENTRYPOINT = (_OPEN_SANDBOX_FALLBACK_ENTRYPOINT,)
+OPEN_SANDBOX_DEFAULT_LANGUAGE = "python"
+OPEN_SANDBOX_DEFAULT_CPU_LIMIT = "1"
+OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT = "2Gi"
+OPEN_SANDBOX_EXECD_PORT = 44772
+OPEN_SANDBOX_DEFAULT_TIMEOUT = 300
+OPEN_SANDBOX_READY_TIMEOUT = 30.0
+OPEN_SANDBOX_POLL_INTERVAL = 0.2
 
 DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)
@@ -456,6 +468,7 @@ HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: float = 5.0
 request_timeout: float = float(
     os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))
 )
+request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ
 DEFAULT_A2A_AGENT_TIMEOUT: float = float(
     os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
 )  # 10 minutes
@@ -867,6 +880,7 @@ openai_compatible_providers: List = [
     "docker_model_runner",
     "ragflow",
     "pinstripes",  # Pinstripes - JSON-configured provider
+    "darkbloom",
 ]
 openai_text_completion_compatible_providers: List = (
     [  # providers that support `/v1/completions`

litellm/integrations/code_interpreter_interception/__init__.py

@@ -0,0 +1,15 @@
+"""
+Code Interpreter Interception Module
+
+Converts the native OpenAI Responses ``code_interpreter`` tool into a function
+tool, runs the model-emitted code in a sandbox, and feeds the result back into
+the agentic loop.
+"""
+
+from litellm.integrations.code_interpreter_interception.handler import (
+    CodeInterpreterInterceptionLogger,
+)
+
+__all__ = [
+    "CodeInterpreterInterceptionLogger",
+]

litellm/integrations/code_interpreter_interception/handler.py

@@ -0,0 +1,839 @@
+"""
+Code Interpreter Interception Handler
+
+CustomLogger that swaps the native OpenAI Responses ``code_interpreter`` tool for
+a function tool, executes the code the model emits inside a sandbox, and feeds the
+captured stdout back through the typed agentic loop plan.
+"""
+
+import json
+import time
+import uuid
+from typing import Any, Literal, TypedDict, cast
+
+import litellm
+from pydantic import ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.integrations.code_interpreter_interception import (
+    CodeInterpreterInterceptionConfig,
+)
+from litellm.types.integrations.custom_logger import (
+    AgenticLoopPlan,
+    AgenticLoopRequestPatch,
+    CHAT_COMPLETION_AGENTIC_SURFACE,
+    NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
+    is_interception_internal_key,
+)
+from litellm.types.llms.openai import (
+    ChatCompletionAssistantMessage,
+    ChatCompletionAssistantToolCall,
+    ChatCompletionToolMessage,
+)
+from litellm.types.utils import (
+    CallTypes,
+    ChatCompletionMessageToolCall,
+    ModelResponse,
+)
+
+LITELLM_CODE_EXECUTION_TOOL_NAME = "litellm_code_execution"
+_INTERCEPTION_ACTIVE_KEY = "_code_interpreter_interception_active"
+_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
+_CONVERTED_STREAM_KEY = "_code_interpreter_interception_converted_stream"
+_LITELLM_METADATA_KEY = "litellm_metadata"
+_CACHE_TTL_SECONDS = 15 * 60
+
+
+class CodeExecutionToolCall(TypedDict, total=False):
+    id: str | None
+    call_id: str | None
+    type: Literal["function"]
+    name: str
+    arguments: str
+
+
+class CodeInterpreterLogOutput(TypedDict):
+    type: Literal["logs"]
+    logs: str
+
+
+class CodeInterpreterCall(TypedDict):
+    id: str
+    type: Literal["code_interpreter_call"]
+    status: Literal["completed"]
+    code: str
+    container_id: str | None
+    outputs: list[CodeInterpreterLogOutput]
+
+
+class CodeExecutionFunctionParameters(TypedDict):
+    type: Literal["object"]
+    properties: dict[str, dict[str, str]]
+    required: list[str]
+
+
+class ResponsesFunctionTool(TypedDict):
+    type: Literal["function"]
+    name: str
+    description: str
+    parameters: CodeExecutionFunctionParameters
+
+
+class ChatCompletionFunctionDefinition(TypedDict):
+    name: str
+    description: str
+    parameters: CodeExecutionFunctionParameters
+
+
+class ChatCompletionFunctionTool(TypedDict):
+    type: Literal["function"]
+    function: ChatCompletionFunctionDefinition
+
+
+CodeExecutionFunctionTool = ResponsesFunctionTool | ChatCompletionFunctionTool
+
+
+class ResponsesFunctionToolChoice(TypedDict):
+    type: Literal["function"]
+    name: str
+
+
+class ChatCompletionFunctionToolChoice(TypedDict):
+    type: Literal["function"]
+    function: dict[str, str]
+
+
+CodeExecutionFunctionToolChoice = (
+    ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice
+)
+
+
+def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None:
+    try:
+        from litellm.sandbox.sandbox_tools import resolve_sandbox_tool
+    except ImportError:
+        return None
+    return resolve_sandbox_tool(sandbox_tool_name)
+
+
+class CodeInterpreterInterceptionLogger(CustomLogger):
+    """
+    CustomLogger that implements transparent code-interpreter execution loops.
+
+    Flow:
+    1. Replace the native ``code_interpreter`` tool with a function tool in the
+       pre-call hook so the model emits code as function-call arguments.
+    2. Detect ``litellm_code_execution`` function calls in the model response.
+    3. Run the emitted code in a sandbox (reused per request via a server-minted
+       sandbox key) and build a typed rerun plan that appends the
+       function_call_output.
+    """
+
+    def __init__(
+        self,
+        enabled: bool = True,
+        enabled_providers: list[str] | None = None,
+        sandbox_tool_name: str | None = None,
+        sandbox_config: Any | None = None,
+    ):
+        super().__init__()
+        self.enabled = enabled
+        self.enabled_providers = enabled_providers
+        self.sandbox_tool_name = sandbox_tool_name
+        self.sandbox_config = sandbox_config
+        self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float]] = {}
+
+    @classmethod
+    def from_config_yaml(
+        cls, config: CodeInterpreterInterceptionConfig
+    ) -> "CodeInterpreterInterceptionLogger":
+        return cls(
+            enabled=bool(config.get("enabled", True)),
+            enabled_providers=config.get("enabled_providers"),
+            sandbox_tool_name=config.get("sandbox_tool_name"),
+        )
+
+    @staticmethod
+    def initialize_from_proxy_config(
+        litellm_settings: dict[str, Any],
+        callback_specific_params: dict[str, Any],
+    ) -> "CodeInterpreterInterceptionLogger":
+        params: CodeInterpreterInterceptionConfig = {}
+        if "code_interpreter_interception_params" in litellm_settings:
+            params = litellm_settings["code_interpreter_interception_params"]
+        elif "code_interpreter_interception" in callback_specific_params and isinstance(
+            callback_specific_params["code_interpreter_interception"], dict
+        ):
+            params = cast(
+                CodeInterpreterInterceptionConfig,
+                callback_specific_params["code_interpreter_interception"],
+            )
+        return CodeInterpreterInterceptionLogger.from_config_yaml(params)
+
+    async def async_pre_call_deployment_hook(
+        self, kwargs: dict[str, Any], call_type: CallTypes | None
+    ) -> dict | None:
+        if not kwargs.get("_agentic_loop_depth"):
+            kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
+            kwargs.pop(_SANDBOX_KEY, None)
+            self._strip_interception_metadata(kwargs)
+        if not self.enabled:
+            return None
+        if call_type not in (
+            CallTypes.responses,
+            CallTypes.aresponses,
+            CallTypes.completion,
+            CallTypes.acompletion,
+        ):
+            return None
+        if (
+            self.enabled_providers is not None
+            and self._resolve_provider(kwargs) not in self.enabled_providers
+        ):
+            return None
+
+        tools = kwargs.get("tools")
+        if not isinstance(tools, list):
+            return None
+        if not any(
+            isinstance(tool, dict) and tool.get("type") == "code_interpreter"
+            for tool in tools
+        ):
+            return None
+
+        kwargs[_INTERCEPTION_ACTIVE_KEY] = True
+        kwargs[_SANDBOX_KEY] = uuid.uuid4().hex
+        if kwargs.get("stream"):
+            kwargs["stream"] = False
+            kwargs[_CONVERTED_STREAM_KEY] = True
+        self._write_interception_metadata(kwargs)
+
+        function_tool = self._get_function_tool(call_type=call_type)
+        kwargs["tools"] = [
+            (
+                function_tool
+                if isinstance(tool, dict) and tool.get("type") == "code_interpreter"
+                else tool
+            )
+            for tool in tools
+        ]
+        if self._tool_choice_targets_code_interpreter(kwargs.get("tool_choice")):
+            kwargs["tool_choice"] = self._get_function_tool_choice(call_type=call_type)
+        return kwargs
+
+    @staticmethod
+    def _strip_interception_metadata(kwargs: dict[str, Any]) -> None:
+        metadata = kwargs.get(_LITELLM_METADATA_KEY)
+        if not isinstance(metadata, dict):
+            return
+        filtered_metadata = {
+            key: value
+            for key, value in metadata.items()
+            if not is_interception_internal_key(key)
+            and not key.startswith("_agentic_loop")
+            and key != "max_agentic_loops"
+        }
+        if filtered_metadata:
+            kwargs[_LITELLM_METADATA_KEY] = filtered_metadata
+        else:
+            kwargs.pop(_LITELLM_METADATA_KEY, None)
+
+    @staticmethod
+    def _write_interception_metadata(kwargs: dict[str, Any]) -> None:
+        metadata = kwargs.get(_LITELLM_METADATA_KEY)
+        metadata = dict(metadata) if isinstance(metadata, dict) else {}
+        for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _CONVERTED_STREAM_KEY):
+            if key in kwargs:
+                metadata[key] = kwargs[key]
+        kwargs[_LITELLM_METADATA_KEY] = metadata
+
+    @staticmethod
+    def _get_function_parameters() -> CodeExecutionFunctionParameters:
+        return {
+            "type": "object",
+            "properties": {"code": {"type": "string"}},
+            "required": ["code"],
+        }
+
+    def _get_function_tool(
+        self, call_type: CallTypes | None
+    ) -> CodeExecutionFunctionTool:
+        description = "Execute python code in a sandbox and return stdout."
+        if call_type in (CallTypes.completion, CallTypes.acompletion):
+            return {
+                "type": "function",
+                "function": {
+                    "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                    "description": description,
+                    "parameters": self._get_function_parameters(),
+                },
+            }
+        return {
+            "type": "function",
+            "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+            "description": description,
+            "parameters": self._get_function_parameters(),
+        }
+
+    @staticmethod
+    def _get_function_tool_choice(
+        call_type: CallTypes | None,
+    ) -> CodeExecutionFunctionToolChoice:
+        if call_type in (CallTypes.completion, CallTypes.acompletion):
+            return {
+                "type": "function",
+                "function": {"name": LITELLM_CODE_EXECUTION_TOOL_NAME},
+            }
+        return {
+            "type": "function",
+            "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+        }
+
+    @staticmethod
+    def _tool_choice_targets_code_interpreter(tool_choice: Any) -> bool:
+        if not isinstance(tool_choice, dict):
+            return False
+        function = tool_choice.get("function")
+        return (
+            tool_choice.get("type") == "code_interpreter"
+            or tool_choice.get("name") == "code_interpreter"
+            or tool_choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+            or (
+                isinstance(function, dict)
+                and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+            )
+        )
+
+    def _resolve_provider(self, kwargs: dict[str, Any]) -> str | None:
+        provider = kwargs.get("custom_llm_provider")
+        if provider:
+            return provider
+        model = kwargs.get("model")
+        if not isinstance(model, str):
+            return None
+        try:
+            return litellm.get_llm_provider(model=model)[1]
+        except Exception:
+            return None
+
+    async def async_should_run_agentic_loop(
+        self,
+        response: Any,
+        model: str,
+        messages: list[dict],
+        tools: list[dict] | None,
+        stream: bool,
+        custom_llm_provider: str,
+        kwargs: dict,
+    ) -> tuple[bool, dict]:
+        if not self.enabled:
+            return False, {}
+        if not kwargs.get(_INTERCEPTION_ACTIVE_KEY):
+            return False, {}
+        if (
+            self.enabled_providers is not None
+            and custom_llm_provider not in self.enabled_providers
+        ):
+            return False, {}
+
+        tool_calls = (
+            self._extract_chat_completion_code_execution_tool_calls(response=response)
+            if kwargs.get("_agentic_loop_api_surface")
+            == CHAT_COMPLETION_AGENTIC_SURFACE
+            else self._extract_code_execution_tool_calls(response=response)
+        )
+        if not tool_calls:
+            return False, {}
+
+        return True, {"tool_calls": tool_calls}
+
+    async def async_build_agentic_loop_plan(
+        self,
+        tools: dict,
+        model: str,
+        messages: list[dict],
+        response: Any,
+        anthropic_messages_provider_config: Any,
+        anthropic_messages_optional_request_params: dict,
+        logging_obj: Any,
+        stream: bool,
+        kwargs: dict,
+    ) -> AgenticLoopPlan:
+        if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE:
+            return await self._build_chat_completion_agentic_loop_plan(
+                tools=tools,
+                model=model,
+                messages=messages,
+                optional_params=anthropic_messages_optional_request_params,
+                kwargs=kwargs,
+            )
+
+        await self._prune_expired_cache()
+        tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
+        sandbox_key = kwargs.get(_SANDBOX_KEY)
+        container, params = await self._get_or_create_container(cache_key=sandbox_key)
+
+        try:
+            container_id = cast(str | None, getattr(container, "id", None))
+            input_list = self._normalize_messages(messages)
+            code_interpreter_calls: list[CodeInterpreterCall] = []
+            for tool_call in tool_calls:
+                arguments = tool_call.get("arguments", "")
+                code = self._parse_code(arguments)
+                stdout = await self._run_tool_call(
+                    container=container, params=params, arguments=arguments
+                )
+                input_list.append(
+                    {
+                        "type": "function_call",
+                        "call_id": tool_call.get("call_id"),
+                        "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                        "arguments": arguments,
+                    }
+                )
+                input_list.append(
+                    {
+                        "type": "function_call_output",
+                        "call_id": tool_call.get("call_id"),
+                        "output": stdout,
+                    }
+                )
+                code_interpreter_calls.append(
+                    {
+                        "id": f"ci_{uuid.uuid4().hex}",
+                        "type": "code_interpreter_call",
+                        "status": "completed",
+                        "code": code,
+                        "container_id": container_id,
+                        "outputs": (
+                            [{"type": "logs", "logs": stdout}] if stdout else []
+                        ),
+                    }
+                )
+        except Exception:
+            await self._delete_container_for_cache_key(sandbox_key)
+            raise
+
+        optional_params = anthropic_messages_optional_request_params
+        request_patch = AgenticLoopRequestPatch(
+            model=model,
+            messages=input_list,
+            tools=self._get_followup_tools(
+                tools=optional_params.get("tools"),
+                call_type=CallTypes.responses,
+            ),
+            optional_params=self._get_followup_optional_params(optional_params),
+            kwargs=self._filter_agentic_loop_kwargs(kwargs),
+        )
+
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=request_patch,
+            metadata={
+                "tool_type": "code_interpreter",
+                "sandbox_key": sandbox_key or "",
+                "code_interpreter_calls": code_interpreter_calls,
+            },
+        )
+
+    async def _build_chat_completion_agentic_loop_plan(
+        self,
+        tools: dict[str, object],
+        model: str,
+        messages: list[dict],
+        optional_params: dict[str, object],
+        kwargs: dict[str, object],
+    ) -> AgenticLoopPlan:
+        await self._prune_expired_cache()
+        tool_calls = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
+        sandbox_key = cast(str | None, kwargs.get(_SANDBOX_KEY))
+        container, params = await self._get_or_create_container(cache_key=sandbox_key)
+
+        try:
+            container_id = cast(str | None, getattr(container, "id", None))
+            tool_results = [
+                await self._build_chat_completion_tool_result(
+                    container=container,
+                    params=params,
+                    tool_call=tool_call,
+                    container_id=container_id,
+                )
+                for tool_call in tool_calls
+            ]
+        except Exception:
+            await self._delete_container_for_cache_key(sandbox_key)
+            raise
+        tool_messages = [result[0] for result in tool_results]
+        code_interpreter_calls = [result[1] for result in tool_results]
+
+        request_patch = AgenticLoopRequestPatch(
+            model=model,
+            messages=list(messages)
+            + [self._build_chat_completion_assistant_message(tool_calls)]
+            + tool_messages,
+            tools=self._get_followup_tools(
+                tools=optional_params.get("tools"),
+                call_type=CallTypes.completion,
+            ),
+            optional_params=self._get_followup_optional_params(optional_params),
+            kwargs=self._filter_agentic_loop_kwargs(kwargs),
+        )
+
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=request_patch,
+            metadata={
+                "tool_type": "code_interpreter",
+                "sandbox_key": sandbox_key or "",
+                "code_interpreter_calls": code_interpreter_calls,
+                "response_format": "openai",
+            },
+        )
+
+    async def _build_chat_completion_tool_result(
+        self,
+        container: object,
+        params: dict[str, Any] | None,
+        tool_call: CodeExecutionToolCall,
+        container_id: str | None,
+    ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
+        arguments = tool_call.get("arguments", "")
+        code = self._parse_code(arguments)
+        stdout = await self._run_tool_call(
+            container=container, params=params, arguments=arguments
+        )
+        tool_call_id = (
+            tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex
+        )
+        return (
+            {
+                "role": "tool",
+                "tool_call_id": tool_call_id,
+                "content": stdout,
+            },
+            {
+                "id": f"ci_{uuid.uuid4().hex}",
+                "type": "code_interpreter_call",
+                "status": "completed",
+                "code": code,
+                "container_id": container_id,
+                "outputs": [{"type": "logs", "logs": stdout}] if stdout else [],
+            },
+        )
+
+    async def async_agentic_loop_cleanup_hook(
+        self, plan: AgenticLoopPlan, kwargs: dict
+    ) -> None:
+        metadata = plan.metadata or {} if plan else {}
+        await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+
+    @staticmethod
+    def _filter_agentic_loop_kwargs(kwargs: dict[str, object]) -> dict[str, object]:
+        return {
+            k: v
+            for k, v in kwargs.items()
+            if k not in {"litellm_logging_obj", "acompletion"}
+            and not is_interception_internal_key(
+                k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES
+            )
+        }
+
+    def _get_followup_tools(
+        self, tools: object, call_type: CallTypes | None
+    ) -> list[dict[str, Any]] | None:
+        if not isinstance(tools, list):
+            return None
+        return [
+            (
+                self._get_function_tool(call_type=call_type)
+                if isinstance(tool, dict) and tool.get("type") == "code_interpreter"
+                else tool
+            )
+            for tool in tools
+        ]
+
+    def _get_followup_optional_params(
+        self, optional_params: dict[str, object]
+    ) -> dict[str, object]:
+        drop_tool_choice = self._tool_choice_targets_code_interpreter(
+            optional_params.get("tool_choice")
+        )
+        return {
+            k: v
+            for k, v in optional_params.items()
+            if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
+        }
+
+    async def async_post_agentic_loop_response_hook(
+        self, response: Any, plan: AgenticLoopPlan, kwargs: dict
+    ) -> Any:
+        metadata = plan.metadata or {} if plan else {}
+        await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+
+        calls = metadata.get("code_interpreter_calls")
+        if not calls:
+            return response
+
+        is_dict = isinstance(response, dict)
+        output = (
+            response.get("output") if is_dict else getattr(response, "output", None)
+        )
+        if not isinstance(output, list):
+            return response
+
+        def _item_type(item: Any) -> Any:
+            return (
+                item.get("type")
+                if isinstance(item, dict)
+                else getattr(item, "type", None)
+            )
+
+        insert_at = next(
+            (i for i, item in enumerate(output) if _item_type(item) == "message"),
+            len(output),
+        )
+        new_output = output[:insert_at] + list(calls) + output[insert_at:]
+        if is_dict:
+            response["output"] = new_output
+        else:
+            response.output = new_output
+        return response
+
+    @staticmethod
+    def _parse_code(arguments: str) -> str:
+        try:
+            return json.loads(arguments).get("code", "") if arguments else ""
+        except (json.JSONDecodeError, TypeError, AttributeError):
+            return ""
+
+    async def _run_tool_call(
+        self, container: Any, params: dict[str, Any] | None, arguments: str
+    ) -> str:
+        try:
+            code = json.loads(arguments).get("code", "") if arguments else ""
+        except (json.JSONDecodeError, TypeError):
+            return "[invalid tool arguments: could not parse code]"
+
+        result = await self._run_code(container=container, params=params, code=code)
+        if getattr(result, "error", None):
+            error = result.error
+            message = (
+                error.get("value") or error.get("name")
+                if isinstance(error, dict)
+                else str(error)
+            )
+            return f"[execution error] {message}"
+        return getattr(result, "stdout", "") or ""
+
+    async def _get_or_create_container(
+        self, cache_key: str | None
+    ) -> tuple[Any, dict[str, Any] | None]:
+        if cache_key:
+            cached = self._container_cache.get(cache_key)
+            if cached is not None:
+                return cached[0], cached[1]
+
+        container, params = await self._create_container()
+        if cache_key:
+            self._container_cache[cache_key] = (container, params, time.time())
+        return container, params
+
+    async def _create_container(self) -> tuple[Any, dict[str, Any] | None]:
+        if self.sandbox_config is not None:
+            return await self.sandbox_config.acreate_sandbox(), None
+
+        params = _resolve_sandbox_tool(self.sandbox_tool_name)
+        if params is None:
+            raise ValueError(
+                "CodeInterpreterInterception: no sandbox available. Provide a "
+                "sandbox_config or configure a sandbox tool resolvable via "
+                "sandbox_tool_name."
+            )
+        container = await litellm.acreate_sandbox(
+            provider=params["sandbox_provider"],
+            api_key=params.get("api_key"),
+            api_base=params.get("api_base"),
+        )
+        return container, params
+
+    async def _run_code(
+        self, container: Any, params: dict[str, Any] | None, code: str
+    ) -> Any:
+        if self.sandbox_config is not None:
+            return await self.sandbox_config.arun_code(container=container, code=code)
+        if params is None:
+            raise ValueError(
+                "CodeInterpreterInterception: no sandbox available to run code."
+            )
+        return await litellm.arun_code(
+            provider=params["sandbox_provider"],
+            container=container,
+            code=code,
+            api_key=params.get("api_key"),
+        )
+
+    async def _delete_container(
+        self, container: Any, params: dict[str, Any] | None
+    ) -> None:
+        try:
+            if self.sandbox_config is not None:
+                await self.sandbox_config.adelete_sandbox(container=container)
+                return
+            if params is None:
+                return
+            await litellm.adelete_sandbox(
+                provider=params["sandbox_provider"],
+                container=container,
+                api_key=params.get("api_key"),
+                api_base=params.get("api_base"),
+            )
+        except Exception:
+            verbose_logger.exception(
+                "CodeInterpreterInterception: failed to delete sandbox container"
+            )
+
+    async def _delete_container_for_cache_key(self, cache_key: str | None) -> None:
+        if not cache_key:
+            return
+        cached = self._container_cache.pop(cache_key, None)
+        if cached is None:
+            return
+        await self._delete_container(container=cached[0], params=cached[1])
+
+    def _normalize_messages(self, messages: Any) -> list[dict[str, Any]]:
+        if isinstance(messages, str):
+            return [{"role": "user", "content": messages}]
+        if isinstance(messages, list):
+            return list(messages)
+        return []
+
+    def _extract_code_execution_tool_calls(
+        self, response: object
+    ) -> list[CodeExecutionToolCall]:
+        if isinstance(response, dict):
+            output = response.get("output", [])
+        else:
+            output = getattr(response, "output", []) or []
+        if not isinstance(output, list):
+            return []
+
+        return [
+            {
+                "call_id": (
+                    item.get("call_id")
+                    if isinstance(item, dict)
+                    else getattr(item, "call_id", None)
+                ),
+                "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                "arguments": (
+                    item.get("arguments")
+                    if isinstance(item, dict)
+                    else getattr(item, "arguments", "")
+                ),
+            }
+            for item in output
+            if self._is_code_execution_call(item)
+        ]
+
+    def _extract_chat_completion_code_execution_tool_calls(
+        self, response: ModelResponse | dict[str, Any]
+    ) -> list[CodeExecutionToolCall]:
+        model_response = self._to_model_response(response)
+        if model_response is None:
+            return []
+        choices = model_response.choices or []
+        if not choices:
+            return []
+        message = choices[0].message
+        tool_calls = message.tool_calls or []
+
+        return [
+            normalized
+            for tool_call in tool_calls
+            if (normalized := self._normalize_chat_completion_tool_call(tool_call))
+            is not None
+        ]
+
+    @staticmethod
+    def _normalize_chat_completion_tool_call(
+        tool_call: ChatCompletionMessageToolCall,
+    ) -> CodeExecutionToolCall | None:
+        if (
+            tool_call.type != "function"
+            or tool_call.function.name != LITELLM_CODE_EXECUTION_TOOL_NAME
+        ):
+            return None
+
+        arguments = tool_call.function.arguments
+        if isinstance(arguments, dict):
+            arguments = json.dumps(arguments)
+        elif not isinstance(arguments, str):
+            arguments = "" if arguments is None else str(arguments)
+
+        return {
+            "id": tool_call.id,
+            "call_id": tool_call.id,
+            "type": "function",
+            "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+            "arguments": arguments,
+        }
+
+    @staticmethod
+    def _build_chat_completion_assistant_message(
+        tool_calls: list[CodeExecutionToolCall],
+    ) -> ChatCompletionAssistantMessage:
+        return {
+            "role": "assistant",
+            "tool_calls": [
+                cast(
+                    ChatCompletionAssistantToolCall,
+                    {
+                        "id": tool_call.get("id"),
+                        "type": "function",
+                        "function": {
+                            "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                            "arguments": tool_call.get("arguments", ""),
+                        },
+                    },
+                )
+                for tool_call in tool_calls
+            ],
+        }
+
+    @staticmethod
+    def _to_model_response(
+        response: ModelResponse | dict[str, Any],
+    ) -> ModelResponse | None:
+        if isinstance(response, ModelResponse):
+            return response
+        try:
+            return ModelResponse(**response)
+        except (TypeError, ValidationError):
+            return None
+
+    def _is_code_execution_call(self, item: Any) -> bool:
+        if isinstance(item, dict):
+            return (
+                item.get("type") == "function_call"
+                and item.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+            )
+        return (
+            getattr(item, "type", None) == "function_call"
+            and getattr(item, "name", None) == LITELLM_CODE_EXECUTION_TOOL_NAME
+        )
+
+    async def _prune_expired_cache(self) -> None:
+        now = time.time()
+        expired = [
+            (cache_key, container, params)
+            for cache_key, (
+                container,
+                params,
+                created_at,
+            ) in self._container_cache.items()
+            if now - created_at > _CACHE_TTL_SECONDS
+        ]
+        for cache_key, container, params in expired:
+            self._container_cache.pop(cache_key, None)
+            await self._delete_container(container=container, params=params)

litellm/integrations/custom_logger.py

@@ -718,6 +718,24 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         """
         return response
 
+    async def async_agentic_loop_cleanup_hook(
+        self,
+        plan: AgenticLoopPlan,
+        kwargs: dict,
+    ) -> None:
+        """
+        Release resources held for an agentic-loop iteration.
+
+        Runs in a ``finally`` around the follow-up provider call, so it fires
+        whether the rerun returns normally, hits a loop safety abort, or raises
+        an upstream error. Implementations must be idempotent because the
+        post-response hook may already have released the same resource on the
+        success path. Use ``plan.metadata`` to locate what to clean up.
+
+        Default does nothing.
+        """
+        return None
+
     async def async_should_run_chat_completion_agentic_loop(
         self,
         response: Any,

litellm/integrations/otel/model/config.py

@@ -1,6 +1,7 @@
 """Typed configuration for the OpenTelemetry instrumentation."""
 
 from enum import Enum
+from functools import lru_cache
 from typing import Any, List
 
 from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
@@ -47,7 +48,12 @@ class _OTelV2Flag(BaseSettings):
     enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV))
 
 
+@lru_cache(maxsize=1)
 def is_otel_v2_enabled() -> bool:
+    # Resolved once at startup and cached: constructing the pydantic-settings
+    # model re-scans the environment and cost ~28us, which on the proxy hot path
+    # (auth, logging-callback setup) compounded into a measurable throughput
+    # regression. Tests that toggle the env must call ``is_otel_v2_enabled.cache_clear()``.
     return _OTelV2Flag().enabled
 
 

litellm/interactions/litellm_responses_transformation/transformation.py

@@ -300,9 +300,6 @@ class LiteLLMResponsesInteractionsConfig:
                 "total_output_tokens": getattr(usage, "output_tokens", 0),
             }
 
-        # Add role
-        interactions_response_dict["role"] = "model"
-
         # Add updated (same as created for now)
         interactions_response_dict["updated"] = created
 

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -0,0 +1,332 @@
+# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
+
+import json
+from typing import cast
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.integrations.custom_logger import (
+    CHAT_COMPLETION_AGENTIC_SURFACE,
+    NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
+    AgenticLoopPlan,
+    AgenticLoopRequestPatch,
+    is_interception_internal_key,
+)
+from litellm.types.utils import ModelResponse
+from litellm.utils import CustomStreamWrapper
+
+_FOLLOWUP_INTERNAL_PARAMS = frozenset(
+    (
+        "acompletion",
+        "litellm_logging_obj",
+        "custom_llm_provider",
+        "model_alias_map",
+        "stream_response",
+        "custom_prompt_dict",
+        "_agentic_loop_api_surface",
+    )
+)
+
+
+def _gate_overridden(callback: CustomLogger) -> bool:
+    base = CustomLogger.async_should_run_agentic_loop
+    func = type(callback).async_should_run_agentic_loop
+    return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
+
+
+def _build_plan_overridden(callback: CustomLogger) -> bool:
+    base = CustomLogger.async_build_agentic_loop_plan
+    func = type(callback).async_build_agentic_loop_plan
+    return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
+
+
+def _post_hook_overridden(callback: CustomLogger) -> bool:
+    base = CustomLogger.async_post_agentic_loop_response_hook
+    func = type(callback).async_post_agentic_loop_response_hook
+    return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
+
+
+def _coerce_int(value: object, default: int) -> int:
+    return int(value) if isinstance(value, (int, str)) else default
+
+
+def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
+    depth = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
+    max_loops = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
+    raw_fingerprints = kwargs.get("_agentic_loop_fingerprints")
+    fingerprints = (
+        [str(fp) for fp in raw_fingerprints]
+        if isinstance(raw_fingerprints, list)
+        else []
+    )
+    return depth, max_loops, fingerprints
+
+
+def _fingerprint_tools(tool_calls: object) -> str:
+    try:
+        return json.dumps(tool_calls, sort_keys=True, default=str)
+    except Exception:
+        return str(tool_calls)
+
+
+def _check_agentic_loop_safety(
+    tool_calls: object,
+    fingerprints: list[str],
+    depth: int,
+    max_loops: int,
+    model: str,
+) -> str:
+    fingerprint = _fingerprint_tools(tool_calls)
+    if fingerprint in fingerprints:
+        raise ValueError(
+            "Agentic loop detected repeated tool-call fingerprint; aborting rerun"
+        )
+    if depth >= max_loops:
+        raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
+    return fingerprint
+
+
+def _wrap_response_as_fake_stream(response: object) -> object:
+    if getattr(response, "object", None) == "chat.completion.chunk":
+        return response
+    if not hasattr(response, "choices"):
+        return response
+    from litellm.llms.base_llm.base_model_iterator import (
+        convert_model_response_to_streaming,
+    )
+
+    return convert_model_response_to_streaming(cast(ModelResponse, response))
+
+
+def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None:
+    metadata = kwargs_for_followup.get("litellm_metadata")
+    metadata = dict(metadata) if isinstance(metadata, dict) else {}
+    for key, value in kwargs_for_followup.items():
+        if (
+            key.startswith("_agentic_loop")
+            or key == "max_agentic_loops"
+            or is_interception_internal_key(key)
+        ):
+            metadata[key] = value
+    kwargs_for_followup["litellm_metadata"] = metadata
+
+
+def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]:
+    return {
+        k: v
+        for k, v in source.items()
+        if not is_interception_internal_key(
+            k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES
+        )
+        and k not in _FOLLOWUP_INTERNAL_PARAMS
+    }
+
+
+async def _execute_chat_completion_agentic_plan(
+    *,
+    plan: AgenticLoopPlan,
+    callback: CustomLogger,
+    model: str,
+    optional_params: dict[str, object],
+    kwargs: dict[str, object],
+    logging_obj: object,
+    custom_llm_provider: str,
+    depth: int,
+    max_loops: int,
+    fingerprints: list[str],
+    fingerprint: str,
+) -> object:
+    import litellm
+
+    patch = plan.request_patch or AgenticLoopRequestPatch()
+    if patch.messages is None:
+        raise ValueError("Agentic loop plan missing patched messages")
+
+    full_model_name = patch.model or model
+    if "/" not in full_model_name:
+        full_model_name = f"{custom_llm_provider}/{full_model_name}"
+
+    optional_params_for_followup = {**optional_params, **patch.optional_params}
+    if patch.tools is not None:
+        optional_params_for_followup["tools"] = patch.tools
+        if "tool_choice" not in patch.optional_params:
+            optional_params_for_followup.pop("tool_choice", None)
+
+    kwargs_for_followup = _filter_followup_kwargs(kwargs)
+    kwargs_for_followup.update(
+        {
+            k: v
+            for k, v in _filter_followup_kwargs(patch.kwargs).items()
+            if k not in optional_params_for_followup
+        }
+    )
+    kwargs_for_followup["_agentic_loop_depth"] = depth + 1
+    kwargs_for_followup["max_agentic_loops"] = max_loops
+    kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
+    _add_agentic_loop_metadata(kwargs_for_followup)
+
+    try:
+        response_followup = await litellm.acompletion(
+            model=full_model_name,
+            messages=patch.messages,
+            **optional_params_for_followup,
+            **kwargs_for_followup,
+        )
+        if _post_hook_overridden(callback):
+            try:
+                response_followup = (
+                    await callback.async_post_agentic_loop_response_hook(
+                        response=response_followup, plan=plan, kwargs=kwargs
+                    )
+                )
+            except Exception as e:
+                _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+                verbose_logger.exception(
+                    "LiteLLM.AgenticHookError: Exception in "
+                    "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s",
+                    _call_id,
+                    model,
+                    str(e),
+                )
+        if kwargs.get("_code_interpreter_interception_converted_stream") and not depth:
+            return _wrap_response_as_fake_stream(response_followup)
+        return response_followup
+    finally:
+        try:
+            await callback.async_agentic_loop_cleanup_hook(plan=plan, kwargs=kwargs)
+        except Exception as e:
+            _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+            verbose_logger.exception(
+                "LiteLLM.AgenticHookError: Exception in "
+                "async_agentic_loop_cleanup_hook [call_id=%s model=%s]: %s",
+                _call_id,
+                model,
+                str(e),
+            )
+
+
+async def maybe_run_chat_completion_agentic_loop(
+    *,
+    response: ModelResponse,
+    model: str,
+    messages: list,
+    optional_params: dict,
+    kwargs: dict,
+    logging_obj: object,
+    custom_llm_provider: str,
+    stream: bool,
+) -> ModelResponse | CustomStreamWrapper | None:
+    import litellm
+
+    callbacks = litellm.callbacks + (
+        getattr(logging_obj, "dynamic_success_callbacks", None) or []
+    )
+    depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
+    tools = optional_params.get("tools", [])
+
+    for callback in callbacks:
+        if not isinstance(callback, CustomLogger):
+            continue
+        if not _gate_overridden(callback):
+            continue
+
+        gate_kwargs = {
+            **kwargs,
+            "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
+            "custom_llm_provider": custom_llm_provider,
+        }
+        try:
+            should_run, tool_calls = await callback.async_should_run_agentic_loop(
+                response=response,
+                model=model,
+                messages=messages,
+                tools=tools,
+                stream=stream,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=gate_kwargs,
+            )
+        except Exception as e:
+            verbose_logger.exception(
+                "LiteLLM.AgenticHookError: Exception in chat completion agentic gate: %s",
+                str(e),
+            )
+            continue
+
+        if not should_run:
+            continue
+
+        fingerprint = _check_agentic_loop_safety(
+            tool_calls=tool_calls,
+            fingerprints=fingerprints,
+            depth=depth,
+            max_loops=max_loops,
+            model=model,
+        )
+
+        try:
+            plan_kwargs = {
+                **kwargs,
+                "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
+                "custom_llm_provider": custom_llm_provider,
+            }
+            if not _build_plan_overridden(callback):
+                return await callback.async_run_agentic_loop(
+                    tools=tool_calls,
+                    model=model,
+                    messages=messages,
+                    response=response,
+                    anthropic_messages_provider_config=None,
+                    anthropic_messages_optional_request_params=optional_params,
+                    logging_obj=logging_obj,
+                    stream=stream,
+                    kwargs=plan_kwargs,
+                )
+
+            plan = await callback.async_build_agentic_loop_plan(
+                tools=tool_calls,
+                model=model,
+                messages=messages,
+                response=response,
+                anthropic_messages_provider_config=None,
+                anthropic_messages_optional_request_params=optional_params,
+                logging_obj=logging_obj,
+                stream=stream,
+                kwargs=plan_kwargs,
+            )
+
+            if plan.response_override is not None:
+                return plan.response_override
+            if plan.terminate:
+                return response
+            if not plan.run_agentic_loop:
+                continue
+
+            return await _execute_chat_completion_agentic_plan(
+                plan=plan,
+                callback=callback,
+                model=model,
+                optional_params=optional_params,
+                kwargs=kwargs,
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                depth=depth,
+                max_loops=max_loops,
+                fingerprints=fingerprints,
+                fingerprint=fingerprint,
+            )
+        except Exception as e:
+            verbose_logger.exception(
+                "LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s",
+                str(e),
+            )
+
+    if (
+        kwargs.get("_code_interpreter_interception_converted_stream")
+        and not depth
+        and hasattr(response, "choices")
+    ):
+        return cast(
+            "ModelResponse | CustomStreamWrapper",
+            _wrap_response_as_fake_stream(response),
+        )
+    return None

litellm/litellm_core_utils/completion_timeout.py

@@ -6,10 +6,7 @@ from typing import Callable, Optional, Union
 
 import httpx
 
-from litellm.constants import (
-    COMPLETION_HTTP_FALLBACK_SECONDS,
-    DEFAULT_REQUEST_TIMEOUT_SECONDS,
-)
+from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS
 
 
 class CompletionTimeout:
@@ -22,17 +19,13 @@ class CompletionTimeout:
         """
         Used when ``model_timeout`` and kwargs timeouts are all unset.
 
-        ``global_timeout`` is :attr:`litellm.request_timeout` (numeric / string), not
-        :class:`httpx.Timeout`.
-
-        If it equals :data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS` (6000),
-        return :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`. Same if
-        ``None``. Otherwise return ``float(global_timeout)``.
+        ``global_timeout`` is the explicitly-configured ``litellm.request_timeout``
+        (numeric / string) or ``None`` when it was never set. ``None`` falls back to
+        :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`; any explicit value
+        (including ``6000``) is honored.
         """
         if global_timeout is None:
             return COMPLETION_HTTP_FALLBACK_SECONDS
-        if float(global_timeout) == float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
-            return COMPLETION_HTTP_FALLBACK_SECONDS
         return float(global_timeout)
 
     @staticmethod
@@ -50,11 +43,10 @@ class CompletionTimeout:
         1. ``model_timeout`` (call argument / merged ``litellm_params``)
         2. ``kwargs["timeout"]``
         3. ``kwargs["request_timeout"]``
-        4. Fallback from ``global_timeout`` (:attr:`litellm.request_timeout`) — if it is
-           the package default (6000), use 600 instead.
+        4. ``global_timeout`` (the explicitly-configured ``litellm.request_timeout``),
+           or 600 when nothing was configured.
 
         Coerce :class:`httpx.Timeout` when the provider does not support it.
-        Explicit ``6000`` on the model or in kwargs is kept as ``6000``.
         """
         resolved: Union[float, str, httpx.Timeout]
         if model_timeout is not None:

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -86,9 +86,7 @@ def get_supported_openai_params(
                 model=model
             )
         elif request_type == "transcription":
-            return litellm.FireworksAIAudioTranscriptionConfig().get_supported_openai_params(
-                model=model
-            )
+            return None
         else:
             return litellm.FireworksAIConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "nvidia_nim":
@@ -191,7 +189,9 @@ def get_supported_openai_params(
         )
     elif custom_llm_provider == "sambanova":
         if request_type == "embeddings":
-            litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(model=model)
+            return litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(
+                model=model
+            )
         else:
             return litellm.SambanovaConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "nebius":

litellm/litellm_core_utils/request_timeout_resolver.py

@@ -0,0 +1,29 @@
+"""Single source of truth for whether ``litellm.request_timeout`` was configured.
+
+``litellm.request_timeout`` always holds a value (the package default,
+:data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS`), so a bare read can't
+tell "user asked for this" from "nobody set it". This resolver answers that:
+
+* ``request_timeout_explicitly_set`` is the authoritative signal, set when the
+  value comes from the ``REQUEST_TIMEOUT`` env var or ``litellm_settings``.
+* A runtime value that differs from the package default (e.g. ``litellm.request_timeout
+  = 300`` in SDK code) is also treated as explicit, for backwards compatibility.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
+
+
+def get_configured_request_timeout() -> Optional[float]:
+    """Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
+    import litellm
+
+    timeout = float(litellm.request_timeout)
+    if litellm.request_timeout_explicitly_set:
+        return timeout
+    if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
+        return timeout
+    return None

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -12,6 +12,7 @@ class SensitiveDataMasker:
         visible_prefix: int = 4,
         visible_suffix: int = 4,
         mask_char: str = "*",
+        mask_short_values: bool = True,
     ):
         self.sensitive_patterns = sensitive_patterns or {
             "password",
@@ -38,12 +39,17 @@ class SensitiveDataMasker:
         self.visible_prefix = visible_prefix
         self.visible_suffix = visible_suffix
         self.mask_char = mask_char
+        self.mask_short_values = mask_short_values
 
     def _mask_value(self, value: str) -> str:
-        if not value or len(str(value)) < (self.visible_prefix + self.visible_suffix):
-            return value
-
         value_str = str(value)
+        if not value_str:
+            return value
+        if len(value_str) <= (self.visible_prefix + self.visible_suffix):
+            return (
+                self.mask_char * len(value_str) if self.mask_short_values else value_str
+            )
+
         masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
 
         # Handle the case where visible_suffix is 0 to avoid showing the entire string

litellm/litellm_core_utils/streaming_handler.py

@@ -6,6 +6,7 @@ import logging
 import threading
 import time
 import traceback
+from dataclasses import dataclass
 from typing import (
     Any,
     AsyncIterator,
@@ -97,6 +98,19 @@ def print_verbose(print_statement):
         pass
 
 
+@dataclass(frozen=True, slots=True)
+class _ProviderChunkParsed:
+    response_obj: dict[str, Any]
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderChunkEarlyReturn:
+    value: Any
+
+
+_ProviderChunkResult = Union[_ProviderChunkParsed, _ProviderChunkEarlyReturn]
+
+
 class CustomStreamWrapper:
     def __init__(
         self,
@@ -1145,381 +1159,392 @@ class CustomStreamWrapper:
                 del model_response.choices[0].delta.reasoning_content
         return
 
+    def _dispatch_provider_chunk(
+        self,
+        chunk: Any,
+        model_response: ModelResponseStream,
+        completion_obj: dict[str, Any],
+    ) -> _ProviderChunkResult:
+        response_obj: dict[str, Any] = {}
+        if (
+            isinstance(chunk, ModelResponseStream)
+            and self.custom_llm_provider is not None
+            and self.custom_llm_provider in litellm._custom_providers
+        ):
+            _has_content = bool(
+                chunk.choices
+                and chunk.choices[0].delta is not None
+                and (
+                    chunk.choices[0].delta.content or chunk.choices[0].delta.tool_calls
+                )
+            )
+            if self.received_finish_reason is not None:
+                if not _has_content:
+                    raise StopIteration
+            if chunk.choices and chunk.choices[0].finish_reason:
+                self.received_finish_reason = chunk.choices[0].finish_reason
+                if not _has_content:
+                    return _ProviderChunkEarlyReturn(None)
+                # Strip finish_reason from the content chunk so it appears
+                # only on the trailing empty-delta chunk (OpenAI spec).
+                # finish_reason_handler() will emit the proper terminal chunk.
+                chunk.choices[0].finish_reason = None  # type: ignore[assignment]
+            return _ProviderChunkEarlyReturn(chunk)
+
+        if (
+            isinstance(chunk, dict)
+            and generic_chunk_has_all_required_fields(
+                chunk=chunk
+            )  # check if chunk is a generic streaming chunk
+        ) or (
+            self.custom_llm_provider
+            and self.custom_llm_provider in litellm._custom_providers
+        ):
+            if self.received_finish_reason is not None:
+                _chunk_has_content = isinstance(chunk, dict) and (
+                    bool(chunk.get("text", ""))
+                    or chunk.get("tool_use") is not None
+                    # Usage-only final chunks are valid and needed to surface
+                    # finish_reason/usage to downstream translators.
+                    or chunk.get("usage") is not None
+                )
+                if not _chunk_has_content and (
+                    not isinstance(chunk, dict)
+                    or "provider_specific_fields" not in chunk
+                ):
+                    raise StopIteration
+            anthropic_response_obj: GChunk = cast(GChunk, chunk)
+            completion_obj["content"] = anthropic_response_obj["text"]
+            if anthropic_response_obj["is_finished"]:
+                self.received_finish_reason = anthropic_response_obj["finish_reason"]
+
+            if anthropic_response_obj["finish_reason"]:
+                self.intermittent_finish_reason = anthropic_response_obj[
+                    "finish_reason"
+                ]
+
+            if anthropic_response_obj["usage"] is not None:
+                setattr(
+                    model_response,
+                    "usage",
+                    litellm.Usage(**anthropic_response_obj["usage"]),
+                )
+
+            if (
+                "tool_use" in anthropic_response_obj
+                and anthropic_response_obj["tool_use"] is not None
+            ):
+                completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]]
+
+            if (
+                "provider_specific_fields" in anthropic_response_obj
+                and anthropic_response_obj["provider_specific_fields"] is not None
+            ):
+                for key, value in anthropic_response_obj[
+                    "provider_specific_fields"
+                ].items():
+                    setattr(model_response, key, value)
+
+            response_obj = cast(dict[str, Any], anthropic_response_obj)
+        elif self.model == "replicate" or self.custom_llm_provider == "replicate":
+            response_obj = self.handle_replicate_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif self.custom_llm_provider and self.custom_llm_provider == "predibase":
+            response_obj = self.handle_predibase_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif (
+            self.custom_llm_provider and self.custom_llm_provider == "baseten"
+        ):  # baseten doesn't provide streaming
+            completion_obj["content"] = self.handle_baseten_chunk(chunk)
+        elif (
+            self.custom_llm_provider and self.custom_llm_provider == "ai21"
+        ):  # ai21 doesn't provide streaming
+            response_obj = self.handle_ai21_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif self.custom_llm_provider and self.custom_llm_provider == "maritalk":
+            response_obj = self.handle_maritalk_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
+            completion_obj["content"] = chunk[0].outputs[0].text
+        elif (
+            self.custom_llm_provider and self.custom_llm_provider == "aleph_alpha"
+        ):  # aleph alpha doesn't provide streaming
+            response_obj = self.handle_aleph_alpha_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif self.custom_llm_provider == "nlp_cloud":
+            try:
+                response_obj = self.handle_nlp_cloud_chunk(chunk)
+                completion_obj["content"] = response_obj["text"]
+                if response_obj["is_finished"]:
+                    self.received_finish_reason = response_obj["finish_reason"]
+            except Exception as e:
+                if self.received_finish_reason:
+                    raise e
+                else:
+                    if self.sent_first_chunk is False:
+                        raise Exception("An unknown error occurred with the stream")
+                    self.received_finish_reason = "stop"
+        elif self.custom_llm_provider == "vertex_ai" and not isinstance(
+            chunk, ModelResponseStream
+        ):
+            chunk = cast(Any, chunk)
+            import proto  # type: ignore
+
+            if hasattr(chunk, "candidates") is True:
+                try:
+                    try:
+                        completion_obj["content"] = chunk.text  # type: ignore
+                    except Exception as e:
+                        original_exception = e
+                        if "Part has no text." in str(e):
+                            ## check for function calling
+                            function_call = (
+                                chunk.candidates[0].content.parts[0].function_call  # type: ignore
+                            )
+
+                            args_dict = {}
+
+                            # Check if it's a RepeatedComposite instance
+                            for key, val in function_call.args.items():
+                                if isinstance(
+                                    val,
+                                    proto.marshal.collections.repeated.RepeatedComposite,  # type: ignore
+                                ):
+                                    # If so, convert to list
+                                    args_dict[key] = [v for v in val]
+                                else:
+                                    args_dict[key] = val
+
+                            try:
+                                args_str = json.dumps(args_dict)
+                            except Exception as e:
+                                raise e
+                            _delta_obj = litellm.utils.Delta(
+                                content=None,
+                                tool_calls=[
+                                    {
+                                        "id": f"call_{str(uuid.uuid4())}",
+                                        "function": {
+                                            "arguments": args_str,
+                                            "name": function_call.name,
+                                        },
+                                        "type": "function",
+                                    }
+                                ],
+                            )
+                            _streaming_response = StreamingChoices(delta=_delta_obj)
+                            _model_response = ModelResponseStream()
+                            _model_response.choices = [_streaming_response]
+                            response_obj = {"original_chunk": _model_response}
+                        else:
+                            raise original_exception
+                    if (
+                        hasattr(chunk.candidates[0], "finish_reason")  # type: ignore
+                        and chunk.candidates[0].finish_reason.name  # type: ignore
+                        != "FINISH_REASON_UNSPECIFIED"
+                    ):  # every non-final chunk in vertex ai has this
+                        self.received_finish_reason = map_finish_reason(  # type: ignore
+                            chunk.candidates[0].finish_reason.name
+                        )
+                except Exception:
+                    if chunk.candidates[0].finish_reason.name == "SAFETY":  # type: ignore
+                        raise Exception(
+                            f"The response was blocked by VertexAI. {str(chunk)}"
+                        )
+            else:
+                completion_obj["content"] = str(chunk)
+        elif self.custom_llm_provider == "petals":
+            if self.completion_stream is None or len(self.completion_stream) == 0:
+                if self.received_finish_reason is not None:
+                    raise StopIteration
+                else:
+                    self.received_finish_reason = "stop"
+            chunk_size = 30
+            stream = cast(Any, self.completion_stream)
+            new_chunk = stream[:chunk_size]
+            completion_obj["content"] = new_chunk
+            self.completion_stream = stream[chunk_size:]
+        elif self.custom_llm_provider == "palm":
+            # fake streaming
+            response_obj = {}
+            if self.completion_stream is None or len(self.completion_stream) == 0:
+                if self.received_finish_reason is not None:
+                    raise StopIteration
+                else:
+                    self.received_finish_reason = "stop"
+            chunk_size = 30
+            stream = cast(Any, self.completion_stream)
+            new_chunk = stream[:chunk_size]
+            completion_obj["content"] = new_chunk
+            self.completion_stream = stream[chunk_size:]
+        elif self.custom_llm_provider == "triton":
+            response_obj = self.handle_triton_stream(chunk)
+            completion_obj["content"] = response_obj["text"]
+            print_verbose(f"completion obj content: {completion_obj['content']}")
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif self.custom_llm_provider == "text-completion-openai":
+            response_obj = self.handle_openai_text_completion_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            print_verbose(f"completion obj content: {completion_obj['content']}")
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+            if response_obj["usage"] is not None:
+                setattr(
+                    model_response,
+                    "usage",
+                    litellm.Usage(
+                        prompt_tokens=response_obj["usage"].prompt_tokens,
+                        completion_tokens=response_obj["usage"].completion_tokens,
+                        total_tokens=response_obj["usage"].total_tokens,
+                    ),
+                )
+        elif self.custom_llm_provider == "text-completion-codestral":
+            if not isinstance(chunk, str):
+                raise ValueError(f"chunk is not a string: {chunk}")
+            response_obj = cast(
+                dict[str, Any],
+                litellm.CodestralTextCompletionConfig()._chunk_parser(chunk),
+            )
+            completion_obj["content"] = response_obj["text"]
+            print_verbose(f"completion obj content: {completion_obj['content']}")
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+            if "usage" in response_obj is not None:
+                setattr(
+                    model_response,
+                    "usage",
+                    litellm.Usage(
+                        prompt_tokens=response_obj["usage"].prompt_tokens,
+                        completion_tokens=response_obj["usage"].completion_tokens,
+                        total_tokens=response_obj["usage"].total_tokens,
+                    ),
+                )
+        elif self.custom_llm_provider == "azure_text":
+            response_obj = self.handle_azure_text_completion_chunk(chunk)
+            completion_obj["content"] = response_obj["text"]
+            print_verbose(f"completion obj content: {completion_obj['content']}")
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        elif self.custom_llm_provider == "cached_response":
+            chunk = cast(ModelResponseStream, chunk)
+            response_obj = {
+                "text": chunk.choices[0].delta.content,
+                "is_finished": True,
+                "finish_reason": chunk.choices[0].finish_reason,
+                "original_chunk": chunk,
+                "tool_calls": (
+                    chunk.choices[0].delta.tool_calls
+                    if hasattr(chunk.choices[0].delta, "tool_calls")
+                    else None
+                ),
+            }
+
+            completion_obj["content"] = response_obj["text"]
+            if response_obj["tool_calls"] is not None:
+                completion_obj["tool_calls"] = response_obj["tool_calls"]
+            print_verbose(f"completion obj content: {completion_obj['content']}")
+            if hasattr(chunk, "id"):
+                model_response.id = chunk.id
+                self.response_id = chunk.id
+            if hasattr(chunk, "system_fingerprint"):
+                self.system_fingerprint = chunk.system_fingerprint
+            if response_obj["is_finished"]:
+                self.received_finish_reason = response_obj["finish_reason"]
+        else:  # openai / azure chat model
+            if self.custom_llm_provider in [
+                LlmProviders.AZURE.value,
+                LlmProviders.AZURE_AI.value,
+            ]:
+                if isinstance(chunk, BaseModel) and hasattr(chunk, "model"):
+                    # for azure, we need to pass the model from the original chunk
+                    self.model = getattr(chunk, "model", self.model)
+            response_obj = self.handle_openai_chat_completion_chunk(chunk)
+            if response_obj is None:
+                return _ProviderChunkEarlyReturn(None)
+            completion_obj["content"] = response_obj["text"]
+            self.intermittent_finish_reason = response_obj.get("finish_reason", None)
+            if response_obj["is_finished"]:
+                if response_obj["finish_reason"] == "error":
+                    raise Exception(
+                        "{} raised a streaming error - finish_reason: error, no content string given. Received Chunk={}".format(
+                            self.custom_llm_provider, response_obj
+                        )
+                    )
+                self.received_finish_reason = response_obj["finish_reason"]
+            if response_obj.get("original_chunk", None) is not None:
+                if hasattr(response_obj["original_chunk"], "id"):
+                    model_response = self.set_model_id(
+                        response_obj["original_chunk"].id, model_response
+                    )
+                if hasattr(response_obj["original_chunk"], "system_fingerprint"):
+                    model_response.system_fingerprint = response_obj[
+                        "original_chunk"
+                    ].system_fingerprint
+                    self.system_fingerprint = response_obj[
+                        "original_chunk"
+                    ].system_fingerprint
+            if response_obj["logprobs"] is not None:
+                model_response.choices[0].logprobs = response_obj["logprobs"]
+
+            if response_obj["usage"] is not None:
+                if isinstance(response_obj["usage"], dict):
+                    setattr(
+                        model_response,
+                        "usage",
+                        litellm.Usage(
+                            prompt_tokens=response_obj["usage"].get(
+                                "prompt_tokens", None
+                            )
+                            or None,
+                            completion_tokens=response_obj["usage"].get(
+                                "completion_tokens", None
+                            )
+                            or None,
+                            total_tokens=response_obj["usage"].get("total_tokens", None)
+                            or None,
+                        ),
+                    )
+                elif isinstance(response_obj["usage"], Usage):
+                    setattr(
+                        model_response,
+                        "usage",
+                        response_obj["usage"],
+                    )
+                elif isinstance(response_obj["usage"], BaseModel):
+                    setattr(
+                        model_response,
+                        "usage",
+                        litellm.Usage(**response_obj["usage"].model_dump()),
+                    )
+        return _ProviderChunkParsed(response_obj)
+
     def chunk_creator(self, chunk: Any):  # type: ignore
         if hasattr(chunk, "id"):
             self.response_id = chunk.id
         model_response = self.model_response_creator()
-        response_obj: Dict[str, Any] = {}
+        response_obj: dict[str, Any] = {}
         try:
             # return this for all models
-            completion_obj: Dict[str, Any] = {"content": ""}
-            from litellm.types.utils import GenericStreamingChunk as GChunk
-
-            if (
-                isinstance(chunk, ModelResponseStream)
-                and self.custom_llm_provider is not None
-                and self.custom_llm_provider in litellm._custom_providers
-            ):
-                _has_content = bool(
-                    chunk.choices
-                    and chunk.choices[0].delta is not None
-                    and (
-                        chunk.choices[0].delta.content
-                        or chunk.choices[0].delta.tool_calls
-                    )
-                )
-                if self.received_finish_reason is not None:
-                    if not _has_content:
-                        raise StopIteration
-                if chunk.choices and chunk.choices[0].finish_reason:
-                    self.received_finish_reason = chunk.choices[0].finish_reason
-                    if not _has_content:
-                        return None
-                    # Strip finish_reason from the content chunk so it appears
-                    # only on the trailing empty-delta chunk (OpenAI spec).
-                    # finish_reason_handler() will emit the proper terminal chunk.
-                    chunk.choices[0].finish_reason = None  # type: ignore[assignment]
-                return chunk
-
-            if (
-                isinstance(chunk, dict)
-                and generic_chunk_has_all_required_fields(
-                    chunk=chunk
-                )  # check if chunk is a generic streaming chunk
-            ) or (
-                self.custom_llm_provider
-                and self.custom_llm_provider in litellm._custom_providers
-            ):
-                if self.received_finish_reason is not None:
-                    _chunk_has_content = isinstance(chunk, dict) and (
-                        bool(chunk.get("text", ""))
-                        or chunk.get("tool_use") is not None
-                        # Usage-only final chunks are valid and needed to surface
-                        # finish_reason/usage to downstream translators.
-                        or chunk.get("usage") is not None
-                    )
-                    if not _chunk_has_content and (
-                        not isinstance(chunk, dict)
-                        or "provider_specific_fields" not in chunk
-                    ):
-                        raise StopIteration
-                anthropic_response_obj: GChunk = cast(GChunk, chunk)
-                completion_obj["content"] = anthropic_response_obj["text"]
-                if anthropic_response_obj["is_finished"]:
-                    self.received_finish_reason = anthropic_response_obj[
-                        "finish_reason"
-                    ]
-
-                if anthropic_response_obj["finish_reason"]:
-                    self.intermittent_finish_reason = anthropic_response_obj[
-                        "finish_reason"
-                    ]
-
-                if anthropic_response_obj["usage"] is not None:
-                    setattr(
-                        model_response,
-                        "usage",
-                        litellm.Usage(**anthropic_response_obj["usage"]),
-                    )
-
-                if (
-                    "tool_use" in anthropic_response_obj
-                    and anthropic_response_obj["tool_use"] is not None
-                ):
-                    completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]]
-
-                if (
-                    "provider_specific_fields" in anthropic_response_obj
-                    and anthropic_response_obj["provider_specific_fields"] is not None
-                ):
-                    for key, value in anthropic_response_obj[
-                        "provider_specific_fields"
-                    ].items():
-                        setattr(model_response, key, value)
-
-                response_obj = cast(Dict[str, Any], anthropic_response_obj)
-            elif self.model == "replicate" or self.custom_llm_provider == "replicate":
-                response_obj = self.handle_replicate_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif self.custom_llm_provider and self.custom_llm_provider == "predibase":
-                response_obj = self.handle_predibase_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif (
-                self.custom_llm_provider and self.custom_llm_provider == "baseten"
-            ):  # baseten doesn't provide streaming
-                completion_obj["content"] = self.handle_baseten_chunk(chunk)
-            elif (
-                self.custom_llm_provider and self.custom_llm_provider == "ai21"
-            ):  # ai21 doesn't provide streaming
-                response_obj = self.handle_ai21_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif self.custom_llm_provider and self.custom_llm_provider == "maritalk":
-                response_obj = self.handle_maritalk_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
-                completion_obj["content"] = chunk[0].outputs[0].text
-            elif (
-                self.custom_llm_provider and self.custom_llm_provider == "aleph_alpha"
-            ):  # aleph alpha doesn't provide streaming
-                response_obj = self.handle_aleph_alpha_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif self.custom_llm_provider == "nlp_cloud":
-                try:
-                    response_obj = self.handle_nlp_cloud_chunk(chunk)
-                    completion_obj["content"] = response_obj["text"]
-                    if response_obj["is_finished"]:
-                        self.received_finish_reason = response_obj["finish_reason"]
-                except Exception as e:
-                    if self.received_finish_reason:
-                        raise e
-                    else:
-                        if self.sent_first_chunk is False:
-                            raise Exception("An unknown error occurred with the stream")
-                        self.received_finish_reason = "stop"
-            elif self.custom_llm_provider == "vertex_ai" and not isinstance(
-                chunk, ModelResponseStream
-            ):
-                import proto  # type: ignore
-
-                if hasattr(chunk, "candidates") is True:
-                    try:
-                        try:
-                            completion_obj["content"] = chunk.text  # type: ignore
-                        except Exception as e:
-                            original_exception = e
-                            if "Part has no text." in str(e):
-                                ## check for function calling
-                                function_call = (
-                                    chunk.candidates[0].content.parts[0].function_call  # type: ignore
-                                )
-
-                                args_dict = {}
-
-                                # Check if it's a RepeatedComposite instance
-                                for key, val in function_call.args.items():
-                                    if isinstance(
-                                        val,
-                                        proto.marshal.collections.repeated.RepeatedComposite,  # type: ignore
-                                    ):
-                                        # If so, convert to list
-                                        args_dict[key] = [v for v in val]
-                                    else:
-                                        args_dict[key] = val
-
-                                try:
-                                    args_str = json.dumps(args_dict)
-                                except Exception as e:
-                                    raise e
-                                _delta_obj = litellm.utils.Delta(
-                                    content=None,
-                                    tool_calls=[
-                                        {
-                                            "id": f"call_{str(uuid.uuid4())}",
-                                            "function": {
-                                                "arguments": args_str,
-                                                "name": function_call.name,
-                                            },
-                                            "type": "function",
-                                        }
-                                    ],
-                                )
-                                _streaming_response = StreamingChoices(delta=_delta_obj)
-                                _model_response = ModelResponseStream()
-                                _model_response.choices = [_streaming_response]
-                                response_obj = {"original_chunk": _model_response}
-                            else:
-                                raise original_exception
-                        if (
-                            hasattr(chunk.candidates[0], "finish_reason")  # type: ignore
-                            and chunk.candidates[0].finish_reason.name  # type: ignore
-                            != "FINISH_REASON_UNSPECIFIED"
-                        ):  # every non-final chunk in vertex ai has this
-                            self.received_finish_reason = map_finish_reason(  # type: ignore
-                                chunk.candidates[0].finish_reason.name
-                            )
-                    except Exception:
-                        if chunk.candidates[0].finish_reason.name == "SAFETY":  # type: ignore
-                            raise Exception(
-                                f"The response was blocked by VertexAI. {str(chunk)}"
-                            )
-                else:
-                    completion_obj["content"] = str(chunk)
-            elif self.custom_llm_provider == "petals":
-                if self.completion_stream is None or len(self.completion_stream) == 0:
-                    if self.received_finish_reason is not None:
-                        raise StopIteration
-                    else:
-                        self.received_finish_reason = "stop"
-                chunk_size = 30
-                new_chunk = self.completion_stream[:chunk_size]  # type: ignore[index]
-                completion_obj["content"] = new_chunk
-                self.completion_stream = self.completion_stream[chunk_size:]  # type: ignore[index]
-            elif self.custom_llm_provider == "palm":
-                # fake streaming
-                response_obj = {}
-                if self.completion_stream is None or len(self.completion_stream) == 0:
-                    if self.received_finish_reason is not None:
-                        raise StopIteration
-                    else:
-                        self.received_finish_reason = "stop"
-                chunk_size = 30
-                new_chunk = self.completion_stream[:chunk_size]  # type: ignore[index]
-                completion_obj["content"] = new_chunk
-                self.completion_stream = self.completion_stream[chunk_size:]  # type: ignore[index]
-            elif self.custom_llm_provider == "triton":
-                response_obj = self.handle_triton_stream(chunk)
-                completion_obj["content"] = response_obj["text"]
-                print_verbose(f"completion obj content: {completion_obj['content']}")
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif self.custom_llm_provider == "text-completion-openai":
-                response_obj = self.handle_openai_text_completion_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                print_verbose(f"completion obj content: {completion_obj['content']}")
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-                if response_obj["usage"] is not None:
-                    setattr(
-                        model_response,
-                        "usage",
-                        litellm.Usage(
-                            prompt_tokens=response_obj["usage"].prompt_tokens,
-                            completion_tokens=response_obj["usage"].completion_tokens,
-                            total_tokens=response_obj["usage"].total_tokens,
-                        ),
-                    )
-            elif self.custom_llm_provider == "text-completion-codestral":
-                if not isinstance(chunk, str):
-                    raise ValueError(f"chunk is not a string: {chunk}")
-                response_obj = cast(
-                    Dict[str, Any],
-                    litellm.CodestralTextCompletionConfig()._chunk_parser(chunk),
-                )
-                completion_obj["content"] = response_obj["text"]
-                print_verbose(f"completion obj content: {completion_obj['content']}")
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-                if "usage" in response_obj is not None:
-                    setattr(
-                        model_response,
-                        "usage",
-                        litellm.Usage(
-                            prompt_tokens=response_obj["usage"].prompt_tokens,
-                            completion_tokens=response_obj["usage"].completion_tokens,
-                            total_tokens=response_obj["usage"].total_tokens,
-                        ),
-                    )
-            elif self.custom_llm_provider == "azure_text":
-                response_obj = self.handle_azure_text_completion_chunk(chunk)
-                completion_obj["content"] = response_obj["text"]
-                print_verbose(f"completion obj content: {completion_obj['content']}")
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            elif self.custom_llm_provider == "cached_response":
-                chunk = cast(ModelResponseStream, chunk)
-                response_obj = {
-                    "text": chunk.choices[0].delta.content,
-                    "is_finished": True,
-                    "finish_reason": chunk.choices[0].finish_reason,
-                    "original_chunk": chunk,
-                    "tool_calls": (
-                        chunk.choices[0].delta.tool_calls
-                        if hasattr(chunk.choices[0].delta, "tool_calls")
-                        else None
-                    ),
-                }
-
-                completion_obj["content"] = response_obj["text"]
-                if response_obj["tool_calls"] is not None:
-                    completion_obj["tool_calls"] = response_obj["tool_calls"]
-                print_verbose(f"completion obj content: {completion_obj['content']}")
-                if hasattr(chunk, "id"):
-                    model_response.id = chunk.id
-                    self.response_id = chunk.id
-                if hasattr(chunk, "system_fingerprint"):
-                    self.system_fingerprint = chunk.system_fingerprint
-                if response_obj["is_finished"]:
-                    self.received_finish_reason = response_obj["finish_reason"]
-            else:  # openai / azure chat model
-                if self.custom_llm_provider in [
-                    LlmProviders.AZURE.value,
-                    LlmProviders.AZURE_AI.value,
-                ]:
-                    if isinstance(chunk, BaseModel) and hasattr(chunk, "model"):
-                        # for azure, we need to pass the model from the original chunk
-                        self.model = getattr(chunk, "model", self.model)
-                response_obj = self.handle_openai_chat_completion_chunk(chunk)
-                if response_obj is None:
-                    return
-                completion_obj["content"] = response_obj["text"]
-                self.intermittent_finish_reason = response_obj.get(
-                    "finish_reason", None
-                )
-                if response_obj["is_finished"]:
-                    if response_obj["finish_reason"] == "error":
-                        raise Exception(
-                            "{} raised a streaming error - finish_reason: error, no content string given. Received Chunk={}".format(
-                                self.custom_llm_provider, response_obj
-                            )
-                        )
-                    self.received_finish_reason = response_obj["finish_reason"]
-                if response_obj.get("original_chunk", None) is not None:
-                    if hasattr(response_obj["original_chunk"], "id"):
-                        model_response = self.set_model_id(
-                            response_obj["original_chunk"].id, model_response
-                        )
-                    if hasattr(response_obj["original_chunk"], "system_fingerprint"):
-                        model_response.system_fingerprint = response_obj[
-                            "original_chunk"
-                        ].system_fingerprint
-                        self.system_fingerprint = response_obj[
-                            "original_chunk"
-                        ].system_fingerprint
-                if response_obj["logprobs"] is not None:
-                    model_response.choices[0].logprobs = response_obj["logprobs"]
-
-                if response_obj["usage"] is not None:
-                    if isinstance(response_obj["usage"], dict):
-                        setattr(
-                            model_response,
-                            "usage",
-                            litellm.Usage(
-                                prompt_tokens=response_obj["usage"].get(
-                                    "prompt_tokens", None
-                                )
-                                or None,
-                                completion_tokens=response_obj["usage"].get(
-                                    "completion_tokens", None
-                                )
-                                or None,
-                                total_tokens=response_obj["usage"].get(
-                                    "total_tokens", None
-                                )
-                                or None,
-                            ),
-                        )
-                    elif isinstance(response_obj["usage"], Usage):
-                        setattr(
-                            model_response,
-                            "usage",
-                            response_obj["usage"],
-                        )
-                    elif isinstance(response_obj["usage"], BaseModel):
-                        setattr(
-                            model_response,
-                            "usage",
-                            litellm.Usage(**response_obj["usage"].model_dump()),
-                        )
+            completion_obj: dict[str, Any] = {"content": ""}
+            dispatch_result = self._dispatch_provider_chunk(
+                chunk=chunk,
+                model_response=model_response,
+                completion_obj=completion_obj,
+            )
+            if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
+                return dispatch_result.value
+            response_obj = dispatch_result.response_obj
 
             model_response.model = self.model
             ## FUNCTION CALL PARSING
@@ -1980,11 +2005,29 @@ class CustomStreamWrapper:
 
         except StopIteration:
             if self.sent_last_chunk is True:
-                complete_streaming_response = litellm.stream_chunk_builder(
-                    chunks=self.chunks,
-                    messages=self.messages,
-                    logging_obj=self.logging_obj,
-                )
+                try:
+                    complete_streaming_response = litellm.stream_chunk_builder(
+                        chunks=self.chunks,
+                        messages=self.messages,
+                        logging_obj=self.logging_obj,
+                    )
+                except Exception as e:
+                    # stream_chunk_builder can re-raise (as APIError) on large agentic
+                    # streams. The raise originates inside this except-StopIteration block,
+                    # so the sibling `except Exception` below does not catch it; it would
+                    # escape __next__ and drop the request from SpendLogs. Recover
+                    # best-effort usage from the raw chunks so cost is still tracked
+                    verbose_logger.warning(
+                        "stream_chunk_builder raised at end-of-stream (%s); logging "
+                        "best-effort usage from chunks.",
+                        str(e),
+                    )
+                    try:
+                        complete_streaming_response = self.model_response_creator(
+                            chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+                        )
+                    except Exception:
+                        complete_streaming_response = None
 
                 response = self.model_response_creator()
                 if complete_streaming_response is not None:
@@ -2209,11 +2252,27 @@ class CustomStreamWrapper:
         except (StopAsyncIteration, StopIteration):
             if self.sent_last_chunk is True:
                 # log the final chunk with accurate streaming values
-                complete_streaming_response = litellm.stream_chunk_builder(
-                    chunks=self.chunks,
-                    messages=self.messages,
-                    logging_obj=self.logging_obj,
-                )
+                try:
+                    complete_streaming_response = litellm.stream_chunk_builder(
+                        chunks=self.chunks,
+                        messages=self.messages,
+                        logging_obj=self.logging_obj,
+                    )
+                except Exception as e:
+                    # see sync __next__: a raise from stream_chunk_builder inside this
+                    # except handler escapes __anext__ and drops the request from SpendLogs.
+                    # Recover best-effort usage from the raw chunks so cost is still tracked
+                    verbose_logger.warning(
+                        "stream_chunk_builder raised at end-of-stream (%s); logging "
+                        "best-effort usage from chunks.",
+                        str(e),
+                    )
+                    try:
+                        complete_streaming_response = self.model_response_creator(
+                            chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+                        )
+                    except Exception:
+                        complete_streaming_response = None
 
                 response = self.model_response_creator()
                 if complete_streaming_response is not None:

litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py

@@ -84,8 +84,14 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
         )
         # Optional routing overrides for the advisor sub-call (e.g. proxy routing).
         # If not set in the tool definition, litellm resolves from env vars.
-        advisor_api_key: Optional[str] = advisor_tool.get("api_key")
-        advisor_api_base: Optional[str] = advisor_tool.get("api_base")
+        # The advisor tool is caller-controlled; only honor a client-supplied
+        # api_base/api_key when the proxy has enabled clientside credentials,
+        # otherwise let litellm resolve from server config.
+        advisor_api_key: Optional[str] = None
+        advisor_api_base: Optional[str] = None
+        if _allow_client_side_advisor_credentials():
+            advisor_api_key = advisor_tool.get("api_key")
+            advisor_api_base = advisor_tool.get("api_base")
 
         # Build the synthetic tool definition the provider will receive.
         synthetic_advisor_tool = _make_synthetic_advisor_tool()
@@ -181,6 +187,20 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
 # ---------------------------------------------------------------------------
 
 
+def _allow_client_side_advisor_credentials() -> bool:
+    """Whether a caller-supplied advisor api_base/api_key may be honored.
+
+    Gated on the proxy's ``allow_client_side_credentials`` opt-in. When the
+    interceptor runs outside the proxy (SDK use), there is no admin boundary
+    to protect, so client-supplied routing is allowed.
+    """
+    try:
+        from litellm.proxy.proxy_server import general_settings
+    except (ImportError, ModuleNotFoundError):
+        return True
+    return general_settings.get("allow_client_side_credentials") is True
+
+
 def _make_synthetic_advisor_tool() -> Dict:
     """Build a regular tool definition the executor provider can understand."""
     return {

litellm/llms/apiserpent/search/transformation.py

@@ -53,7 +53,13 @@ class APISerpentSearchConfig(BaseSearchConfig):
         api_base: Optional[str] = None,
         **kwargs,
     ) -> Dict:
-        api_key = api_key or get_secret_str("APISERPENT_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("APISERPENT_API_KEY",),
+            base_env_var="APISERPENT_API_BASE",
+            default_api_base=APISERPENT_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "APISERPENT_API_KEY is not set. Set `APISERPENT_API_KEY` environment variable."

litellm/llms/base_llm/sandbox/transformation.py

@@ -8,10 +8,14 @@ run code -> delete container; `code_interpreter_tool` combines all three.
 
 from typing import Any, Union
 
+import httpx
+
 from pydantic import Field, PrivateAttr
 
 from litellm.types.llms.base import LiteLLMPydanticObjectBase
 
+SANDBOX_MAX_OUTPUT_BYTES = 10 * 1024 * 1024
+
 
 class ContainerHandle(LiteLLMPydanticObjectBase):
     """A live sandbox container. Carries everything needed to reach it again."""
@@ -53,7 +57,7 @@ class BaseSandboxConfig:
         *,
         template: str | None = None,
         timeout: int | None = None,
-        allow_internet_access: bool = True,
+        allow_internet_access: bool | None = None,
         api_key: str | None = None,
         **kwargs,
     ) -> ContainerHandle:
@@ -77,3 +81,16 @@ class BaseSandboxConfig:
         **kwargs,
     ) -> bool:
         raise NotImplementedError("adelete_sandbox must be implemented by provider")
+
+    async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
+        lines: list[str] = []
+        total = 0
+        async for line in response.aiter_lines():
+            total += len(line.encode("utf-8"))
+            if total > SANDBOX_MAX_OUTPUT_BYTES:
+                raise ValueError(
+                    f"Sandbox output exceeded {SANDBOX_MAX_OUTPUT_BYTES} bytes; aborting "
+                    "to avoid unbounded memory use."
+                )
+            lines.append(line)
+        return lines

litellm/llms/base_llm/search/transformation.py

@@ -3,11 +3,13 @@ Base Search transformation configuration.
 """
 
 from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
+from urllib.parse import urlsplit
 
 import httpx
 from pydantic import PrivateAttr
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.base import LiteLLMPydanticObjectBase
 
 if TYPE_CHECKING:
@@ -16,6 +18,29 @@ else:
     LiteLLMLoggingObj = Any
 
 
+def _search_host(url: str) -> str:
+    return urlsplit(url).netloc.lower()
+
+
+def _is_trusted_search_api_base(
+    caller_api_base: str,
+    default_api_base: str | None,
+    base_env_var: str | None,
+) -> bool:
+    candidate = _search_host(caller_api_base)
+    if not candidate:
+        return False
+    trusted = {
+        _search_host(base)
+        for base in (
+            default_api_base,
+            get_secret_str(base_env_var) if base_env_var else None,
+        )
+        if base
+    }
+    return candidate in trusted
+
+
 class SearchResult(LiteLLMPydanticObjectBase):
     """Single search result."""
 
@@ -86,6 +111,60 @@ class BaseSearchConfig:
             "max_tokens_per_page",
         }
 
+    def _assert_trusted_api_base_for_server_credential(
+        self,
+        caller_api_base: str | None,
+        default_api_base: str | None,
+        base_env_var: str | None,
+        credential_name: str,
+    ) -> None:
+        """
+        Block sending a server-managed credential to a caller-chosen host.
+
+        A caller-supplied api_base is honored when constructing the request URL, so
+        falling back to a server-configured secret while the caller controls the host
+        leaks that secret. The provider default and the operator's own api_base
+        override are the only trusted destinations for a server-managed credential.
+        """
+        if not caller_api_base:
+            return
+        if _is_trusted_search_api_base(caller_api_base, default_api_base, base_env_var):
+            return
+        raise ValueError(
+            f"Refusing to send the server-configured {credential_name} to the "
+            f"caller-supplied api_base '{caller_api_base}'. Pass an explicit api_key "
+            f"when overriding api_base for this search provider."
+        )
+
+    def resolve_server_api_key(
+        self,
+        *,
+        caller_api_key: str | None,
+        caller_api_base: str | None,
+        key_env_vars: tuple[str, ...],
+        base_env_var: str | None,
+        default_api_base: str | None,
+    ) -> str | None:
+        """
+        Resolve a single-secret search API key, falling back to a server-managed
+        secret only when the request targets a trusted host.
+
+        Returns the caller's key when provided, otherwise the first set
+        server-managed secret (or None when none is set, for keyless providers).
+        """
+        if caller_api_key:
+            return caller_api_key
+        server_key = next(
+            (key for key in (get_secret_str(var) for var in key_env_vars) if key),
+            None,
+        )
+        if server_key is None:
+            return None
+        self._assert_trusted_api_base_for_server_credential(
+            caller_api_base, default_api_base, base_env_var, key_env_vars[0]
+        )
+        return server_key
+
     def validate_environment(
         self,
         headers: Dict,

litellm/llms/bedrock/base_aws_llm.py

@@ -10,7 +10,6 @@ from typing import (
     Callable,
     ClassVar,
     Dict,
-    List,
     Literal,
     Optional,
     Tuple,
@@ -210,32 +209,11 @@ class BaseAWSLLM:
         """
         Return a boto3.Credentials object
         """
-        ## CHECK IS  'os.environ/' passed in
-        params_to_check: List[Optional[str]] = [
-            aws_access_key_id,
-            aws_secret_access_key,
-            aws_session_token,
-            aws_region_name,
-            aws_session_name,
-            aws_profile_name,
-            aws_role_name,
-            aws_web_identity_token,
-            aws_sts_endpoint,
-            aws_external_id,
-        ]
-
-        # Iterate over parameters and update if needed
-        for i, param in enumerate(params_to_check):
-            if param and param.startswith("os.environ/"):
-                _v = get_secret(param)
-                if _v is not None and isinstance(_v, str):
-                    params_to_check[i] = _v
-            elif param is None:  # check if uppercase value in env
-                key = self.aws_authentication_params[i]
-                if key.upper() in os.environ:
-                    params_to_check[i] = os.getenv(key.upper())
-
-        # Assign updated values back to parameters
+        # Only config-sourced credentials are expanded against the environment.
+        # os.environ/<VAR> references in the model config are resolved at load time,
+        # so any reference still present at this point is caller-supplied input and is
+        # left as-is rather than expanded into a process environment variable. Each
+        # unset param falls back to its matching fixed AWS_* ambient env var.
         (
             aws_access_key_id,
             aws_secret_access_key,
@@ -247,7 +225,21 @@ class BaseAWSLLM:
             aws_web_identity_token,
             aws_sts_endpoint,
             aws_external_id,
-        ) = params_to_check
+        ) = tuple(
+            value if value is not None else os.getenv(env_var)
+            for value, env_var in (
+                (aws_access_key_id, "AWS_ACCESS_KEY_ID"),
+                (aws_secret_access_key, "AWS_SECRET_ACCESS_KEY"),
+                (aws_session_token, "AWS_SESSION_TOKEN"),
+                (aws_region_name, "AWS_REGION_NAME"),
+                (aws_session_name, "AWS_SESSION_NAME"),
+                (aws_profile_name, "AWS_PROFILE_NAME"),
+                (aws_role_name, "AWS_ROLE_NAME"),
+                (aws_web_identity_token, "AWS_WEB_IDENTITY_TOKEN"),
+                (aws_sts_endpoint, "AWS_STS_ENDPOINT"),
+                (aws_external_id, "AWS_EXTERNAL_ID"),
+            )
+        )
 
         verbose_logger.debug(
             "in get credentials\n"
@@ -845,6 +837,20 @@ class BaseAWSLLM:
             f"IN Web Identity Token: {aws_web_identity_token} | Role Name: {aws_role_name} | Session Name: {aws_session_name}"
         )
 
+        # get_secret() expands environment-variable references (an os.environ/<VAR>
+        # prefix, or a bare name matching an environment variable). Config-sourced
+        # references are expanded at load time, so such a reference reaching here is
+        # caller-supplied input; reject it rather than expanding a process-environment
+        # value for use as the token.
+        if (
+            aws_web_identity_token.startswith("os.environ/")
+            or aws_web_identity_token in os.environ
+        ):
+            raise AwsAuthError(
+                message="Invalid web identity token reference.",
+                status_code=400,
+            )
+
         oidc_token = get_secret(aws_web_identity_token)
 
         if oidc_token is None:

litellm/llms/bedrock/chat/invoke_handler.py

@@ -70,6 +70,7 @@ from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import (
     BedrockError,
     ModelResponseIterator,
+    build_bedrock_stream_error,
     get_bedrock_response_stream_shape,
     get_bedrock_tool_name,
 )
@@ -1841,23 +1842,7 @@ class AWSEventStreamDecoder:
         parsed_response = self.parser.parse(response_dict, response_stream_shape)
 
         if response_dict["status_code"] != 200:
-            decoded_body = response_dict["body"].decode()
-            if isinstance(decoded_body, dict):
-                error_message = decoded_body.get("message")
-            elif isinstance(decoded_body, str):
-                error_message = decoded_body
-            else:
-                error_message = ""
-            exception_status = response_dict["headers"].get(":exception-type")
-            error_message = exception_status + " " + error_message
-            raise BedrockError(
-                status_code=response_dict["status_code"],
-                message=(
-                    json.dumps(error_message)
-                    if isinstance(error_message, dict)
-                    else error_message
-                ),
-            )
+            raise build_bedrock_stream_error(response_dict, response_stream_shape)
         if "chunk" in parsed_response:
             chunk = parsed_response.get("chunk")
             if not chunk:

litellm/llms/bedrock/common_utils.py

@@ -7,9 +7,21 @@ Common utilities used across bedrock chat/embedding/image generation
 import functools
 import json
 import os
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    TypedDict,
+    Union,
+)
 
 if TYPE_CHECKING:
+    from botocore.model import Shape
+
     from litellm.types.llms.bedrock import BedrockCreateBatchRequest
 
 import httpx
@@ -1132,6 +1144,39 @@ def get_bedrock_response_stream_shape():
     return _load_bedrock_response_stream_shape()
 
 
+class BedrockEventStreamResponseDict(TypedDict):
+    status_code: int
+    headers: Mapping[str, str]
+    body: bytes
+
+
+def build_bedrock_stream_error(
+    response_dict: BedrockEventStreamResponseDict,
+    response_stream_shape: Shape | None,
+) -> BedrockError:
+    """Build a BedrockError for a non-200 event-stream error event.
+
+    botocore hard-codes HTTP 400 on every mid-stream error event, so the modeled
+    ResponseStream member's httpStatusCode is the real status. Resolve it from the
+    shape and fall back to the raw status when the type is not modeled.
+    """
+    exception_type = response_dict["headers"].get(":exception-type")
+    decoded_body = response_dict["body"].decode()
+    message = f"{exception_type} {decoded_body}" if exception_type else decoded_body
+
+    status_code = response_dict["status_code"]
+    if exception_type is not None and response_stream_shape is not None:
+        member = response_stream_shape.members.get(exception_type)
+        if member is not None:
+            modeled_status = (
+                (member.metadata or {}).get("error", {}).get("httpStatusCode")
+            )
+            if modeled_status is not None:
+                status_code = int(modeled_status)
+
+    return BedrockError(status_code=status_code, message=message)
+
+
 class BedrockEventStreamDecoderBase:
     """
     Base class for event stream decoding for Bedrock
@@ -1156,23 +1201,7 @@ class BedrockEventStreamDecoderBase:
         parsed_response = self.parser.parse(response_dict, response_stream_shape)
 
         if response_dict["status_code"] != 200:
-            decoded_body = response_dict["body"].decode()
-            if isinstance(decoded_body, dict):
-                error_message = decoded_body.get("message")
-            elif isinstance(decoded_body, str):
-                error_message = decoded_body
-            else:
-                error_message = ""
-            exception_status = response_dict["headers"].get(":exception-type")
-            error_message = exception_status + " " + error_message
-            raise BedrockError(
-                status_code=response_dict["status_code"],
-                message=(
-                    json.dumps(error_message)
-                    if isinstance(error_message, dict)
-                    else error_message
-                ),
-            )
+            raise build_bedrock_stream_error(response_dict, response_stream_shape)
         if "chunk" in parsed_response:
             chunk = parsed_response.get("chunk")
             if not chunk:

litellm/llms/brave/search/transformation.py

@@ -115,7 +115,13 @@ class BraveSearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("BRAVE_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("BRAVE_API_KEY",),
+            base_env_var="BRAVE_API_BASE",
+            default_api_base=self.BRAVE_API_BASE,
+        )
 
         if not api_key:
             raise ValueError(

litellm/llms/cloudflare/chat/transformation.py

@@ -1,26 +1,15 @@
-import json
-import time
-from typing import AsyncIterator, Iterator, List, Optional, Union
+from typing import List, Optional, Union
 
 import httpx
 
-import litellm
-from litellm.litellm_core_utils.url_utils import encode_url_path_segments
-from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
-from litellm.llms.base_llm.chat.transformation import (
-    BaseConfig,
-    BaseLLMException,
-    LiteLLMLoggingObj,
+from litellm._logging import verbose_logger
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.secret_managers.main import (
+    get_secret_str,
+    normalize_nonempty_secret_str,
 )
-from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import (
-    ChatCompletionToolCallChunk,
-    ChatCompletionUsageBlock,
-    GenericStreamingChunk,
-    ModelResponse,
-    Usage,
-)
 
 
 class CloudflareError(BaseLLMException):
@@ -34,26 +23,46 @@ class CloudflareError(BaseLLMException):
             message=message,
             request=self.request,
             response=self.response,
-        )  # Call the base class constructor with the parameters it needs
+        )
 
 
-class CloudflareChatConfig(BaseConfig):
-    max_tokens: Optional[int] = None
-    stream: Optional[bool] = None
-
-    def __init__(
+class CloudflareChatConfig(OpenAIGPTConfig):
+    def get_complete_url(
         self,
-        max_tokens: Optional[int] = None,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
         stream: Optional[bool] = None,
-    ) -> None:
-        locals_ = locals().copy()
-        for key, value in locals_.items():
-            if key != "self" and value is not None:
-                setattr(self.__class__, key, value)
+    ) -> str:
+        return super().get_complete_url(
+            api_base=self._resolve_api_base(api_base),
+            api_key=api_key,
+            model=model,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            stream=stream,
+        )
 
-    @classmethod
-    def get_config(cls):
-        return super().get_config()
+    @staticmethod
+    def _resolve_api_base(api_base: Optional[str]) -> str:
+        if not api_base:
+            account_id = normalize_nonempty_secret_str(
+                get_secret_str("CLOUDFLARE_ACCOUNT_ID")
+            )
+            if account_id is None:
+                raise ValueError(
+                    "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID in the environment or pass api_base explicitly"
+                )
+            return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1"
+        trimmed = api_base.rstrip("/")
+        if trimmed.endswith("/ai/run"):
+            verbose_logger.warning(
+                "Cloudflare api_base ending in '/ai/run' is the legacy Workers AI path and no longer serves OpenAI-compatible requests; rewriting to the '/ai/v1' endpoint"
+            )
+            return f"{trimmed[: -len('/ai/run')]}/ai/v1"
+        return api_base
 
     def validate_environment(
         self,
@@ -67,107 +76,18 @@ class CloudflareChatConfig(BaseConfig):
     ) -> dict:
         if api_key is None:
             raise ValueError(
-                "Missing CloudflareError API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params"
+                "Missing Cloudflare API Key - A call is being made to cloudflare but no key is set either in the environment variables or via params"
             )
-        headers = {
-            "accept": "application/json",
-            "content-type": "apbplication/json",
-            "Authorization": "Bearer " + api_key,
-        }
-        return headers
-
-    def get_complete_url(
-        self,
-        api_base: Optional[str],
-        api_key: Optional[str],
-        model: str,
-        optional_params: dict,
-        litellm_params: dict,
-        stream: Optional[bool] = None,
-    ) -> str:
-        if api_base is None:
-            account_id = get_secret_str("CLOUDFLARE_ACCOUNT_ID")
-            api_base = (
-                f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
-            )
-        encoded_model = encode_url_path_segments(model, field_name="model")
-        return api_base + encoded_model
-
-    def get_supported_openai_params(self, model: str) -> List[str]:
-        return [
-            "stream",
-            "max_tokens",
-        ]
-
-    def map_openai_params(
-        self,
-        non_default_params: dict,
-        optional_params: dict,
-        model: str,
-        drop_params: bool,
-    ) -> dict:
-        supported_openai_params = self.get_supported_openai_params(model=model)
-        for param, value in non_default_params.items():
-            if param == "max_completion_tokens":
-                optional_params["max_tokens"] = value
-            elif param in supported_openai_params:
-                optional_params[param] = value
-        return optional_params
-
-    def transform_request(
-        self,
-        model: str,
-        messages: List[AllMessageValues],
-        optional_params: dict,
-        litellm_params: dict,
-        headers: dict,
-    ) -> dict:
-        config = litellm.CloudflareChatConfig.get_config()
-        for k, v in config.items():
-            if k not in optional_params:
-                optional_params[k] = v
-
-        data = {
-            "messages": messages,
-            **optional_params,
-        }
-        return data
-
-    def transform_response(
-        self,
-        model: str,
-        raw_response: httpx.Response,
-        model_response: ModelResponse,
-        logging_obj: LiteLLMLoggingObj,
-        request_data: dict,
-        messages: List[AllMessageValues],
-        optional_params: dict,
-        litellm_params: dict,
-        encoding: str,
-        api_key: Optional[str] = None,
-        json_mode: Optional[bool] = None,
-    ) -> ModelResponse:
-        completion_response = raw_response.json()
-
-        # Support both "response" and "response_text" keys (newer models like Nemotron use "response_text")
-        result = completion_response["result"]
-        model_response.choices[0].message.content = result.get("response") if result.get("response") is not None else result.get("response_text", "")  # type: ignore
-
-        prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model)
-        completion_tokens = len(
-            encoding.encode(model_response["choices"][0]["message"].get("content", ""))
+        return super().validate_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=api_key,
+            api_base=api_base,
         )
 
-        model_response.created = int(time.time())
-        model_response.model = "cloudflare/" + model
-        usage = Usage(
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=prompt_tokens + completion_tokens,
-        )
-        setattr(model_response, "usage", usage)
-        return model_response
-
     def get_error_class(
         self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
     ) -> BaseLLMException:
@@ -175,48 +95,3 @@ class CloudflareChatConfig(BaseConfig):
             status_code=status_code,
             message=error_message,
         )
-
-    def get_model_response_iterator(
-        self,
-        streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
-        sync_stream: bool,
-        json_mode: Optional[bool] = False,
-    ):
-        return CloudflareChatResponseIterator(
-            streaming_response=streaming_response,
-            sync_stream=sync_stream,
-            json_mode=json_mode,
-        )
-
-
-class CloudflareChatResponseIterator(BaseModelResponseIterator):
-    def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
-        try:
-            text = ""
-            tool_use: Optional[ChatCompletionToolCallChunk] = None
-            is_finished = False
-            finish_reason = ""
-            usage: Optional[ChatCompletionUsageBlock] = None
-            provider_specific_fields = None
-
-            index = int(chunk.get("index", 0))
-
-            if "response" in chunk and chunk["response"] is not None:
-                text = chunk["response"]
-            elif "response_text" in chunk and chunk["response_text"] is not None:
-                text = chunk["response_text"]
-
-            returned_chunk = GenericStreamingChunk(
-                text=text,
-                tool_use=tool_use,
-                is_finished=is_finished,
-                finish_reason=finish_reason,
-                usage=usage,
-                index=index,
-                provider_specific_fields=provider_specific_fields,
-            )
-
-            return returned_chunk
-
-        except json.JSONDecodeError:
-            raise ValueError(f"Failed to decode JSON from chunk: {chunk}")

litellm/llms/custom_httpx/http_handler.py

@@ -42,6 +42,9 @@ from litellm.constants import (
     HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
 )
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.litellm_core_utils.request_timeout_resolver import (
+    get_configured_request_timeout,
+)
 from litellm.types.llms.custom_http import *
 
 if TYPE_CHECKING:
@@ -134,6 +137,18 @@ _DEFAULT_TIMEOUT = httpx.Timeout(
     timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
     connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
 )
+
+
+def _default_cached_client_timeout() -> httpx.Timeout:
+    """Timeout for cached default httpx clients; honors an explicit litellm.request_timeout."""
+    configured = get_configured_request_timeout()
+    if configured is None:
+        return _DEFAULT_TIMEOUT
+    return httpx.Timeout(
+        timeout=configured, connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS
+    )
+
+
 _STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0
 _STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
     max_workers=50,
@@ -1379,7 +1394,7 @@ def get_async_httpx_client(
         _new_client = AsyncHTTPHandler(**handler_params)
     else:
         _new_client = AsyncHTTPHandler(
-            timeout=_DEFAULT_TIMEOUT,
+            timeout=_default_cached_client_timeout(),
             shared_session=shared_session,
         )
 
@@ -1428,7 +1443,7 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler:
         }
         _new_client = HTTPHandler(**handler_params)
     else:
-        _new_client = HTTPHandler(timeout=_DEFAULT_TIMEOUT)
+        _new_client = HTTPHandler(timeout=_default_cached_client_timeout())
 
     cache.set_cache(
         key=_cache_key_name,

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1,5 +1,6 @@
 import json
 import ssl
+from functools import lru_cache
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 from typing import (
     TYPE_CHECKING,
@@ -13,6 +14,7 @@ from typing import (
     Tuple,
     Union,
     cast,
+    get_type_hints,
 )
 
 import httpx  # type: ignore
@@ -26,6 +28,7 @@ from litellm._logging import _redact_string, verbose_logger
 from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
+from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
@@ -101,6 +104,7 @@ from litellm.types.llms.openai import (
     HttpxBinaryResponseContent,
     OpenAIFileObject,
     ResponseInputParam,
+    ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
 )
 from litellm.types.rerank import RerankResponse
@@ -135,6 +139,7 @@ from litellm.utils import (
     ImageResponse,
     ModelResponse,
     ProviderConfigManager,
+    async_pre_call_deployment_hook,
 )
 
 from .http_handler import get_shared_realtime_ssl_context
@@ -184,6 +189,47 @@ def _google_genai_streaming_hidden_params(
     }
 
 
+@lru_cache(maxsize=None)
+def _responses_api_optional_request_param_names() -> frozenset[str]:
+    return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
+
+
+def _custom_logger_callbacks(logging_obj: Any) -> list[Any]:
+    from litellm.integrations.custom_logger import CustomLogger
+    from litellm.litellm_core_utils.litellm_logging import (
+        get_custom_logger_compatible_class,
+    )
+
+    dynamic_success_callbacks = getattr(logging_obj, "dynamic_success_callbacks", None)
+    callbacks = list(litellm.callbacks)
+    if isinstance(dynamic_success_callbacks, (list, tuple)):
+        callbacks.extend(dynamic_success_callbacks)
+
+    custom_loggers: list[Any] = []
+    for cb in callbacks:
+        if isinstance(cb, str):
+            resolved = get_custom_logger_compatible_class(cb)  # type: ignore[arg-type]
+            if resolved is None:
+                continue
+            cb = resolved
+        if isinstance(cb, CustomLogger):
+            custom_loggers.append(cb)
+    return custom_loggers
+
+
+def _has_pre_call_deployment_hook(logging_obj: Any) -> bool:
+    from litellm.integrations.custom_logger import CustomLogger
+
+    base_func = CustomLogger.async_pre_call_deployment_hook
+    for cb in _custom_logger_callbacks(logging_obj):
+        cb_func = getattr(type(cb), "async_pre_call_deployment_hook", base_func)
+        if getattr(cb_func, "__func__", cb_func) is not getattr(
+            base_func, "__func__", base_func
+        ):
+            return True
+    return False
+
+
 class BaseLLMHTTPHandler:
     async def _make_common_async_call(
         self,
@@ -1833,6 +1879,9 @@ class BaseLLMHTTPHandler:
         data = provider_config.transform_search_request(
             query=query,
             optional_params=optional_params,
+            api_key=api_key,
+            api_base=api_base,
+            headers=headers or {},
         )
 
         # Get complete URL (pass data for providers that need request body for URL construction)
@@ -2224,12 +2273,92 @@ class BaseLLMHTTPHandler:
             )
         raise ValueError("anthropic_messages_handler is not implemented for sync calls")
 
+    def _run_sync_responses_pre_call_deployment_hook(
+        self,
+        *,
+        model: str,
+        input: Union[str, ResponseInputParam],
+        custom_llm_provider: str,
+        response_api_optional_request_params: dict[str, Any],
+        litellm_params: GenericLiteLLMParams,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> tuple[
+        str,
+        Union[str, ResponseInputParam],
+        str,
+        dict[str, Any],
+        GenericLiteLLMParams,
+    ]:
+        if not _has_pre_call_deployment_hook(logging_obj):
+            return (
+                model,
+                input,
+                custom_llm_provider,
+                response_api_optional_request_params,
+                litellm_params,
+            )
+
+        modified_kwargs = run_async_function(
+            async_pre_call_deployment_hook,
+            {
+                **dict(litellm_params),
+                **response_api_optional_request_params,
+                "model": model,
+                "input": input,
+                "custom_llm_provider": custom_llm_provider,
+            },
+            CallTypes.responses.value,
+        )
+        if modified_kwargs is None:
+            return (
+                model,
+                input,
+                custom_llm_provider,
+                response_api_optional_request_params,
+                litellm_params,
+            )
+
+        optional_param_names = _responses_api_optional_request_param_names()
+        updated_response_params = {
+            **response_api_optional_request_params,
+            **{
+                key: value
+                for key, value in modified_kwargs.items()
+                if key in optional_param_names
+            },
+        }
+        updated_litellm_params = GenericLiteLLMParams(
+            **{
+                **dict(litellm_params),
+                **{
+                    key: value
+                    for key, value in modified_kwargs.items()
+                    if key not in optional_param_names
+                    and key not in {"model", "input", "custom_llm_provider"}
+                },
+            }
+        )
+        return (
+            str(modified_kwargs["model"]) if "model" in modified_kwargs else model,
+            cast(
+                Union[str, ResponseInputParam],
+                modified_kwargs["input"] if "input" in modified_kwargs else input,
+            ),
+            (
+                str(modified_kwargs["custom_llm_provider"])
+                if "custom_llm_provider" in modified_kwargs
+                else custom_llm_provider
+            ),
+            updated_response_params,
+            updated_litellm_params,
+        )
+
     def response_api_handler(
         self,
         model: str,
         input: Union[str, ResponseInputParam],
         responses_api_provider_config: BaseResponsesAPIConfig,
-        response_api_optional_request_params: Dict,
+        response_api_optional_request_params: dict[str, Any],
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
@@ -2276,6 +2405,21 @@ class BaseLLMHTTPHandler:
                 shared_session=shared_session,
             )
 
+        (
+            model,
+            input,
+            custom_llm_provider,
+            response_api_optional_request_params,
+            litellm_params,
+        ) = self._run_sync_responses_pre_call_deployment_hook(
+            model=model,
+            input=input,
+            custom_llm_provider=custom_llm_provider,
+            response_api_optional_request_params=response_api_optional_request_params,
+            litellm_params=litellm_params,
+            logging_obj=logging_obj,
+        )
+
         if client is None or not isinstance(client, HTTPHandler):
             sync_httpx_client = _get_httpx_client(
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)}
@@ -2407,12 +2551,36 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
-        return responses_api_provider_config.transform_response_api_response(
-            model=model,
-            raw_response=response,
-            logging_obj=logging_obj,
+        initial_response = (
+            responses_api_provider_config.transform_response_api_response(
+                model=model,
+                raw_response=response,
+                logging_obj=logging_obj,
+            )
         )
 
+        if self._has_agentic_completion_hook(logging_obj):
+            final_response = run_async_function(
+                self._call_agentic_completion_hooks,
+                response=initial_response,
+                model=model,
+                messages=(
+                    input
+                    if isinstance(input, list)
+                    else [{"role": "user", "content": input}]
+                ),
+                anthropic_messages_provider_config=responses_api_provider_config,
+                anthropic_messages_optional_request_params=response_api_optional_request_params,
+                logging_obj=logging_obj,
+                stream=False,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=dict(litellm_params),
+                api_surface="responses",
+            )
+            return final_response if final_response is not None else initial_response
+
+        return initial_response
+
     async def async_response_api_handler(
         self,
         model: str,
@@ -2570,12 +2738,44 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
-        return responses_api_provider_config.transform_response_api_response(
-            model=model,
-            raw_response=response,
-            logging_obj=logging_obj,
+        initial_response = (
+            responses_api_provider_config.transform_response_api_response(
+                model=model,
+                raw_response=response,
+                logging_obj=logging_obj,
+            )
         )
 
+        final_response = await self._call_agentic_completion_hooks(
+            response=initial_response,
+            model=model,
+            messages=(
+                input
+                if isinstance(input, list)
+                else [{"role": "user", "content": input}]
+            ),
+            anthropic_messages_provider_config=responses_api_provider_config,
+            anthropic_messages_optional_request_params=response_api_optional_request_params,
+            logging_obj=logging_obj,
+            stream=False,
+            custom_llm_provider=custom_llm_provider,
+            kwargs=dict(litellm_params),
+            api_surface="responses",
+        )
+
+        result = final_response if final_response is not None else initial_response
+        if litellm_params.get(
+            "_code_interpreter_interception_converted_stream"
+        ) and not litellm_params.get("_agentic_loop_depth"):
+            return self._wrap_responses_response_as_fake_stream(
+                result=result,
+                model=model,
+                responses_api_provider_config=responses_api_provider_config,
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+            )
+        return result
+
     async def async_delete_response_api_handler(
         self,
         response_id: str,
@@ -4734,22 +4934,9 @@ class BaseLLMHTTPHandler:
         agentic callback is detected too.
         """
         from litellm.integrations.custom_logger import CustomLogger
-        from litellm.litellm_core_utils.litellm_logging import (
-            get_custom_logger_compatible_class,
-        )
 
         base_func = CustomLogger.async_should_run_agentic_loop
-        callbacks = litellm.callbacks + (
-            getattr(logging_obj, "dynamic_success_callbacks", None) or []
-        )
-        for cb in callbacks:
-            if isinstance(cb, str):
-                resolved = get_custom_logger_compatible_class(cb)  # type: ignore[arg-type]
-                if resolved is None:
-                    continue
-                cb = resolved
-            if not isinstance(cb, CustomLogger):
-                continue
+        for cb in _custom_logger_callbacks(logging_obj):
             cb_func = getattr(type(cb), "async_should_run_agentic_loop", base_func)
             if getattr(cb_func, "__func__", cb_func) is not getattr(
                 base_func, "__func__", base_func
@@ -4875,6 +5062,132 @@ class BaseLLMHTTPHandler:
 
         return response
 
+    async def _execute_responses_agentic_plan(
+        self,
+        plan: AgenticLoopPlan,
+        model: str,
+        response_api_optional_request_params: dict,
+        logging_obj: "LiteLLMLoggingObj",
+        kwargs: dict,
+        depth: int,
+        max_loops: int,
+        fingerprints: list[str],
+        fingerprint: str,
+        callback: Any | None = None,
+    ) -> Any:
+        patch = plan.request_patch or AgenticLoopRequestPatch()
+        if patch.messages is None:
+            raise ValueError("Agentic loop plan missing patched responses input")
+
+        optional_params = dict(response_api_optional_request_params)
+        optional_params.update(patch.optional_params)
+        if patch.tools is not None:
+            optional_params["tools"] = patch.tools
+        optional_params = {
+            k: v
+            for k, v in optional_params.items()
+            if k != "stream" and k != "_code_interpreter_interception_converted_stream"
+        }
+
+        internal_keys = {"litellm_logging_obj"}
+        kwargs_for_followup = {
+            k: v
+            for k, v in kwargs.items()
+            if not k.startswith("_websearch_interception")
+            and not k.startswith("_compression_interception")
+            and k != "_code_interpreter_interception_converted_stream"
+            and k not in internal_keys
+            and k not in optional_params
+        }
+        kwargs_for_followup.update(patch.kwargs)
+        kwargs_for_followup["_agentic_loop_depth"] = depth + 1
+        kwargs_for_followup["max_agentic_loops"] = max_loops
+        kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
+
+        try:
+            response = await litellm.aresponses(
+                model=patch.model or model,
+                input=patch.messages,
+                **optional_params,
+                **kwargs_for_followup,
+            )
+
+            if callback is not None:
+                try:
+                    response = await callback.async_post_agentic_loop_response_hook(
+                        response=response, plan=plan, kwargs=kwargs
+                    )
+                except Exception as e:
+                    _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+                    verbose_logger.exception(
+                        "LiteLLM.AgenticHookError: Exception in "
+                        "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s",
+                        _call_id,
+                        model,
+                        str(e),
+                    )
+
+            return response
+        finally:
+            if callback is not None:
+                await self._run_agentic_loop_cleanup(
+                    callback=callback,
+                    plan=plan,
+                    kwargs=kwargs,
+                    logging_obj=logging_obj,
+                    model=model,
+                )
+
+    @staticmethod
+    async def _run_agentic_loop_cleanup(
+        callback: Any,
+        plan: AgenticLoopPlan,
+        kwargs: dict,
+        logging_obj: "LiteLLMLoggingObj",
+        model: str,
+    ) -> None:
+        try:
+            await callback.async_agentic_loop_cleanup_hook(plan=plan, kwargs=kwargs)
+        except Exception as e:
+            _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+            verbose_logger.exception(
+                "LiteLLM.AgenticHookError: Exception in "
+                "async_agentic_loop_cleanup_hook [call_id=%s model=%s]: %s",
+                _call_id,
+                model,
+                str(e),
+            )
+
+    def _wrap_responses_response_as_fake_stream(
+        self,
+        result: Any,
+        model: str,
+        responses_api_provider_config: Any,
+        logging_obj: "LiteLLMLoggingObj",
+        custom_llm_provider: str,
+    ) -> Any:
+        """
+        Wrap a completed responses result as a synthetic stream.
+
+        Used when an interceptor forced stream=False to run the agentic loop on
+        the non-streaming path, but the caller originally asked for streaming.
+        """
+        import httpx
+
+        from litellm.responses.streaming_iterator import (
+            MockResponsesAPIStreamingIterator,
+        )
+
+        payload = result.model_dump() if hasattr(result, "model_dump") else result
+        raw_response = httpx.Response(status_code=200, json=payload)
+        return MockResponsesAPIStreamingIterator(
+            response=raw_response,
+            model=model,
+            responses_api_provider_config=responses_api_provider_config,
+            logging_obj=logging_obj,
+            custom_llm_provider=custom_llm_provider,
+        )
+
     async def _execute_chat_completion_agentic_plan(
         self,
         plan: AgenticLoopPlan,
@@ -4940,6 +5253,7 @@ class BaseLLMHTTPHandler:
         stream: bool,
         custom_llm_provider: str,
         kwargs: Dict,
+        api_surface: str = "anthropic_messages",
     ) -> Optional[Any]:
         """
         Call agentic completion hooks for all custom loggers (Anthropic Messages API).
@@ -5046,6 +5360,20 @@ class BaseLLMHTTPHandler:
                 if not plan.run_agentic_loop:
                     continue
 
+                if api_surface == "responses":
+                    return await self._execute_responses_agentic_plan(
+                        plan=plan,
+                        model=model,
+                        response_api_optional_request_params=anthropic_messages_optional_request_params,
+                        logging_obj=logging_obj,
+                        kwargs=kwargs_with_provider,
+                        depth=depth,
+                        max_loops=max_loops,
+                        fingerprints=fingerprints,
+                        fingerprint=fingerprint,
+                        callback=callback,
+                    )
+
                 return await self._execute_anthropic_agentic_plan(
                     plan=plan,
                     model=model,
@@ -5083,7 +5411,7 @@ class BaseLLMHTTPHandler:
             else False
         )
 
-        if websearch_converted_stream:
+        if api_surface == "anthropic_messages" and websearch_converted_stream:
             from typing import cast
 
             from litellm._logging import verbose_logger
@@ -5358,9 +5686,7 @@ class BaseLLMHTTPHandler:
         import websockets
         from websockets.asyncio.client import ClientConnection
 
-        url = self._append_query_params(
-            provider_config.get_complete_url(api_base, model, api_key), query_params
-        )
+        url = provider_config.get_complete_url(api_base, model, api_key)
         headers = provider_config.validate_environment(
             headers=headers,
             model=model,

litellm/llms/dataforseo/search/transformation.py

@@ -61,9 +61,18 @@ class DataForSEOSearchConfig(BaseSearchConfig):
         password = get_secret_str("DATAFORSEO_PASSWORD")
 
         # If api_key is provided in "login:password" format, use it
+        caller_supplied_credentials = bool(api_key and ":" in api_key)
         if api_key and ":" in api_key:
             login, password = api_key.split(":", 1)
 
+        if not caller_supplied_credentials and login and password:
+            self._assert_trusted_api_base_for_server_credential(
+                api_base,
+                self.DATAFORSEO_API_BASE,
+                "DATAFORSEO_API_BASE",
+                "DATAFORSEO_LOGIN",
+            )
+
         if not login:
             raise ValueError(
                 "DATAFORSEO_LOGIN is not set. Set `DATAFORSEO_LOGIN` environment variable or pass credentials in api_key parameter."

litellm/llms/e2b/sandbox/transformation.py

@@ -16,6 +16,7 @@ from litellm.llms.base_llm.sandbox.transformation import (
     BaseSandboxConfig,
     CodeExecutionResult,
     ContainerHandle,
+    SANDBOX_MAX_OUTPUT_BYTES,
 )
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -29,7 +30,7 @@ E2B_DEFAULT_TEMPLATE = "code-interpreter-v1"
 E2B_DEFAULT_DOMAIN = "e2b.app"
 JUPYTER_PORT = 49999
 DEFAULT_SANDBOX_TIMEOUT = 300
-MAX_OUTPUT_BYTES = 10 * 1024 * 1024
+MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES
 
 
 class E2BSandboxConfig(BaseSandboxConfig):
@@ -49,18 +50,22 @@ class E2BSandboxConfig(BaseSandboxConfig):
         *,
         template: str | None = None,
         timeout: int | None = None,
-        allow_internet_access: bool = True,
+        allow_internet_access: bool | None = None,
         api_key: str | None = None,
+        api_base: str | None = None,
         metadata: dict | None = None,
         client: AsyncHTTPHandler | None = None,
         **kwargs,
     ) -> ContainerHandle:
         key = self.validate_environment(api_key=api_key)
+        base = api_base or E2B_API_BASE
         body = {
             "templateID": template or E2B_DEFAULT_TEMPLATE,
             "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT,
             "secure": True,
-            "allow_internet_access": allow_internet_access,
+            "allow_internet_access": (
+                True if allow_internet_access is None else allow_internet_access
+            ),
         }
         if metadata:
             body["metadata"] = metadata
@@ -68,7 +73,7 @@ class E2BSandboxConfig(BaseSandboxConfig):
         response = cast(
             httpx.Response,
             await self._http(client).post(
-                url=f"{E2B_API_BASE}/sandboxes",
+                url=f"{base}/sandboxes",
                 headers={"X-API-Key": key, "Content-Type": "application/json"},
                 json=body,
             ),
@@ -84,6 +89,7 @@ class E2BSandboxConfig(BaseSandboxConfig):
             "envd_access_token": data.get("envdAccessToken"),
             "traffic_access_token": data.get("trafficAccessToken"),
             "api_key": key,
+            "api_base": base,
         }
         return handle
 
@@ -130,6 +136,7 @@ class E2BSandboxConfig(BaseSandboxConfig):
         *,
         container: Union[ContainerHandle, str],
         api_key: str | None = None,
+        api_base: str | None = None,
         client: AsyncHTTPHandler | None = None,
         **kwargs,
     ) -> bool:
@@ -139,11 +146,12 @@ class E2BSandboxConfig(BaseSandboxConfig):
             or handle._hidden_params.get("api_key")
             or self.validate_environment()
         )
+        base = api_base or handle._hidden_params.get("api_base") or E2B_API_BASE
         try:
             response = cast(
                 httpx.Response,
                 await self._http(client).delete(
-                    url=f"{E2B_API_BASE}/sandboxes/{handle.id}",
+                    url=f"{base}/sandboxes/{handle.id}",
                     headers={"X-API-Key": key},
                 ),
             )
@@ -163,20 +171,6 @@ class E2BSandboxConfig(BaseSandboxConfig):
         handle._hidden_params = {}
         return handle
 
-    @staticmethod
-    async def _read_capped_lines(response: httpx.Response) -> list[str]:
-        lines: list[str] = []
-        total = 0
-        async for line in response.aiter_lines():
-            total += len(line.encode("utf-8"))
-            if total > MAX_OUTPUT_BYTES:
-                raise ValueError(
-                    f"Sandbox output exceeded {MAX_OUTPUT_BYTES} bytes; aborting to "
-                    "avoid unbounded memory use."
-                )
-            lines.append(line)
-        return lines
-
     @staticmethod
     def _parse_lines(lines: list[str]) -> CodeExecutionResult:
         def _try_parse(stripped: str):
@@ -187,10 +181,9 @@ class E2BSandboxConfig(BaseSandboxConfig):
 
         messages = tuple(
             parsed
-            for stripped in (line.strip() for line in lines)
-            if stripped
-            for parsed in (_try_parse(stripped),)
-            if parsed is not None
+            for line in lines
+            if (stripped := line.strip())
+            if (parsed := _try_parse(stripped)) is not None
         )
 
         def of_type(message_type: str):

litellm/llms/exa_ai/search/transformation.py

@@ -65,7 +65,13 @@ class ExaAISearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("EXA_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("EXA_API_KEY",),
+            base_env_var="EXA_API_BASE",
+            default_api_base=self.EXA_AI_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "EXA_API_KEY is not set. Set `EXA_API_KEY` environment variable."

litellm/llms/fastcrw/search/transformation.py

@@ -57,7 +57,13 @@ class FastCRWSearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("CRW_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("CRW_API_KEY",),
+            base_env_var="CRW_API_BASE",
+            default_api_base=self.FASTCRW_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "CRW_API_KEY is not set. Set `CRW_API_KEY` environment variable."

litellm/llms/firecrawl/search/transformation.py

@@ -61,7 +61,13 @@ class FirecrawlSearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("FIRECRAWL_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("FIRECRAWL_API_KEY",),
+            base_env_var="FIRECRAWL_API_BASE",
+            default_api_base=self.FIRECRAWL_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "FIRECRAWL_API_KEY is not set. Set `FIRECRAWL_API_KEY` environment variable."

litellm/llms/fireworks_ai/audio_transcription/transformation.py

@@ -1,17 +0,0 @@
-from typing import List
-
-from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
-
-from ...openai.transcriptions.whisper_transformation import (
-    OpenAIWhisperAudioTranscriptionConfig,
-)
-from ..common_utils import FireworksAIMixin
-
-
-class FireworksAIAudioTranscriptionConfig(
-    FireworksAIMixin, OpenAIWhisperAudioTranscriptionConfig
-):
-    def get_supported_openai_params(
-        self, model: str
-    ) -> List[OpenAIAudioTranscriptionOptionalParams]:
-        return ["language", "prompt", "response_format", "timestamp_granularities"]

litellm/llms/fireworks_ai/chat/transformation.py

@@ -1,5 +1,15 @@
 import json
-from typing import Any, List, Literal, Optional, Tuple, Union, cast
+from typing import (
+    Any,
+    AsyncIterator,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
 
 import httpx
 
@@ -15,7 +25,6 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
     AllMessageValues,
-    ChatCompletionImageObject,
     ChatCompletionToolParam,
     OpenAIChatCompletionToolParam,
 )
@@ -25,6 +34,7 @@ from litellm.types.utils import (
     Function,
     Message,
     ModelResponse,
+    ModelResponseStream,
     ProviderSpecificModelInfo,
 )
 from litellm.utils import (
@@ -34,10 +44,34 @@ from litellm.utils import (
     supports_tool_choice,
 )
 
-from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+from ...openai.chat.gpt_transformation import (
+    OpenAIChatCompletionStreamingHandler,
+    OpenAIGPTConfig,
+)
 from ..common_utils import FireworksAIException
 
 
+def _extract_fireworks_hidden_params(payload: dict) -> dict:
+    """
+    Collect Fireworks-specific response fields (perf_metrics, prompt_token_ids,
+    per-choice raw_output and token_ids) from a non-streaming completion payload
+    or a single streaming chunk, so the same data lands in ``_hidden_params`` on
+    both response paths.
+    """
+    choices = [c for c in (payload.get("choices") or []) if isinstance(c, dict)]
+    top_level = {
+        f"fireworks_{field}": payload[field]
+        for field in ("perf_metrics", "prompt_token_ids")
+        if field in payload
+    }
+    per_choice = {
+        f"fireworks_{dest}": [c[field] for c in choices if field in c]
+        for field, dest in (("raw_output", "raw_outputs"), ("token_ids", "token_ids"))
+        if any(field in c for c in choices)
+    }
+    return {**top_level, **per_choice}
+
+
 class FireworksAIConfig(OpenAIGPTConfig):
     """
     Reference: https://docs.fireworks.ai/api-reference/post-chatcompletions
@@ -60,8 +94,7 @@ class FireworksAIConfig(OpenAIGPTConfig):
     logprobs: Optional[int] = None
     reasoning_effort: Optional[str] = None
 
-    # Non OpenAI parameters - Fireworks AI only params
-    prompt_truncate_length: Optional[int] = None
+    prompt_truncate_len: Optional[int] = None
     context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None
 
     def __init__(
@@ -80,7 +113,7 @@ class FireworksAIConfig(OpenAIGPTConfig):
         user: Optional[str] = None,
         logprobs: Optional[int] = None,
         reasoning_effort: Optional[str] = None,
-        prompt_truncate_length: Optional[int] = None,
+        prompt_truncate_len: Optional[int] = None,
         context_length_exceeded_behavior: Optional[Literal["error", "truncate"]] = None,
     ) -> None:
         locals_ = locals().copy()
@@ -108,8 +141,30 @@ class FireworksAIConfig(OpenAIGPTConfig):
             "response_format",
             "user",
             "logprobs",
-            "prompt_truncate_length",
+            "prompt_truncate_len",
             "context_length_exceeded_behavior",
+            "seed",
+            "top_logprobs",
+            "min_p",
+            "typical_p",
+            "repetition_penalty",
+            "mirostat_target",
+            "mirostat_lr",
+            "logit_bias",
+            "echo",
+            "echo_last",
+            "ignore_eos",
+            "prompt_cache_key",
+            "prompt_cache_isolation_key",
+            "raw_output",
+            "perf_metrics_in_response",
+            "return_token_ids",
+            "safe_tokenization",
+            "service_tier",
+            "speculation",
+            "prediction",
+            "stream_options",
+            "sampling_mask",
         ]
 
         # Only add tools for models that support function calling
@@ -133,9 +188,11 @@ class FireworksAIConfig(OpenAIGPTConfig):
         if supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"):
             supported_params.append("tool_choice")
 
-        # Only add reasoning_effort for models that support it
+        # Only add reasoning params for models that support it
         if supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
             supported_params.append("reasoning_effort")
+            supported_params.append("reasoning_history")
+            supported_params.append("thinking")
 
         return supported_params
 
@@ -151,6 +208,18 @@ class FireworksAIConfig(OpenAIGPTConfig):
             param == "tools" and value is not None
             for param, value in non_default_params.items()
         )
+        if (
+            non_default_params.get("thinking") is not None
+            and non_default_params.get("reasoning_effort") is not None
+        ):
+            raise litellm.BadRequestError(
+                message=(
+                    "Fireworks AI chat completions does not support specifying both "
+                    "`thinking` and `reasoning_effort` in the same request."
+                ),
+                model=model,
+                llm_provider="fireworks_ai",
+            )
 
         for param, value in non_default_params.items():
             if param == "tool_choice":
@@ -174,40 +243,19 @@ class FireworksAIConfig(OpenAIGPTConfig):
                     optional_params["response_format"] = value
             elif param == "max_completion_tokens":
                 optional_params["max_tokens"] = value
+            elif param == "reasoning_effort":
+                if value is True:
+                    optional_params["reasoning_effort"] = "medium"
+                elif value is False:
+                    optional_params["reasoning_effort"] = "none"
+                else:
+                    optional_params["reasoning_effort"] = value
             elif param in supported_openai_params:
                 if value is not None:
                     optional_params[param] = value
 
         return optional_params
 
-    def _add_transform_inline_image_block(
-        self,
-        content: ChatCompletionImageObject,
-        model: str,
-        disable_add_transform_inline_image_block: Optional[bool],
-    ) -> ChatCompletionImageObject:
-        """
-        Add transform_inline to the image_url (allows non-vision models to parse documents/images/etc.)
-        - ignore if model is a vision model
-        - ignore if user has disabled this feature
-        """
-        if (
-            "vision" in model or disable_add_transform_inline_image_block
-        ):  # allow user to toggle this feature.
-            return content
-        if isinstance(content["image_url"], str):
-            # Skip base64 data URLs — appending #transform=inline corrupts the
-            # base64 payload and causes an "Incorrect padding" decode error on
-            # the Fireworks side.  Data URLs are already inlined by definition.
-            # Lower-case before checking: URI schemes are case-insensitive (RFC 3986).
-            if not content["image_url"].lower().startswith("data:"):
-                content["image_url"] = f"{content['image_url']}#transform=inline"
-        elif isinstance(content["image_url"], dict):
-            url = content["image_url"]["url"]
-            if not url.lower().startswith("data:"):
-                content["image_url"]["url"] = f"{url}#transform=inline"
-        return content
-
     def _transform_tools(
         self, tools: List[OpenAIChatCompletionToolParam]
     ) -> List[OpenAIChatCompletionToolParam]:
@@ -225,36 +273,46 @@ class FireworksAIConfig(OpenAIGPTConfig):
         self, messages: List[AllMessageValues], model: str, litellm_params: dict
     ) -> List[AllMessageValues]:
         """
-        Add 'transform=inline' to the url of the image_url
+        Strip fields not permitted by FireworksAI from messages.
         """
         from litellm.litellm_core_utils.prompt_templates.common_utils import (
             filter_value_from_dict,
-            migrate_file_to_image_url,
         )
 
-        disable_add_transform_inline_image_block = cast(
-            Optional[bool],
-            litellm_params.get("disable_add_transform_inline_image_block")
-            or litellm.disable_add_transform_inline_image_block,
+        supports_vision_value = self._get_model_cost_capability_exact(
+            model=model, capability="supports_vision"
         )
-        ## For any 'file' message type with pdf content, move to 'image_url' message type
-        for message in messages:
-            if message["role"] == "user":
-                _message_content = message.get("content")
-                if _message_content is not None and isinstance(_message_content, list):
-                    for idx, content in enumerate(_message_content):
-                        if content["type"] == "file":
-                            _message_content[idx] = migrate_file_to_image_url(content)
         for message in messages:
             if message["role"] == "user":
                 _message_content = message.get("content")
                 if _message_content is not None and isinstance(_message_content, list):
                     for content in _message_content:
-                        if content["type"] == "image_url":
-                            content = self._add_transform_inline_image_block(
-                                content=content,
+                        if not isinstance(content, dict):
+                            continue
+                        if content.get("type") == "file":
+                            raise litellm.BadRequestError(
+                                message=(
+                                    "Fireworks AI chat completions does not support "
+                                    "file content blocks. For PDFs, convert pages to "
+                                    "images and send image_url blocks to a Fireworks "
+                                    "vision model, or extract text before calling a "
+                                    "text-only model."
+                                ),
                                 model=model,
-                                disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
+                                llm_provider="fireworks_ai",
+                            )
+                        if (
+                            content.get("type") == "image_url"
+                            and supports_vision_value is False
+                        ):
+                            raise litellm.BadRequestError(
+                                message=(
+                                    f"Fireworks AI model {model} does not support "
+                                    "image inputs. Use a Fireworks vision model or "
+                                    "remove image_url content blocks."
+                                ),
+                                model=model,
+                                llm_provider="fireworks_ai",
                             )
             filter_value_from_dict(cast(dict, message), "cache_control")
             # Remove fields not permitted by FireworksAI (additionalProperties: false
@@ -317,43 +375,55 @@ class FireworksAIConfig(OpenAIGPTConfig):
             return True
         return ("-" + key_short + "-") in short_name
 
-    def _get_model_cost_capability(self, model: str, capability: str) -> Optional[bool]:
+    @staticmethod
+    def _short_model_name(model: str) -> str:
         short_name = model
         if short_name.startswith("fireworks_ai/"):
             short_name = short_name[len("fireworks_ai/") :]
         if short_name.startswith("accounts/fireworks/models/"):
             short_name = short_name[len("accounts/fireworks/models/") :]
+        return short_name
 
-        candidate_keys = [
+    def _get_model_cost_capability_exact(
+        self, model: str, capability: str
+    ) -> Optional[bool]:
+        short_name = self._short_model_name(model)
+        candidate_keys = (
             model,
             f"fireworks_ai/{short_name}",
             f"fireworks_ai/accounts/fireworks/models/{short_name}",
-        ]
-
+        )
         for candidate_key in candidate_keys:
             model_info = litellm.model_cost.get(candidate_key)
             if model_info is not None and model_info.get(capability) is not None:
                 return cast(Optional[bool], model_info.get(capability))
+        return None
 
-        # Fallback: preserve historical substring matching for model name
-        # variants (e.g. fine-tuned or regionally-suffixed versions of a
-        # known model). Pick the *longest* matching entry so a more specific
-        # known model (e.g. "qwen3-8b-instruct") wins over a less specific
-        # one (e.g. "qwen3-8b") when the query model is more specific still.
-        # Use hyphen-aligned matching to avoid false positives where a short
-        # known model name is an unrelated substring of a longer one.
-        best_match_short: Optional[str] = None
-        best_match_value: Optional[bool] = None
-        for key_short, model_info in self._get_fireworks_index():
-            if model_info.get(capability) is None:
-                continue
-            if not self._matches_on_hyphen_boundary(short_name, key_short):
-                continue
-            if best_match_short is None or len(key_short) > len(best_match_short):
-                best_match_short = key_short
-                best_match_value = cast(Optional[bool], model_info.get(capability))
+    def _get_model_cost_capability(self, model: str, capability: str) -> Optional[bool]:
+        exact = self._get_model_cost_capability_exact(
+            model=model, capability=capability
+        )
+        if exact is not None:
+            return exact
 
-        return best_match_value
+        # Fallback: substring matching for model name variants (e.g. fine-tuned
+        # or regionally-suffixed versions of a known model). Pick the *longest*
+        # matching entry so a more specific known model (e.g. "qwen3-8b-instruct")
+        # wins over a less specific one (e.g. "qwen3-8b"). Hyphen-aligned matching
+        # avoids false positives where a short known name is an unrelated
+        # substring of a longer one. This stays a soft signal: capability-gated
+        # hard rejections use the exact lookup so a fuzzy match never blocks a
+        # custom deployment.
+        short_name = self._short_model_name(model)
+        matches = [
+            (key_short, cast(Optional[bool], model_info.get(capability)))
+            for key_short, model_info in self._get_fireworks_index()
+            if model_info.get(capability) is not None
+            and self._matches_on_hyphen_boundary(short_name, key_short)
+        ]
+        if not matches:
+            return None
+        return max(matches, key=lambda match: len(match[0]))[1]
 
     def get_provider_info(self, model: str) -> ProviderSpecificModelInfo:
         supports_function_calling_value = self._get_model_cost_capability(
@@ -362,12 +432,16 @@ class FireworksAIConfig(OpenAIGPTConfig):
         supports_reasoning_value = self._get_model_cost_capability(
             model=model, capability="supports_reasoning"
         )
+        supports_vision_value = self._get_model_cost_capability(
+            model=model, capability="supports_vision"
+        )
+        supports_pdf_input_value = self._get_model_cost_capability(
+            model=model, capability="supports_pdf_input"
+        )
 
         provider_specific_model_info: ProviderSpecificModelInfo = {
             "supports_function_calling": True,
             "supports_prompt_caching": True,  # https://docs.fireworks.ai/guides/prompt-caching
-            "supports_pdf_input": True,  # via document inlining
-            "supports_vision": True,  # via document inlining
         }
 
         if supports_function_calling_value is not None:
@@ -381,6 +455,14 @@ class FireworksAIConfig(OpenAIGPTConfig):
                 supports_reasoning_value
             )
 
+        if supports_vision_value is not None:
+            provider_specific_model_info["supports_vision"] = supports_vision_value
+
+        if supports_pdf_input_value is not None:
+            provider_specific_model_info["supports_pdf_input"] = (
+                supports_pdf_input_value
+            )
+
         return provider_specific_model_info
 
     def transform_request(
@@ -402,6 +484,15 @@ class FireworksAIConfig(OpenAIGPTConfig):
         if "tools" in optional_params and optional_params["tools"] is not None:
             tools = self._transform_tools(tools=optional_params["tools"])
             optional_params["tools"] = tools
+        if optional_params.get("stream"):
+            stream_options = optional_params.get("stream_options")
+            if stream_options is None:
+                optional_params["stream_options"] = {"include_usage": True}
+            elif stream_options.get("include_usage") is not False:
+                optional_params["stream_options"] = {
+                    **stream_options,
+                    "include_usage": True,
+                }
         return super().transform_request(
             model=model,
             messages=messages,
@@ -494,10 +585,25 @@ class FireworksAIConfig(OpenAIGPTConfig):
                 )
             )
 
-        response._hidden_params = {"additional_headers": additional_headers}
+        response._hidden_params = {
+            "additional_headers": additional_headers,
+            **_extract_fireworks_hidden_params(completion_response),
+        }
 
         return response
 
+    def get_model_response_iterator(
+        self,
+        streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+        sync_stream: bool,
+        json_mode: Optional[bool] = False,
+    ) -> Any:
+        return FireworksAIChatCompletionStreamingHandler(
+            streaming_response=streaming_response,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+        )
+
     def _get_openai_compatible_provider_info(
         self, api_base: Optional[str], api_key: Optional[str]
     ) -> Tuple[Optional[str], Optional[str]]:
@@ -554,3 +660,15 @@ class FireworksAIConfig(OpenAIGPTConfig):
             or get_secret_str("FIREWORKSAI_API_KEY")
             or get_secret_str("FIREWORKS_AI_TOKEN")
         )
+
+
+class FireworksAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
+    def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+        parsed = super().chunk_parser(chunk)
+        fireworks_fields = _extract_fireworks_hidden_params(chunk)
+        if fireworks_fields:
+            parsed.provider_specific_fields = {
+                **(getattr(parsed, "provider_specific_fields", None) or {}),
+                **fireworks_fields,
+            }
+        return parsed

litellm/llms/gemini/realtime/transformation.py

@@ -103,6 +103,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         # bypassing spend and budget accounting.
         self._pending_usage_metadata: Optional[dict] = None
 
+    def _include_function_response_id(self) -> bool:
+        """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
+        return True
+
     @staticmethod
     def _usage_detail_alias(details: Any, defaults: Dict[str, int]) -> Dict[str, Any]:
         if not isinstance(details, dict):
@@ -604,10 +608,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             )
 
         # Build Gemini toolResponse format
-        function_response = {
-            "id": call_id,
-            "response": output_dict,
-        }
+        function_response: dict[str, Any] = {"response": output_dict}
+        if self._include_function_response_id() and call_id:
+            function_response["id"] = call_id
         if function_name:
             function_response["name"] = function_name
 

litellm/llms/google_pse/search/transformation.py

@@ -85,7 +85,13 @@ class GooglePSESearchConfig(BaseSearchConfig):
         Google PSE uses API key as a query parameter, not in headers.
         This method is called but headers are not used for authentication.
         """
-        api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("GOOGLE_PSE_API_KEY",),
+            base_env_var="GOOGLE_PSE_API_BASE",
+            default_api_base=self.GOOGLE_PSE_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "GOOGLE_PSE_API_KEY is not set. Set `GOOGLE_PSE_API_KEY` environment variable."
@@ -137,6 +143,7 @@ class GooglePSESearchConfig(BaseSearchConfig):
         query: Union[str, List[str]],
         optional_params: dict,
         api_key: Optional[str] = None,
+        api_base: str | None = None,
         search_engine_id: Optional[str] = None,
         **kwargs,
     ) -> Dict:
@@ -165,8 +172,16 @@ class GooglePSESearchConfig(BaseSearchConfig):
             # Google PSE only supports single string queries
             query = " ".join(query)
 
-        # Get API credentials
-        api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY")
+        # Get API credentials. The key is sent as a query param to api_base, so
+        # resolve it host-aware to avoid leaking a server-managed key to a
+        # caller-supplied host.
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("GOOGLE_PSE_API_KEY",),
+            base_env_var="GOOGLE_PSE_API_BASE",
+            default_api_base=self.GOOGLE_PSE_API_BASE,
+        )
         search_engine_id = search_engine_id or get_secret_str("GOOGLE_PSE_ENGINE_ID")
 
         if not api_key:

litellm/llms/linkup/search/transformation.py

@@ -61,7 +61,13 @@ class LinkupSearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("LINKUP_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("LINKUP_API_KEY",),
+            base_env_var="LINKUP_API_BASE",
+            default_api_base=self.LINKUP_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "LINKUP_API_KEY is not set. Set `LINKUP_API_KEY` environment variable."

litellm/llms/mistral/chat/transformation.py

@@ -247,6 +247,8 @@ class MistralConfig(OpenAIGPTConfig):
         The above statement is not valid now. Need to plan to remove all the #1,2,3
         Mistral API supports content as a list.
         """
+        messages = [self._strip_output_only_fields(m) for m in messages]
+
         ## 1. If 'image_url' or 'file' in content, then transform with base class and mistral-specific handling
         for m in messages:
             _content_block = m.get("content")
@@ -409,6 +411,25 @@ class MistralConfig(OpenAIGPTConfig):
 
         return cleaned_tools
 
+    @classmethod
+    def _strip_output_only_fields(cls, message: AllMessageValues) -> AllMessageValues:
+        """
+        ``reasoning_content`` and ``thinking_blocks`` are output-only fields that
+        LiteLLM attaches to assistant responses. Mistral's input schema forbids
+        unknown fields, so replaying them verbatim in a follow-up turn triggers a
+        422 ``extra_forbidden``. Drop them before the request is sent.
+        """
+        if message["role"] != "assistant":
+            return message
+        return cast(
+            AllMessageValues,
+            {
+                k: v
+                for k, v in message.items()
+                if k not in ("reasoning_content", "thinking_blocks")
+            },
+        )
+
     @classmethod
     def _handle_name_in_message(cls, message: AllMessageValues) -> AllMessageValues:
         """

litellm/llms/openai_like/providers.json

@@ -115,6 +115,14 @@
       "max_completion_tokens": "max_tokens"
     }
   },
+  "darkbloom": {
+    "base_url": "https://api.darkbloom.dev/v1",
+    "api_key_env": "DARKBLOOM_API_KEY",
+    "api_base_env": "DARKBLOOM_API_BASE",
+    "param_mappings": {
+      "max_completion_tokens": "max_tokens"
+    }
+  },
   "neosantara": {
     "base_url": "https://api.neosantara.xyz/v1",
     "api_key_env": "NEOSANTARA_API_KEY",

litellm/llms/opensandbox/__init__.py

@@ -0,0 +1 @@
+

litellm/llms/opensandbox/sandbox/__init__.py

@@ -0,0 +1 @@
+

litellm/llms/opensandbox/sandbox/transformation.py

@@ -0,0 +1,598 @@
+import asyncio
+import json
+import time
+from typing import Union, cast
+
+import httpx
+
+from litellm.constants import (
+    OPEN_SANDBOX_API_BASE_ENV_VAR,
+    OPEN_SANDBOX_API_KEY_ENV_VAR,
+    OPEN_SANDBOX_DEFAULT_CPU_LIMIT,
+    OPEN_SANDBOX_DEFAULT_ENTRYPOINT,
+    OPEN_SANDBOX_DEFAULT_LANGUAGE,
+    OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT,
+    OPEN_SANDBOX_DEFAULT_TEMPLATE,
+    OPEN_SANDBOX_DEFAULT_TIMEOUT,
+    OPEN_SANDBOX_EXECD_PORT,
+    OPEN_SANDBOX_POLL_INTERVAL,
+    OPEN_SANDBOX_READY_TIMEOUT,
+)
+from litellm.llms.base_llm.sandbox.transformation import (
+    BaseSandboxConfig,
+    CodeExecutionResult,
+    ContainerHandle,
+    SANDBOX_MAX_OUTPUT_BYTES,
+)
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    get_async_httpx_client,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+DEFAULT_SANDBOX_TIMEOUT = OPEN_SANDBOX_DEFAULT_TIMEOUT
+DEFAULT_READY_TIMEOUT = OPEN_SANDBOX_READY_TIMEOUT
+DEFAULT_POLL_INTERVAL = OPEN_SANDBOX_POLL_INTERVAL
+MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES
+
+
+class OpenSandboxSandboxConfig(BaseSandboxConfig):
+    def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler:
+        if client is not None:
+            return client
+        return get_async_httpx_client(llm_provider=httpxSpecialProvider.Sandbox)
+
+    def validate_environment(self, api_key: str | None = None, **kwargs) -> str:
+        if api_key is not None:
+            return api_key
+        return get_secret_str(OPEN_SANDBOX_API_KEY_ENV_VAR) or ""
+
+    async def acreate_sandbox(
+        self,
+        *,
+        template: str | None = None,
+        timeout: int | None = None,
+        allow_internet_access: bool | None = None,
+        api_key: str | None = None,
+        api_base: str | None = None,
+        metadata: dict[str, str] | None = None,
+        env_vars: dict[str, str] | None = None,
+        resource_limits: dict[str, str] | None = None,
+        resource_requests: dict[str, str] | None = None,
+        entrypoint: list[str] | tuple[str, ...] | None = None,
+        network_policy: dict[str, object] | None = None,
+        secure_access: bool = False,
+        use_server_proxy: bool = False,
+        ready_timeout: float | None = None,
+        poll_interval: float | None = None,
+        client: AsyncHTTPHandler | None = None,
+        **kwargs,
+    ) -> ContainerHandle:
+        key = self.validate_environment(api_key=api_key)
+        base = self._api_base(api_base)
+        ready_timeout_seconds = (
+            float(ready_timeout) if ready_timeout is not None else DEFAULT_READY_TIMEOUT
+        )
+        poll_interval_seconds = (
+            float(poll_interval) if poll_interval is not None else DEFAULT_POLL_INTERVAL
+        )
+        body = self._create_body(
+            template=template,
+            timeout=timeout,
+            allow_internet_access=allow_internet_access,
+            metadata=metadata,
+            env_vars=env_vars,
+            resource_limits=resource_limits,
+            resource_requests=resource_requests,
+            entrypoint=entrypoint,
+            network_policy=network_policy,
+            secure_access=secure_access,
+        )
+
+        response = cast(
+            httpx.Response,
+            await self._http(client).post(
+                url=f"{base}/sandboxes",
+                headers=self._lifecycle_headers(key),
+                json=body,
+            ),
+        )
+        data = response.json()
+        sandbox_id = str(data["id"])
+
+        if self._sandbox_state(data) != "Running":
+            await self._wait_until_running(
+                sandbox_id=sandbox_id,
+                api_base=base,
+                headers=self._lifecycle_headers(key),
+                client=client,
+                ready_timeout=ready_timeout_seconds,
+                poll_interval=poll_interval_seconds,
+            )
+
+        endpoint, endpoint_headers = await self._wait_for_execd_endpoint(
+            sandbox_id=sandbox_id,
+            api_base=base,
+            headers=self._lifecycle_headers(key),
+            use_server_proxy=use_server_proxy,
+            client=client,
+            ready_timeout=ready_timeout_seconds,
+            poll_interval=poll_interval_seconds,
+        )
+
+        handle = ContainerHandle(id=sandbox_id, provider="opensandbox", domain=base)
+        handle._hidden_params = {
+            "api_base": base,
+            "api_key": key,
+            "execd_endpoint": endpoint,
+            "execd_headers": endpoint_headers,
+            "use_server_proxy": use_server_proxy,
+        }
+        return handle
+
+    async def arun_code(
+        self,
+        *,
+        container: Union[ContainerHandle, str],
+        code: str,
+        api_key: str | None = None,
+        api_base: str | None = None,
+        language: str = OPEN_SANDBOX_DEFAULT_LANGUAGE,
+        use_server_proxy: bool = False,
+        ready_timeout: float | None = None,
+        poll_interval: float | None = None,
+        client: AsyncHTTPHandler | None = None,
+        **kwargs,
+    ) -> CodeExecutionResult:
+        handle = await self._ensure_handle(
+            container=container,
+            api_key=api_key,
+            api_base=api_base,
+            use_server_proxy=use_server_proxy,
+            ready_timeout=(
+                float(ready_timeout)
+                if ready_timeout is not None
+                else DEFAULT_READY_TIMEOUT
+            ),
+            poll_interval=(
+                float(poll_interval)
+                if poll_interval is not None
+                else DEFAULT_POLL_INTERVAL
+            ),
+            client=client,
+        )
+        endpoint = str(handle._hidden_params["execd_endpoint"])
+        endpoint_headers = self._as_str_dict(handle._hidden_params.get("execd_headers"))
+        base = str(
+            handle._hidden_params.get("api_base")
+            or handle.domain
+            or self._api_base(api_base)
+        )
+        lines = await self._post_code(
+            url=f"{self._endpoint_base_url(endpoint, base)}/code",
+            headers={
+                "Content-Type": "application/json",
+                "Accept": "text/event-stream",
+                "Cache-Control": "no-cache",
+                **endpoint_headers,
+            },
+            body={
+                "code": code,
+                "context": {"language": language},
+            },
+            client=client,
+        )
+        return self._parse_lines(lines)
+
+    async def adelete_sandbox(
+        self,
+        *,
+        container: Union[ContainerHandle, str],
+        api_key: str | None = None,
+        api_base: str | None = None,
+        client: AsyncHTTPHandler | None = None,
+        **kwargs,
+    ) -> bool:
+        handle = self._as_handle(container, api_base=api_base)
+        base = str(handle._hidden_params.get("api_base") or self._api_base(api_base))
+        key = self._api_key(api_key=api_key, handle=handle)
+        try:
+            response = cast(
+                httpx.Response,
+                await self._http(client).delete(
+                    url=f"{base}/sandboxes/{handle.id}",
+                    headers=self._lifecycle_headers(key),
+                ),
+            )
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 404:
+                return False
+            raise
+        return 200 <= response.status_code < 300
+
+    async def _ensure_handle(
+        self,
+        *,
+        container: Union[ContainerHandle, str],
+        api_key: str | None,
+        api_base: str | None,
+        use_server_proxy: bool,
+        ready_timeout: float,
+        poll_interval: float,
+        client: AsyncHTTPHandler | None,
+    ) -> ContainerHandle:
+        handle = self._as_handle(container, api_base=api_base)
+        if handle._hidden_params.get("execd_endpoint"):
+            return handle
+
+        base = str(handle._hidden_params.get("api_base") or self._api_base(api_base))
+        key = self._api_key(api_key=api_key, handle=handle)
+        resolved_use_server_proxy = bool(
+            handle._hidden_params.get("use_server_proxy", use_server_proxy)
+        )
+        endpoint, endpoint_headers = await self._wait_for_execd_endpoint(
+            sandbox_id=handle.id,
+            api_base=base,
+            headers=self._lifecycle_headers(key),
+            use_server_proxy=resolved_use_server_proxy,
+            client=client,
+            ready_timeout=ready_timeout,
+            poll_interval=poll_interval,
+        )
+        handle.domain = base
+        handle._hidden_params = {
+            **handle._hidden_params,
+            "api_base": base,
+            "api_key": key,
+            "execd_endpoint": endpoint,
+            "execd_headers": endpoint_headers,
+            "use_server_proxy": resolved_use_server_proxy,
+        }
+        return handle
+
+    async def _wait_until_running(
+        self,
+        *,
+        sandbox_id: str,
+        api_base: str,
+        headers: dict[str, str],
+        client: AsyncHTTPHandler | None,
+        ready_timeout: float,
+        poll_interval: float,
+    ) -> None:
+        deadline = time.monotonic() + ready_timeout
+        while True:
+            response = cast(
+                httpx.Response,
+                await self._http(client).get(
+                    url=f"{api_base}/sandboxes/{sandbox_id}",
+                    headers=headers,
+                ),
+            )
+            data = response.json()
+            state = self._sandbox_state(data)
+            if state == "Running":
+                return
+            if state in {"Failed", "Stopping", "Terminated"}:
+                raise ValueError(f"OpenSandbox sandbox {sandbox_id} entered {state}")
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"OpenSandbox sandbox {sandbox_id} was not Running within "
+                    f"{ready_timeout} seconds"
+                )
+            await asyncio.sleep(poll_interval)
+
+    async def _wait_for_execd_endpoint(
+        self,
+        *,
+        sandbox_id: str,
+        api_base: str,
+        headers: dict[str, str],
+        use_server_proxy: bool,
+        client: AsyncHTTPHandler | None,
+        ready_timeout: float,
+        poll_interval: float,
+    ) -> tuple[str, dict[str, str]]:
+        deadline = time.monotonic() + ready_timeout
+        last_error: Exception | None = None
+        while True:
+            try:
+                return await self._get_execd_endpoint(
+                    sandbox_id=sandbox_id,
+                    api_base=api_base,
+                    headers=headers,
+                    use_server_proxy=use_server_proxy,
+                    client=client,
+                )
+            except httpx.HTTPStatusError as e:
+                if e.response.status_code != 404:
+                    raise
+                last_error = e
+            except ValueError as e:
+                last_error = e
+
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"OpenSandbox execd endpoint for {sandbox_id} was not ready within "
+                    f"{ready_timeout} seconds"
+                ) from last_error
+            await asyncio.sleep(poll_interval)
+
+    async def _get_execd_endpoint(
+        self,
+        *,
+        sandbox_id: str,
+        api_base: str,
+        headers: dict[str, str],
+        use_server_proxy: bool,
+        client: AsyncHTTPHandler | None,
+    ) -> tuple[str, dict[str, str]]:
+        response = cast(
+            httpx.Response,
+            await self._http(client).get(
+                url=f"{api_base}/sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}",
+                headers=headers,
+                params={"use_server_proxy": use_server_proxy},
+            ),
+        )
+        data = response.json()
+        endpoint = data.get("endpoint")
+        if not endpoint:
+            raise ValueError(
+                f"OpenSandbox did not return an execd endpoint for {sandbox_id}"
+            )
+        return str(endpoint), self._as_str_dict(data.get("headers"))
+
+    async def _post_code(
+        self,
+        *,
+        url: str,
+        headers: dict[str, str],
+        body: dict[str, object],
+        client: AsyncHTTPHandler | None,
+    ) -> list[str]:
+        timeout = httpx.Timeout(connect=30.0, read=None, write=30.0, pool=None)
+        response = cast(
+            httpx.Response,
+            await self._http(client).post(
+                url=url,
+                headers=headers,
+                timeout=timeout,
+                json=body,
+                stream=True,
+            ),
+        )
+        return await self._read_capped_lines(response)
+
+    def _api_key(self, *, api_key: str | None, handle: ContainerHandle) -> str:
+        if api_key is not None:
+            return api_key
+        if "api_key" in handle._hidden_params:
+            return str(handle._hidden_params["api_key"])
+        return self.validate_environment()
+
+    @staticmethod
+    def _create_body(
+        *,
+        template: str | None,
+        timeout: int | None,
+        allow_internet_access: bool | None,
+        metadata: dict[str, str] | None,
+        env_vars: dict[str, str] | None,
+        resource_limits: dict[str, str] | None,
+        resource_requests: dict[str, str] | None,
+        entrypoint: list[str] | tuple[str, ...] | None,
+        network_policy: dict[str, object] | None,
+        secure_access: bool,
+    ) -> dict[str, object]:
+        body: dict[str, object] = {
+            "image": {"uri": template or OPEN_SANDBOX_DEFAULT_TEMPLATE},
+            "entrypoint": list(entrypoint or OPEN_SANDBOX_DEFAULT_ENTRYPOINT),
+            "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT,
+            "resourceLimits": resource_limits
+            or OpenSandboxSandboxConfig._default_resource_limits(),
+        }
+        if metadata:
+            body["metadata"] = metadata
+        if env_vars:
+            body["env"] = env_vars
+        if resource_requests:
+            body["resourceRequests"] = resource_requests
+        if network_policy is not None:
+            body["networkPolicy"] = network_policy
+        elif allow_internet_access is not True:
+            body["networkPolicy"] = {"defaultAction": "deny", "egress": []}
+        if secure_access:
+            body["secureAccess"] = True
+        return body
+
+    @staticmethod
+    def _default_resource_limits() -> dict[str, str]:
+        return {
+            "cpu": OPEN_SANDBOX_DEFAULT_CPU_LIMIT,
+            "memory": OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT,
+        }
+
+    @staticmethod
+    def _sandbox_state(data: object) -> str | None:
+        if not isinstance(data, dict):
+            return None
+        status = data.get("status")
+        if not isinstance(status, dict):
+            return None
+        state = status.get("state")
+        return str(state) if state is not None else None
+
+    @staticmethod
+    def _as_str_dict(value: object) -> dict[str, str]:
+        if not isinstance(value, dict):
+            return {}
+        return {str(k): str(v) for k, v in value.items()}
+
+    @staticmethod
+    def _api_base(api_base: str | None) -> str:
+        base = api_base or get_secret_str(OPEN_SANDBOX_API_BASE_ENV_VAR)
+        if not base:
+            raise ValueError(
+                "OpenSandbox api_base is required. Pass api_base or set "
+                f"{OPEN_SANDBOX_API_BASE_ENV_VAR}."
+            )
+        return str(base).rstrip("/")
+
+    @staticmethod
+    def _lifecycle_headers(api_key: str) -> dict[str, str]:
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["OPEN-SANDBOX-API-KEY"] = api_key
+        return headers
+
+    @staticmethod
+    def _endpoint_base_url(endpoint: str, api_base: str) -> str:
+        normalized_endpoint = endpoint.rstrip("/")
+        if normalized_endpoint.startswith(("http://", "https://")):
+            return normalized_endpoint
+        protocol = api_base.split("://", 1)[0] if "://" in api_base else "http"
+        return f"{protocol}://{normalized_endpoint}"
+
+    @staticmethod
+    def _as_handle(
+        container: Union[ContainerHandle, str], *, api_base: str | None
+    ) -> ContainerHandle:
+        if isinstance(container, ContainerHandle):
+            return container
+        handle = ContainerHandle(
+            id=str(container),
+            provider="opensandbox",
+            domain=OpenSandboxSandboxConfig._api_base(api_base),
+        )
+        handle._hidden_params = {}
+        return handle
+
+    @staticmethod
+    def _parse_lines(lines: list[str]) -> CodeExecutionResult:
+        messages = tuple(
+            event
+            for line in lines
+            if (event := OpenSandboxSandboxConfig._parse_sse_line(line)) is not None
+        )
+
+        def of_type(message_type: str):
+            return (m for m in messages if m.get("type") == message_type)
+
+        error = next(
+            (OpenSandboxSandboxConfig._normalize_error(m) for m in of_type("error")),
+            None,
+        )
+        execution_count = next(
+            (
+                OpenSandboxSandboxConfig._as_int(m.get("execution_count"))
+                for m in of_type("execution_count")
+                if OpenSandboxSandboxConfig._as_int(m.get("execution_count"))
+                is not None
+            ),
+            None,
+        )
+
+        return CodeExecutionResult(
+            stdout="".join(str(m.get("text", "")) for m in of_type("stdout")),
+            stderr="".join(str(m.get("text", "")) for m in of_type("stderr")),
+            results=[
+                OpenSandboxSandboxConfig._normalize_result(m) for m in of_type("result")
+            ],
+            error=error,
+            execution_count=execution_count,
+        )
+
+    @staticmethod
+    def _parse_sse_line(line: str) -> dict[str, object] | None:
+        stripped = line.strip()
+        if not stripped or stripped.startswith(
+            (
+                ":",
+                "event:",
+                "id:",
+                "retry:",
+            )
+        ):
+            return None
+        data = stripped[5:].strip() if stripped.startswith("data:") else stripped
+        if not data:
+            return None
+        try:
+            parsed = json.loads(data)
+        except json.JSONDecodeError:
+            return None
+        if not isinstance(parsed, dict):
+            return None
+        if "type" not in parsed and "code" in parsed and "message" in parsed:
+            return {
+                "type": "error",
+                "error": {
+                    "ename": str(parsed["code"]),
+                    "evalue": str(parsed["message"]),
+                    "traceback": [],
+                },
+            }
+        return parsed
+
+    @staticmethod
+    def _normalize_result(message: dict[str, object]) -> dict[str, object]:
+        results = message.get("results")
+        if isinstance(results, dict):
+            return {str(k): v for k, v in results.items()}
+        return {
+            str(k): v
+            for k, v in message.items()
+            if k not in {"type", "timestamp", "execution_count"}
+        }
+
+    @staticmethod
+    def _normalize_error(message: dict[str, object]) -> dict[str, object]:
+        raw_error = message.get("error")
+        if isinstance(raw_error, dict):
+            name = OpenSandboxSandboxConfig._first_non_none_value(
+                raw_error, "ename", "name", default=""
+            )
+            value = OpenSandboxSandboxConfig._first_non_none_value(
+                raw_error, "evalue", "value", default=""
+            )
+            traceback = OpenSandboxSandboxConfig._first_non_none_value(
+                raw_error, "traceback", default=[]
+            )
+            return {
+                "name": name,
+                "value": value,
+                "traceback": traceback,
+            }
+        return {
+            "name": OpenSandboxSandboxConfig._first_non_none_value(
+                message, "name", default=""
+            ),
+            "value": OpenSandboxSandboxConfig._first_non_none_value(
+                message, "value", "text", default=""
+            ),
+            "traceback": OpenSandboxSandboxConfig._first_non_none_value(
+                message, "traceback", default=[]
+            ),
+        }
+
+    @staticmethod
+    def _as_int(value: object) -> int | None:
+        if isinstance(value, int):
+            return value
+        if isinstance(value, str):
+            try:
+                return int(value)
+            except ValueError:
+                return None
+        return None
+
+    @staticmethod
+    def _first_non_none_value(
+        values: dict[str, object], *keys: str, default: object
+    ) -> object:
+        return next(
+            (values[key] for key in keys if key in values and values[key] is not None),
+            default,
+        )

litellm/llms/parallel_ai/search/transformation.py

@@ -67,10 +67,12 @@ class ParallelAISearchConfig(BaseSearchConfig):
         api_base: Optional[str] = None,
         **kwargs,
     ) -> Dict:
-        api_key = (
-            api_key
-            or get_secret_str("PARALLEL_AI_API_KEY")
-            or get_secret_str("PARALLEL_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"),
+            base_env_var="PARALLEL_AI_API_BASE",
+            default_api_base=self.PARALLEL_AI_API_BASE,
         )
         if not api_key:
             raise ValueError(

litellm/llms/perplexity/cost_calculator.py

@@ -98,10 +98,11 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
     if num_search_queries > 0 and search_cost_value is not None:
         # Handle both dict and float formats
         if isinstance(search_cost_value, dict):
-            # Use the "low" size as default - tests expect 0.005 / 1000
-            search_cost_per_query = (
-                _safe_float_cast(search_cost_value.get("search_context_size_low", 0))
-                / 1000
+            # search_context_cost_per_query stores the per-request price in USD
+            # (e.g. sonar low = $0.005/request). Use it directly, matching the
+            # gemini cost calculator which reads the same field per request.
+            search_cost_per_query = _safe_float_cast(
+                search_cost_value.get("search_context_size_low", 0)
             )
         else:
             search_cost_per_query = _safe_float_cast(search_cost_value)

litellm/llms/perplexity/search/transformation.py

@@ -50,7 +50,13 @@ class PerplexitySearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("PERPLEXITYAI_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("PERPLEXITYAI_API_KEY",),
+            base_env_var="PERPLEXITY_API_BASE",
+            default_api_base=self.PERPLEXITY_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "PERPLEXITYAI_API_KEY is not set. Set `PERPLEXITYAI_API_KEY` environment variable."

litellm/llms/searchapi/search/transformation.py

@@ -74,7 +74,13 @@ class SearchAPIConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("SEARCHAPI_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("SEARCHAPI_API_KEY",),
+            base_env_var="SEARCHAPI_API_BASE",
+            default_api_base=self.SEARCHAPI_API_BASE,
+        )
 
         if not api_key:
             raise ValueError(
@@ -114,6 +120,7 @@ class SearchAPIConfig(BaseSearchConfig):
         query: Union[str, List[str]],
         optional_params: dict,
         api_key: Optional[str] = None,
+        api_base: str | None = None,
         search_engine_id: Optional[str] = None,
         **kwargs,
     ) -> Dict:
@@ -137,8 +144,16 @@ class SearchAPIConfig(BaseSearchConfig):
         if isinstance(query, list):
             query = " ".join(query)
 
-        # Get API key from parameter or environment
-        api_key = api_key or get_secret_str("SEARCHAPI_API_KEY")
+        # Get API key from parameter or environment. The key is sent as a query
+        # param to api_base, so resolve it host-aware to avoid leaking a
+        # server-managed key to a caller-supplied host.
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("SEARCHAPI_API_KEY",),
+            base_env_var="SEARCHAPI_API_BASE",
+            default_api_base=self.SEARCHAPI_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "SEARCHAPI_API_KEY is not set. Set `SEARCHAPI_API_KEY` environment variable."

litellm/llms/searxng/search/transformation.py

@@ -61,7 +61,13 @@ class SearXNGSearchConfig(BaseSearchConfig):
         Some instances may require authentication via headers.
         """
         # SearXNG typically doesn't require API keys, but support optional auth
-        api_key = api_key or get_secret_str("SEARXNG_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("SEARXNG_API_KEY",),
+            base_env_var="SEARXNG_API_BASE",
+            default_api_base=None,
+        )
         if api_key:
             headers["Authorization"] = f"Bearer {api_key}"
         headers["Content-Type"] = "application/json"

litellm/llms/serper/search/transformation.py

@@ -55,7 +55,13 @@ class SerperSearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("SERPER_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("SERPER_API_KEY",),
+            base_env_var="SERPER_API_BASE",
+            default_api_base=self.SERPER_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "SERPER_API_KEY is not set. Set `SERPER_API_KEY` environment variable."

litellm/llms/tavily/search/transformation.py

@@ -64,7 +64,13 @@ class TavilySearchConfig(BaseSearchConfig):
         """
         Validate environment and return headers.
         """
-        api_key = api_key or get_secret_str("TAVILY_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("TAVILY_API_KEY",),
+            base_env_var="TAVILY_API_BASE",
+            default_api_base=self.TAVILY_API_BASE,
+        )
         if not api_key:
             raise ValueError(
                 "TAVILY_API_KEY is not set. Set `TAVILY_API_KEY` environment variable."

litellm/llms/tinyfish/search/transformation.py

@@ -67,7 +67,13 @@ class TinyfishSearchConfig(BaseSearchConfig):
         api_base: str | None = None,
         **kwargs: object,
     ) -> dict[str, str]:
-        resolved_key = api_key or get_secret_str("TINYFISH_API_KEY")
+        resolved_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("TINYFISH_API_KEY",),
+            base_env_var="TINYFISH_API_BASE",
+            default_api_base=self.TINYFISH_API_BASE,
+        )
         if not resolved_key:
             raise ValueError(
                 "TINYFISH_API_KEY is not set. Set `TINYFISH_API_KEY` environment variable."

litellm/llms/vertex_ai/realtime/transformation.py

@@ -32,6 +32,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
         self._project = project
         self._location = location
 
+    def _include_function_response_id(self) -> bool:
+        return False
+
     # ------------------------------------------------------------------
     # URL
     # ------------------------------------------------------------------

litellm/llms/you_com/search/transformation.py

@@ -64,7 +64,13 @@ class YouComSearchConfig(BaseSearchConfig):
         endpoint with the `X-API-Key` header. Otherwise fall through to the
         keyless free tier; no auth header is required.
         """
-        api_key = api_key or get_secret_str("YOUCOM_API_KEY")
+        api_key = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("YOUCOM_API_KEY",),
+            base_env_var="YOUCOM_API_BASE",
+            default_api_base=self.YOU_COM_API_BASE,
+        )
         headers["Content-Type"] = "application/json"
         # Pin Accept-Encoding to identity: the keyless `api.you.com/v1/agents/search`
         # endpoint advertises gzip content-encoding but returns body bytes the

litellm/model_prices_and_context_window_backup.json

@@ -571,7 +571,7 @@
         "output_vector_size": 1536
     },
     "amazon.titan-embed-text-v2:0": {
-        "input_cost_per_token": 2e-07,
+        "input_cost_per_token": 2e-08,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8192,
         "max_tokens": 8192,
@@ -10684,6 +10684,268 @@
         "mode": "chat",
         "output_cost_per_token": 1.923e-06
     },
+    "cloudflare/@cf/openai/gpt-oss-120b": {
+        "input_cost_per_token": 3.5e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 7.5e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/google/gemma-2b-it-lora": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 8192,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 0.0
+    },
+    "cloudflare/@cf/meta/llama-3.2-3b-instruct": {
+        "input_cost_per_token": 5.09e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 80000,
+        "max_output_tokens": 80000,
+        "max_tokens": 80000,
+        "mode": "chat",
+        "output_cost_per_token": 3.35e-07
+    },
+    "cloudflare/@cf/meta/llama-guard-3-8b": {
+        "input_cost_per_token": 4.84e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3e-08
+    },
+    "cloudflare/@cf/mistral/mistral-7b-instruct-v0.2-lora": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 15000,
+        "max_output_tokens": 15000,
+        "max_tokens": 15000,
+        "mode": "chat",
+        "output_cost_per_token": 0.0
+    },
+    "cloudflare/@cf/moonshotai/kimi-k2.7-code": {
+        "cache_read_input_token_cost": 1.9e-07,
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": {
+        "input_cost_per_token": 4.97e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 80000,
+        "max_output_tokens": 80000,
+        "max_tokens": 80000,
+        "mode": "chat",
+        "output_cost_per_token": 4.881e-06,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8": {
+        "input_cost_per_token": 1.52e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 32000,
+        "max_output_tokens": 32000,
+        "max_tokens": 32000,
+        "mode": "chat",
+        "output_cost_per_token": 2.87e-07
+    },
+    "cloudflare/@cf/meta/llama-3.2-1b-instruct": {
+        "input_cost_per_token": 2.7e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 60000,
+        "max_output_tokens": 60000,
+        "max_tokens": 60000,
+        "mode": "chat",
+        "output_cost_per_token": 2.01e-07
+    },
+    "cloudflare/@cf/moonshotai/kimi-k2.6": {
+        "cache_read_input_token_cost": 1.6e-07,
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/zai-org/glm-4.7-flash": {
+        "input_cost_per_token": 6.05e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 4e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/meta-llama/llama-2-7b-chat-hf-lora": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 8192,
+        "max_output_tokens": 8192,
+        "max_tokens": 8192,
+        "mode": "chat",
+        "output_cost_per_token": 0.0
+    },
+    "cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast": {
+        "input_cost_per_token": 2.93e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 24000,
+        "max_output_tokens": 24000,
+        "max_tokens": 24000,
+        "mode": "chat",
+        "output_cost_per_token": 2.253e-06,
+        "supports_function_calling": true
+    },
+    "cloudflare/@cf/ibm-granite/granite-4.0-h-micro": {
+        "input_cost_per_token": 1.7e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 131000,
+        "max_output_tokens": 131000,
+        "max_tokens": 131000,
+        "mode": "chat",
+        "output_cost_per_token": 1.12e-07,
+        "supports_function_calling": true
+    },
+    "cloudflare/@cf/qwen/qwen2.5-coder-32b-instruct": {
+        "input_cost_per_token": 6.6e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1e-06
+    },
+    "cloudflare/@cf/zai-org/glm-5.2": {
+        "cache_read_input_token_cost": 2.6e-07,
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/nvidia/nemotron-3-120b-a12b": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/aisingapore/gemma-sea-lion-v4-27b-it": {
+        "input_cost_per_token": 3.51e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.55e-07
+    },
+    "cloudflare/@cf/qwen/qwen3-30b-a3b-fp8": {
+        "input_cost_per_token": 5.09e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 3.35e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/google/gemma-7b-it-lora": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 3500,
+        "max_output_tokens": 3500,
+        "max_tokens": 3500,
+        "mode": "chat",
+        "output_cost_per_token": 0.0
+    },
+    "cloudflare/@cf/google/gemma-4-26b-a4b-it": {
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/mistralai/mistral-small-3.1-24b-instruct": {
+        "input_cost_per_token": 3.51e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.55e-07,
+        "supports_function_calling": true
+    },
+    "cloudflare/@cf/meta/llama-3.2-11b-vision-instruct": {
+        "input_cost_per_token": 4.85e-08,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6.76e-07,
+        "supports_vision": true
+    },
+    "cloudflare/@cf/openai/gpt-oss-20b": {
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true
+    },
+    "cloudflare/@cf/meta/llama-4-scout-17b-16e-instruct": {
+        "input_cost_per_token": 2.7e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 131000,
+        "max_output_tokens": 131000,
+        "max_tokens": 131000,
+        "mode": "chat",
+        "output_cost_per_token": 8.5e-07,
+        "supports_function_calling": true
+    },
+    "cloudflare/@cf/qwen/qwq-32b": {
+        "input_cost_per_token": 6.6e-07,
+        "litellm_provider": "cloudflare",
+        "max_input_tokens": 24000,
+        "max_output_tokens": 24000,
+        "max_tokens": 24000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-06,
+        "supports_reasoning": true
+    },
     "codestral/codestral-2405": {
         "input_cost_per_token": 0.0,
         "litellm_provider": "codestral",
@@ -15011,7 +15273,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": true
     },
     "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
         "input_cost_per_token": 1.2e-06,
@@ -15314,7 +15576,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": true
     },
     "fireworks_ai/qwen3p7-plus": {
         "cache_read_input_token_cost": 8e-08,
@@ -20088,8 +20350,6 @@
         "output_cost_per_token": 8e-06,
         "output_cost_per_token_batches": 4e-06,
         "output_cost_per_token_priority": 1.4e-05,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -20163,8 +20423,6 @@
         "output_cost_per_token": 1.6e-06,
         "output_cost_per_token_batches": 8e-07,
         "output_cost_per_token_priority": 2.8e-06,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -20238,8 +20496,6 @@
         "output_cost_per_token": 4e-07,
         "output_cost_per_token_batches": 2e-07,
         "output_cost_per_token_priority": 8e-07,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -20311,8 +20567,6 @@
         "output_cost_per_token": 1e-05,
         "output_cost_per_token_batches": 5e-06,
         "output_cost_per_token_priority": 1.7e-05,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
@@ -20354,8 +20608,6 @@
         "mode": "chat",
         "output_cost_per_token": 1e-05,
         "output_cost_per_token_batches": 5e-06,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
@@ -20377,8 +20629,6 @@
         "mode": "chat",
         "output_cost_per_token": 1e-05,
         "output_cost_per_token_batches": 5e-06,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
@@ -20667,8 +20917,6 @@
         "output_cost_per_token": 6e-07,
         "output_cost_per_token_batches": 3e-07,
         "output_cost_per_token_priority": 1e-06,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
@@ -21372,8 +21620,6 @@
         "output_cost_per_token": 1e-05,
         "output_cost_per_token_flex": 5e-06,
         "output_cost_per_token_priority": 2e-05,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -21767,6 +22013,8 @@
         "output_cost_per_token_flex": 1.5e-05,
         "output_cost_per_token_batches": 1.5e-05,
         "output_cost_per_token_priority": 6e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -21815,6 +22063,8 @@
         "output_cost_per_token_flex": 1.5e-05,
         "output_cost_per_token_batches": 1.5e-05,
         "output_cost_per_token_priority": 6e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -21859,6 +22109,8 @@
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
         "output_cost_per_token_batches": 9e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/responses",
             "/v1/batch"
@@ -21903,6 +22155,8 @@
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
         "output_cost_per_token_batches": 9e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/responses",
             "/v1/batch"
@@ -21951,6 +22205,8 @@
         "output_cost_per_token_flex": 7.5e-06,
         "output_cost_per_token_batches": 7.5e-06,
         "output_cost_per_token_priority": 3e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -21998,6 +22254,8 @@
         "output_cost_per_token_flex": 7.5e-06,
         "output_cost_per_token_batches": 7.5e-06,
         "output_cost_per_token_priority": 3e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -22038,6 +22296,8 @@
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
         "output_cost_per_token_batches": 9e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/responses",
             "/v1/batch"
@@ -22081,6 +22341,8 @@
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
         "output_cost_per_token_batches": 9e-05,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/responses",
             "/v1/batch"
@@ -22126,6 +22388,8 @@
         "output_cost_per_token_flex": 2.25e-06,
         "output_cost_per_token_batches": 2.25e-06,
         "output_cost_per_token_priority": 9e-06,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -22172,6 +22436,8 @@
         "output_cost_per_token_flex": 2.25e-06,
         "output_cost_per_token_batches": 2.25e-06,
         "output_cost_per_token_priority": 9e-06,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -22215,6 +22481,8 @@
         "output_cost_per_token": 1.25e-06,
         "output_cost_per_token_flex": 6.25e-07,
         "output_cost_per_token_batches": 6.25e-07,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -22258,6 +22526,8 @@
         "output_cost_per_token": 1.25e-06,
         "output_cost_per_token_flex": 6.25e-07,
         "output_cost_per_token_batches": 6.25e-07,
+        "regional_processing_uplift_multiplier_eu": 1.10,
+        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -22296,8 +22566,6 @@
         "mode": "responses",
         "output_cost_per_token": 0.00012,
         "output_cost_per_token_batches": 6e-05,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/batch",
             "/v1/responses"
@@ -22704,8 +22972,6 @@
         "output_cost_per_token": 2e-06,
         "output_cost_per_token_flex": 1e-06,
         "output_cost_per_token_priority": 3.6e-06,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/batch",
@@ -22787,8 +23053,6 @@
         "max_input_tokens": 272000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
-        "regional_processing_uplift_multiplier_eu": 1.10,
-        "regional_processing_uplift_multiplier_us": 1.10,
         "mode": "chat",
         "output_cost_per_token": 4e-07,
         "output_cost_per_token_flex": 2e-07,
@@ -39908,24 +40172,6 @@
         "litellm_provider": "fireworks_ai",
         "mode": "chat"
     },
-    "fireworks_ai/accounts/fireworks/models/whisper-v3": {
-        "max_tokens": 4096,
-        "max_input_tokens": 4096,
-        "max_output_tokens": 4096,
-        "input_cost_per_token": 0.0,
-        "output_cost_per_token": 0.0,
-        "litellm_provider": "fireworks_ai",
-        "mode": "audio_transcription"
-    },
-    "fireworks_ai/accounts/fireworks/models/whisper-v3-turbo": {
-        "max_tokens": 4096,
-        "max_input_tokens": 4096,
-        "max_output_tokens": 4096,
-        "input_cost_per_token": 0.0,
-        "output_cost_per_token": 0.0,
-        "litellm_provider": "fireworks_ai",
-        "mode": "audio_transcription"
-    },
     "fireworks_ai/accounts/fireworks/models/yi-34b": {
         "max_tokens": 4096,
         "max_input_tokens": 4096,
@@ -43061,6 +43307,40 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "darkbloom/gemma-4-26b": {
+        "input_cost_per_token": 3e-08,
+        "litellm_provider": "darkbloom",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1.65e-07,
+        "source": "https://www.darkbloom.dev/",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "darkbloom/gpt-oss-20b": {
+        "input_cost_per_token": 1.45e-08,
+        "litellm_provider": "darkbloom",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 7e-08,
+        "source": "https://www.darkbloom.dev/",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
     "deepseek/deepseek-v4-pro": {
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 3.625e-09,

litellm/ocr/main.py

@@ -10,7 +10,7 @@ import os
 import re
 from functools import partial
 from io import IOBase
-from typing import Any, Coroutine, Dict, Optional, Union
+from typing import Any, Callable, Coroutine, Dict, Optional, Union, cast
 
 import httpx
 
@@ -20,6 +20,7 @@ from litellm.constants import request_timeout
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.ocr.rust_bridge import RustOcr, load_rust_ocr, rust_ocr_enabled
 from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import ProviderConfigManager, client
 
@@ -28,6 +29,82 @@ base_llm_http_handler = BaseLLMHTTPHandler()
 #################################################
 
 
+def _timeout_to_seconds(
+    timeout: Optional[Union[float, httpx.Timeout]],
+) -> Optional[float]:
+    """Convert the Python OCR timeout to a single seconds value for the Rust bridge.
+
+    The Rust HTTP client takes one duration; ``httpx.Timeout`` carries separate
+    connect/read/write/pool values, so pick the read deadline as the closest
+    analog to a total-request timeout.
+    """
+    if timeout is None:
+        return None
+    if isinstance(timeout, httpx.Timeout):
+        return timeout.read
+    return float(timeout)
+
+
+def _run_rust_ocr(
+    rust_ocr: RustOcr,
+    logging_obj: LiteLLMLoggingObj,
+    provider_config: BaseOCRConfig,
+    resolve_api_key: Callable[[str], Optional[str]],
+    model: str,
+    document: dict[str, object],
+    api_key: Optional[str],
+    api_base: Optional[str],
+    optional_params: dict[str, object],
+    litellm_params: dict[str, object],
+    timeout_seconds: Optional[float],
+) -> OCRResponse:
+    """Run the Mistral OCR call through the Rust bridge and wrap the result.
+
+    Resolves the key the same way the Python path does so secret-manager backends
+    (AWS/Azure/GCP/Vault) work; the Rust bridge's own fallback only reads the
+    process environment. The request that Rust actually sends (resolved URL and
+    headers) is mirrored into pre_call so logs match the wire. Dependencies are
+    injected so this stays unit-testable without patching module globals.
+    """
+    resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY")
+    resolved_headers = provider_config.validate_environment(
+        headers={},
+        model=model,
+        api_key=resolved_api_key,
+        api_base=api_base,
+        litellm_params=litellm_params,
+    )
+    resolved_complete_url = provider_config.get_complete_url(
+        api_base=api_base,
+        model=model,
+        optional_params=optional_params,
+        litellm_params=litellm_params,
+    )
+    logging_obj.pre_call(
+        input="OCR document processing",
+        api_key=resolved_api_key,
+        additional_args={
+            "complete_input_dict": {
+                "model": model,
+                "document": document,
+                **optional_params,
+            },
+            "api_base": resolved_complete_url,
+            "headers": resolved_headers,
+        },
+    )
+    return OCRResponse.model_validate(
+        rust_ocr(
+            model=model,
+            document=document,
+            api_key=resolved_api_key,
+            api_base=api_base,
+            optional_params=optional_params,
+            timeout_seconds=timeout_seconds,
+        )
+    )
+
+
 @client
 async def aocr(
     model: str,
@@ -220,7 +297,7 @@ def ocr(
     """
     local_vars = locals()
     try:
-        litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj")  # type: ignore
+        litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj"))
         litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
         _is_async = kwargs.pop("aocr", False) is True
 
@@ -261,7 +338,6 @@ def ocr(
         if dynamic_api_base:
             api_base = dynamic_api_base
 
-        # Get provider config
         ocr_provider_config: Optional[BaseOCRConfig] = (
             ProviderConfigManager.get_provider_ocr_config(
                 model=model,
@@ -278,17 +354,14 @@ def ocr(
             f"OCR call - model: {model}, provider: {custom_llm_provider}"
         )
 
-        # Get litellm params using GenericLiteLLMParams (same as responses API)
         litellm_params = GenericLiteLLMParams(**kwargs)
 
-        # Extract OCR-specific parameters from kwargs
         supported_params = ocr_provider_config.get_supported_ocr_params(model=model)
         non_default_params = {}
         for param in supported_params:
             if param in kwargs:
                 non_default_params[param] = kwargs.pop(param)
 
-        # Map parameters to provider-specific format
         optional_params = ocr_provider_config.map_ocr_params(
             non_default_params=non_default_params,
             optional_params={},
@@ -297,7 +370,8 @@ def ocr(
 
         verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}")
 
-        # Pre Call logging
+        effective_timeout = timeout or request_timeout
+
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
             model=model,
@@ -309,12 +383,35 @@ def ocr(
             custom_llm_provider=custom_llm_provider,
         )
 
-        # Call the handler - pass document dict directly
+        # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge.
+        if custom_llm_provider == "mistral" and rust_ocr_enabled():
+            rust_ocr = load_rust_ocr()
+            if rust_ocr is None:
+                verbose_logger.debug(
+                    "Rust OCR bridge unavailable; falling back to Python path"
+                )
+            else:
+                from litellm.secret_managers.main import get_secret_str
+
+                return _run_rust_ocr(
+                    rust_ocr=rust_ocr,
+                    logging_obj=litellm_logging_obj,
+                    provider_config=ocr_provider_config,
+                    resolve_api_key=get_secret_str,
+                    model=model,
+                    document=document,
+                    api_key=api_key,
+                    api_base=api_base,
+                    optional_params=optional_params,
+                    litellm_params=dict(litellm_params),
+                    timeout_seconds=_timeout_to_seconds(effective_timeout),
+                )
+
         response = base_llm_http_handler.ocr(
             model=model,
-            document=document,  # Pass the entire document dict
+            document=document,
             optional_params=optional_params,
-            timeout=timeout or request_timeout,
+            timeout=effective_timeout,
             logging_obj=litellm_logging_obj,
             api_key=api_key,
             api_base=api_base,

litellm/ocr/rust_bridge.py

@@ -0,0 +1,74 @@
+"""
+Optional Rust-backed OCR path.
+
+Enable with ``litellm.use_litellm_rust()``; the sync ``litellm.ocr()`` entrypoint
+then routes supported Mistral calls through the compiled ``litellm_python_bridge``
+extension, which performs the whole OCR call (URL, headers, HTTP, parse) in Rust.
+
+No module-level ``litellm`` imports keep this a leaf so ``litellm/ocr/main.py``
+can import it statically without forming an import cycle.
+"""
+
+from __future__ import annotations
+
+from typing import Final, Protocol, cast
+
+
+class RustOcr(Protocol):
+    """Signature of the compiled ``litellm_python_bridge.ocr`` entrypoint."""
+
+    def __call__(
+        self,
+        model: str,
+        document: dict[str, object],
+        api_key: str | None,
+        api_base: str | None,
+        optional_params: dict[str, object],
+        timeout_seconds: float | None,
+    ) -> dict[str, object]: ...
+
+
+class _Unset:
+    """Sentinel type so ``ocr=None`` can clear a prior injection while omission preserves it."""
+
+
+_UNSET: Final[_Unset] = _Unset()
+
+_rust_ocr_enabled = False
+_rust_ocr_impl: RustOcr | None = None
+
+
+def use_litellm_rust(
+    enabled: bool = True, *, ocr: RustOcr | None | _Unset = _UNSET
+) -> None:
+    """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension.
+
+    ``ocr`` injects the bridge callable; when omitted the compiled extension is
+    loaded on demand and any previously injected bridge is preserved. Pass
+    ``ocr=None`` explicitly to clear a prior injection.
+    """
+    global _rust_ocr_enabled, _rust_ocr_impl
+    _rust_ocr_enabled = enabled
+    if not isinstance(ocr, _Unset):
+        _rust_ocr_impl = ocr
+
+
+def rust_ocr_enabled() -> bool:
+    """Whether the Rust OCR path has been turned on via ``use_litellm_rust()``."""
+    return _rust_ocr_enabled
+
+
+def load_rust_ocr() -> RustOcr | None:
+    """Return the Rust OCR callable, or ``None`` when no bridge is available.
+
+    Prefers an injected implementation, otherwise loads the compiled
+    ``litellm_python_bridge`` extension; a missing extension yields ``None`` so
+    the caller can fall back to the Python path instead of hard-failing.
+    """
+    if _rust_ocr_impl is not None:
+        return _rust_ocr_impl
+    try:
+        import litellm_python_bridge
+    except ImportError:
+        return None
+    return cast(RustOcr, litellm_python_bridge.ocr)

litellm/provider_endpoints_support_backup.json

@@ -1835,6 +1835,23 @@
         "interactions": true
       }
     },
+    "darkbloom": {
+      "display_name": "Darkbloom (`darkbloom`)",
+      "url": "https://docs.litellm.ai/docs/providers/darkbloom",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": false,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "predibase": {
       "display_name": "Predibase (`predibase`)",
       "url": "https://docs.litellm.ai/docs/providers/predibase",

litellm/proxy/_experimental/mcp_server/AGENTS.md

@@ -0,0 +1,95 @@
+# Experimental MCP Server Change Guidelines
+
+Read @../../../../CLAUDE.md and @CLAUDE.md before changing this package.
+
+This directory owns the proxy-hosted MCP server implementation. Keep changes
+inside the module that owns the behavior, and only reach outside this package
+when the public type contract, database schema, dashboard, or cross-proxy route
+wiring must change with it.
+
+## File Structure
+
+Respect the current package boundaries:
+
+```text
+litellm/proxy/_experimental/mcp_server/
+  AGENTS.md
+  CLAUDE.md
+  server.py                  # ASGI/MCP route handling, sessions, tool calls   [PR7: 7-arm only — move BYOK/OAuth pre-fetch into resolver]
+  mcp_server_manager.py      # upstream server registry, clients, tool routing  [PR7: _create_mcp_client swaps resolve_mcp_auth -> resolve_credentials]
+  auth/
+    user_api_key_auth_mcp.py # LiteLLM admission auth and MCP request headers
+    token_exchange.py        # OAuth token exchange handling                    [unchanged; V1TokenExchangeAdapter delegates here]
+    litellm_auth_handler.py  # authenticated-user adapter for MCP sessions
+  outbound_credentials/      # NEW — typed upstream-credential resolution (resolve_credentials + arms)
+    __init__.py              # public surface: resolve_credentials, the configs, CredError
+    result.py                # Ok | Error union (pure stdlib)
+    types.py                 # AuthConfig union, CredError, Subject, ServerSpec
+    httpx_auth.py            # NoOpAuth, StaticHeaderAuth (every mode -> one httpx.Auth)
+    resolver.py              # resolve_credentials(): exhaustive per-mode match + assert_never
+    seams.py                 # injected Protocols (one per cache-touching mode)
+    v1_adapters.py           # v1-backed seam bodies; delegate to auth/oauth2/db owners
+    adapter.py               # to_subject / to_server_spec / raise_public (v1 <-> v2 boundary)
+  discoverable_endpoints.py  # MCP OAuth metadata, authorize, token, callback
+  byok_oauth_endpoints.py    # BYOK OAuth UI/API flow
+  oauth_utils.py             # redirect URI and proxy base URL validation
+  oauth2_token_cache.py      # OAuth2 and per-user token resolution/cache        [PR7: resolve_mcp_auth removed; cache class stays, V1OAuth2CacheAdapter delegates to async_get_token]
+  db.py                      # MCP server, credential, env var, submission DB access  [unchanged; V1ByokStore delegates to _get_byok_credential / get_user_credential]
+  toolset_db.py              # MCP toolset DB access
+  rest_endpoints.py          # proxy REST facade for listing/calling MCP tools   [PR7: 7-arm only — pass identity + inbound token down instead of mcp_auth_header]
+  openapi_to_mcp_generator.py# OpenAPI spec to MCP tool generation
+  sampling_handler.py        # MCP sampling to LiteLLM completion flow
+  elicitation_handler.py     # MCP elicitation relay flow
+  semantic_tool_filter.py    # semantic filtering of available MCP tools
+  guardrail_translation/
+    handler.py               # MCP guardrail result translation
+  sse_transport.py           # SSE transport implementation
+  mcp_context.py             # contextvars for MCP request/session metadata
+  mcp_debug.py               # debug helpers
+  tool_registry.py           # in-memory MCP tool registry helpers
+  cost_calculator.py         # MCP tool cost calculation
+  ui_session_utils.py        # dashboard session auth context helpers
+  utils.py                   # shared primitives used by several modules
+```
+
+Do not add broad catch-all modules. Prefer the existing owner above, and add a
+new file only for a distinct capability that would otherwise make an existing
+module materially harder to understand.
+
+## Implementation Rules
+
+- Preserve the boundary between LiteLLM admission auth and upstream MCP auth.
+  Admission belongs in `auth/user_api_key_auth_mcp.py`; upstream token exchange,
+  delegated auth, per-user OAuth, BYOK, and raw header forwarding belong in the
+  dedicated OAuth/header modules.
+- Treat `none`, bearer/API key, OAuth, OAuth token exchange, delegated upstream
+  auth, SSE, streamable HTTP, and stdio as separate flows. Do not collapse them
+  behind a single generic branch unless tests prove every mode still behaves
+  correctly.
+- Be especially careful with `available_on_public_internet: false` combined with
+  `delegate_auth_to_upstream: true`. The local `CLAUDE.md` explains the anonymous
+  upstream PKCE path that must remain intentional.
+- Keep database-backed fields in sync across migrations, typed models under
+  `litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this
+  package, and dashboard state when the field is user-visible.
+- Use the official MCP SDK types and established LiteLLM Pydantic models where
+  they exist. Avoid untyped protocol dictionaries at package boundaries.
+- Keep security-sensitive logic easy to audit. Header forwarding, IP filtering,
+  public internet checks, token storage, env var interpolation, and credential
+  encryption need focused tests for both allowed and rejected paths.
+- Avoid adding comments to new code unless they explain non-obvious security or
+  protocol behavior. Prefer clear names and small functions.
+
+## Tests
+
+Mirror this package under `tests/test_litellm/proxy/_experimental/mcp_server/`.
+For regressions, extend the existing mapped test file instead of creating a new
+one. Use subdirectories that match the implementation path, such as
+`auth/test_token_exchange.py` for `auth/token_exchange.py` and
+`guardrail_translation/test_mcp_guardrail_handler.py` for
+`guardrail_translation/handler.py`.
+
+Use `tests/mcp_tests/` only when extending an existing broader MCP integration
+scenario that already lives there. Route, auth, tool listing, tool execution,
+OAuth, sampling, elicitation, DB, and dashboard-session changes should have
+focused coverage in the mirrored `tests/test_litellm/...` path first.

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -12,6 +12,7 @@ from litellm.proxy._types import (
     LiteLLM_TeamTable,
     ProxyException,
     SpecialHeaders,
+    SpecialMCPServerNames,
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
@@ -642,6 +643,15 @@ class MCPRequestHandler:
                     user_api_key_auth
                 )
             )
+
+            # The key explicitly opted out of every MCP server. This overrides
+            # team inheritance and additive grants (mirrors no-default-models).
+            if (
+                SpecialMCPServerNames.no_mcp_servers.value
+                in allowed_mcp_servers_for_key
+            ):
+                return []
+
             allowed_mcp_servers_for_team = (
                 await MCPRequestHandler._get_allowed_mcp_servers_for_team(
                     user_api_key_auth
@@ -1058,6 +1068,13 @@ class MCPRequestHandler:
             if key_object_permission is None:
                 return []
 
+            # Sentinel opt-out: surface it unexpanded so the caller can short-circuit
+            # to zero servers instead of inheriting the team.
+            if SpecialMCPServerNames.no_mcp_servers.value in (
+                key_object_permission.mcp_servers or []
+            ):
+                return [SpecialMCPServerNames.no_mcp_servers.value]
+
             # Permission entries may be server_ids OR names/aliases — expand to ids.
             direct_mcp_servers = global_mcp_server_manager.expand_permission_list(
                 key_object_permission.mcp_servers or []

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -80,6 +80,7 @@ from litellm.proxy._types import (
     MCPEnvVar,
     MCPTransport,
     MCPTransportType,
+    SpecialMCPServerNames,
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
@@ -1349,6 +1350,17 @@ class MCPServerManager:
         allow_all_server_ids = self.get_allow_all_keys_server_ids()
 
         try:
+            # The key explicitly opted out of every MCP server. Return zero before
+            # layering on allow_all_keys servers so the opt-out is absolute.
+            key_object_permission = (
+                user_api_key_auth.object_permission if user_api_key_auth else None
+            )
+            if key_object_permission is not None and (
+                SpecialMCPServerNames.no_mcp_servers.value
+                in (key_object_permission.mcp_servers or [])
+            ):
+                return []
+
             # Check if object_permission.mcp_servers is explicitly set
             has_explicit_object_permission = False
             if user_api_key_auth and user_api_key_auth.object_permission:

litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py

@@ -0,0 +1,73 @@
+"""Typed upstream-credential resolution for MCP servers.
+
+This subpackage houses the typed credential vocabulary and the ``resolve_credentials``
+dispatch. A server declares one per-mode config from the ``AuthConfig`` discriminated union;
+``UpstreamCredentialProvider.resolve_credentials`` selects one arm and returns an ``httpx.Auth``
+or a typed ``CredError``. Failures are modeled as values via :mod:`.result` (``Result[T,
+CredError]``) rather than raised, so every seam is total. Nothing here is wired onto a live
+request path yet.
+"""
+
+from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import (
+    NoOpAuth,
+    StaticHeaderAuth,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.resolver import (
+    UpstreamCredentialProvider,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
+    Error,
+    Ok,
+    Result,
+)
+from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+    Ambient,
+    ApiKeyConfig,
+    ApiKeySource,
+    AssumeRole,
+    AuthConfig,
+    AuthorizationCodeConfig,
+    AuthSpecKind,
+    AwsCredentialSource,
+    AwsSigV4Config,
+    Byok,
+    ClientCredentialsConfig,
+    CredError,
+    NoneConfig,
+    PassthroughConfig,
+    ServerSpec,
+    SharedKey,
+    StaticKeys,
+    Subject,
+    TokenExchangeConfig,
+    parse_auth_spec_kind,
+)
+
+__all__ = [
+    "Ok",
+    "Error",
+    "Result",
+    "NoOpAuth",
+    "StaticHeaderAuth",
+    "UpstreamCredentialProvider",
+    "AuthSpecKind",
+    "CredError",
+    "Subject",
+    "ServerSpec",
+    "AuthConfig",
+    "parse_auth_spec_kind",
+    "AuthorizationCodeConfig",
+    "ClientCredentialsConfig",
+    "TokenExchangeConfig",
+    "ApiKeyConfig",
+    "ApiKeySource",
+    "SharedKey",
+    "Byok",
+    "PassthroughConfig",
+    "NoneConfig",
+    "AwsSigV4Config",
+    "AwsCredentialSource",
+    "StaticKeys",
+    "AssumeRole",
+    "Ambient",
+]

litellm/proxy/_experimental/mcp_server/outbound_credentials/httpx_auth.py

@@ -0,0 +1,45 @@
+"""Concrete `httpx.Auth` objects the resolver returns for the self-contained modes.
+
+These are the egress credential as the SDK consumes it: an `httpx.Auth` attached to the
+upstream `AsyncClient`. The OAuth-flow modes (`authorization_code`, `client_credentials`,
+`token_exchange`) return SDK-provided auth objects instead and land later.
+
+`auth_flow` mutating the outbound request is the `httpx.Auth` contract, not a house-style
+violation: the request is httpx's object, and these carry no state of their own.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+
+import httpx
+from pydantic import SecretStr
+
+
+class NoOpAuth(httpx.Auth):
+    """Attaches nothing — the `none` mode (and the seam-level default)."""
+
+    def auth_flow(
+        self, request: httpx.Request
+    ) -> Generator[httpx.Request, httpx.Response, None]:
+        yield request
+
+
+class StaticHeaderAuth(httpx.Auth):
+    """Sets one fixed header on every request — the `api_key` family and `passthrough`.
+
+    The header value is a live credential (a bearer token, an API key, a forwarded user
+    token), so it is held as a `SecretStr` and unwrapped only when written onto the request.
+    That keeps it masked in reprs, `vars()`, tracebacks, and structured logs, matching the
+    `SecretStr` discipline the config models use.
+    """
+
+    def __init__(self, header_value: str, header_name: str = "Authorization") -> None:
+        self.header_name = header_name
+        self._header_value = SecretStr(header_value)
+
+    def auth_flow(
+        self, request: httpx.Request
+    ) -> Generator[httpx.Request, httpx.Response, None]:
+        request.headers[self.header_name] = self._header_value.get_secret_value()
+        yield request