scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_batch_ui_logs

Browse source 
# Conflicts:
#	tests/proxy_unit_tests/test_check_batch_cost.py
This commit is contained in:
mateo-berri 2026-09-04 12:00:14 -07:00
commit 7e51fbc819
395 changed files with 19196 additions and 7415 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.github/actions/setup-uv-with-retries/action.yml vendored
View file

@ -1,11 +1,7 @@
name: "Set up uv with retries"
description: >-
Install uv via astral-sh/setup-uv, retrying on transient failures. Even with
an exact pinned version, the action resolves the artifact URL by fetching
https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a
single request with no retry, timeout, or fallback, so one connection-level
network error ("fetch failed") fails the whole job before any test runs.
Retrying the full step covers the manifest fetch and the binary download.
Install uv via astral-sh/setup-uv, retrying the full setup step so manifest
resolution and binary downloads get fresh attempts after transient failures.
inputs:
version:
@ -18,7 +14,7 @@ runs:
- name: Set up uv (attempt 1)
id: attempt-1
continue-on-error: true
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
version: ${{ inputs.version }}
@ -31,7 +27,7 @@ runs:
id: attempt-2
if: steps.attempt-1.outcome == 'failure'
continue-on-error: true
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
version: ${{ inputs.version }}
@ -42,6 +38,6 @@ runs:
- name: Set up uv (attempt 3)
if: steps.attempt-2.outcome == 'failure'
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
version: ${{ inputs.version }}

5
.github/ci-coverage-allowlist.yml vendored
View file

@ -79,6 +79,11 @@ test_paths:
- tests/load_tests/test_otel_load_test.py
- tests/load_tests/test_vertex_embeddings_load_test.py
- tests/load_tests/test_vertex_load_tests.py
- reason: >-
Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run
locally rather than in pull-request jobs
paths:
- tests/load_tests/test_granian_admission_saturation.py
- reason: >-
A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a

29
.github/workflows/_test-unit-base.yml vendored
View file

@ -57,9 +57,15 @@ permissions:
jobs:
run:
name: Run tests
name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }}
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.job-timeout-minutes }}
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
env:
UV_PYTHON: ${{ matrix.python-version }}
permissions:
contents: read
pull-requests: read
@ -82,7 +88,7 @@ jobs:
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
python-version: ${{ matrix.python-version }}
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
@ -96,12 +102,10 @@ jobs:
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
path: ${{ env.UV_CACHE_DIR }}
key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
@ -113,6 +117,7 @@ jobs:
timeout-minutes: 8
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
@ -134,13 +139,7 @@ jobs:
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
DIST: ${{ inputs.dist }}
# coverage.py's sys.monitoring backend (PEP 669), the cheapest core it has.
# It is only the default from Python 3.14, and these shards run 3.12, so it
# has to be asked for. Coverage refuses it when branch measurement is on
# (`branch_right_left` needs > 3.14.0a5) and falls back to the slow core with
# a `no-sysmon` warning, so turning on `branch = true` here means giving this
# back until the runners move to 3.14.
COVERAGE_CORE: sysmon
COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} \
@ -167,7 +166,7 @@ jobs:
fi
- name: Save coverage report
if: always() && steps.changes.outputs.decision != 'skip'
if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip'
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}

1
.github/workflows/test-unit.yml vendored
View file

@ -96,7 +96,6 @@ jobs:
- shard: misc
artifact-name: misc
test-path: >-
tests/sdk_function_trace
tests/test_litellm/batches
tests/test_litellm/secret_managers
tests/test_litellm/a2a_protocol

16
basedpyright-code-budget.json
View file

@ -3,7 +3,7 @@
"limit": 14074
},
"reportArgumentType": {
"limit": 2214
"limit": 2206
},
"reportAssignmentType": {
"limit": 319
@ -18,7 +18,7 @@
"limit": 40
},
"reportDeprecated": {
"limit": 211
"limit": 209
},
"reportDuplicateImport": {
"limit": 19
@ -45,7 +45,7 @@
"limit": 24
},
"reportInvalidTypeForm": {
"limit": 34
"limit": 30
},
"reportInvalidTypeVarUse": {
"limit": 2
@ -57,7 +57,7 @@
"limit": 5601
},
"reportMissingTypeArgument": {
"limit": 15287
"limit": 15285
},
"reportMissingTypeStubs": {
"limit": 40
@ -99,19 +99,19 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44362
"limit": 44360
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38323
"limit": 38309
},
"reportUnknownParameterType": {
"limit": 19624
"limit": 19622
},
"reportUnknownVariableType": {
"limit": 29861
"limit": 29846
},
"reportUnnecessaryCast": {
"limit": 111

1
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -202,6 +202,7 @@ class CheckBatchCost:
metadata: dict[str, object] = {
"user_api_key_user_id": job.created_by,
"user_api_key": api_key,
"user_api_key_hash": api_key,
"user_api_key_team_id": team_id,
**(await self._get_user_info(batch_id, job.created_by)),
}

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -5,6 +5,7 @@ datasource client {
generator client {
provider = "prisma-client-py"
recursive_type_depth = -1
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
}

5
litellm-rust/AGENTS.md
View file

@ -1,17 +1,18 @@
# AGENTS.md
litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
litellm-rust has five crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
## Crates
| Crate | Role |
|-------|------|
| litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
| litellm-config | Config-loading boundary. Returns resolved core deployment data and optionally delegates loading to Python. |
| litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
Dependency direction is acyclic: `litellm-config` depends on `litellm-core`, the gateway depends on both, and `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`. The interop foundation depends on no LiteLLM domain crate.
## Where a route lives

1
litellm-rust/CLAUDE.md
View file

@ -24,6 +24,7 @@ the base when behavior is genuinely different, and say so explicitly in the PR.
## Crates (see AGENTS.md)
`litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
`litellm-config` is the config-loading boundary and returns resolved core types.
`litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop`
holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate

33
litellm-rust/Cargo.lock generated
View file

@ -1412,8 +1412,8 @@ dependencies = [
"base64",
"futures-channel",
"futures-util",
"litellm-config",
"litellm-core",
"pyo3",
"reqwest",
"serde",
"serde_json",
@ -1425,6 +1425,16 @@ dependencies = [
"tracing",
]
[[package]]
name = "litellm-config"
version = "0.1.0"
dependencies = [
"litellm-core",
"pyo3",
"serde_json",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-core"
version = "0.1.0"
@ -1435,14 +1445,17 @@ dependencies = [
"aws-sigv4",
"aws-smithy-runtime-api",
"aws-types",
"base64",
"rand 0.8.7",
"reqwest",
"rstest",
"serde",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.19",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
@ -1461,7 +1474,6 @@ dependencies = [
"tokio",
"tokio-tungstenite",
"tracing",
"tracing-subscriber",
]
[[package]]
@ -1511,6 +1523,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "mio"
version = "1.2.2"
@ -1969,6 +1991,7 @@ dependencies = [
"hyper-util",
"js-sys",
"log",
"mime_guess",
"percent-encoding",
"pin-project-lite",
"quinn",
@ -2736,6 +2759,12 @@ version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-ident"
version = "1.0.24"

4
litellm-rust/Cargo.toml
View file

@ -1,6 +1,7 @@
[workspace]
members = [
"crates/core",
"crates/config",
"crates/ai-gateway",
"crates/python-interop",
"crates/python-bridge",
@ -17,6 +18,7 @@ repository = "https://github.com/BerriAI/litellm"
tracing = "0.1"
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
litellm-core = { path = "crates/core" }
litellm-config = { path = "crates/config" }
litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
litellm-python-interop = { path = "crates/python-interop" }
axum = "0.7"
@ -24,7 +26,7 @@ pyo3 = "0.29.2"
pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
pythonize = "0.29.0"
rand = "0.8"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "http2", "stream"] }
rstest = "0.26.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["float_roundtrip"] }

4
litellm-rust/README.md
View file

@ -25,11 +25,12 @@ coverage and production evidence.
| Crate | Role |
|-------|------|
| litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. |
| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
| litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
## Layout
@ -38,6 +39,7 @@ crates/
core/ The SDK: route modules + provider transforms.
src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client
src/providers/anthropic/messages/transformation.rs
config/ Config loading and resolved deployments.
ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints.
python-interop/ Domain-neutral PyO3 conversion and GIL primitives.
python-bridge/ PyO3 API adapter for Python LiteLLM.

2
litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md
View file

@ -45,7 +45,7 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
22. A Python -> Rust bridge keeps the Python side minimal: the Python interface only marshals inputs and calls the Rust interface, with no transform, handler, or business logic. Aim for well under 100 lines of interface code per route; if the Python grows past that, the logic belongs in Rust.
23. Do not bloat `litellm/main.py`. A route's provider dispatch lives in a thin dispatch class under `litellm/llms/<provider>/<route>/` that calls the Rust bridge; `main.py` only instantiates it and calls its sync/async method.
24. Do not add new feature flags unless explicitly requested. Reuse the existing litellm rust rollout mechanism (`use_litellm_rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
24. Do not add new feature flags unless explicitly requested. Reuse the existing LiteLLM Rust rollout mechanism (`litellm.rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
## Checks before push

9
litellm-rust/crates/ai-gateway/AGENTS.md
View file

@ -9,19 +9,15 @@ such as `litellm_core::messages::messages`. No provider handler lives here.
src/
main.rs # entrypoint: build AppState (router + master key), bind, serve
state.rs # AppState — shared Arc<Router> + master_key
gil.rs # GIL-activity tracker (records Python acquisitions)
auth/ # authentication as an axum extractor — added to handler args
mod.rs # RequireMasterKey: FromRequestParts, single master key (LITELLM_MASTER_KEY)
routes/ # one module per route, all matching the same template
AGENTS.md # ← the route template (read this before adding a route)
mod.rs # app(): merges every module's router()
health.rs # simple route (one file): router() + liveness/readiness
gil.rs # simple route (one file): router() + GET /health/gil
realtime/ # route with logic → axum surface + a no-axum service:
mod.rs # router() + handler + WS<->events adapter (the axum surface)
service.rs # business logic (select deployment, call provider) — no axum, testable
python/ # Python interop (feature: python-config) — load-time only
mod.rs, config.rs, AGENTS.md
```
## Rules
@ -53,5 +49,6 @@ proxy in a later phase. Health routes don't add the extractor (unauthenticated).
## Python interop
Anything that calls into Python lives in `python/` and is **load-time only** — see
`python/AGENTS.md`. The realtime data path never takes the GIL.
Python-backed loading lives in `litellm-config` and is **load-time only**. The
gateway's `python-config` feature forwards to that crate. The realtime data path
never takes the GIL.

2
litellm-rust/crates/ai-gateway/ARCHITECTURE.md
View file

@ -9,4 +9,6 @@ flowchart LR
C[client] <--> G[Rust ai-gateway<br/>LLM inference]
G <--> O[OpenAI realtime]
G -. spend tracking callback .-> P[litellm proxy]
F[litellm-config<br/>load-time only] --> G
F -. Python backend .-> P
```

6
litellm-rust/crates/ai-gateway/Cargo.toml
View file

@ -16,6 +16,7 @@ required-features = ["server"]
[dependencies]
tracing.workspace = true
litellm-core = { workspace = true, features = ["bedrock-auth"] }
litellm-config.workspace = true
# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
# Python proxy callbacks API.
reqwest.workspace = true
@ -31,14 +32,15 @@ subtle = { workspace = true, optional = true }
# sha2 hashes the master key into user_api_key_hash (matches the proxy's
# SHA-256 hash_token) so the plaintext credential never enters a log payload.
sha2 = { workspace = true, optional = true }
pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
tower = { version = "0.5.3", features = ["util"], optional = true }
[features]
default = []
server = ["dep:axum", "dep:subtle", "dep:sha2"]
# Build the gateway's config from the proxy YAML via an embedded Python
# interpreter (links libpython; requires `litellm` importable at runtime).
python-config = ["dep:pyo3"]
python-config = ["litellm-config/python"]
trace-parity = ["server", "dep:tower", "litellm-core/observability"]
[dev-dependencies]
futures-channel = "0.3"

22
litellm-rust/crates/ai-gateway/README.md
View file

@ -6,25 +6,30 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame.
## Crates
`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route:
`litellm-rust` has five crates. A crate is a layer or shared foundation, not a route:
| Crate | Role |
|-------|------|
| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. |
| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. |
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
- **Client endpoint:** `wss://<host>/v1/realtime?model=<model>` (WebSocket)
- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset)
- **Health:** `GET /health/readiness`, `GET /health/liveness`, `GET /health/gil`
- **Health:** `GET /health/readiness`, `GET /health/liveness`
- **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging))
> **Realtime serving is pure Rust.** Python is used at **load time only** — to
> read the config once at boot. The realtime hot path never touches Python.
The former `/health/gil` route and its acquisition counter were removed. They
only observed the single startup config load and did not prove that every GIL
acquisition was instrumented
## Configuration (config.yaml)
The gateway loads its `model_list` from a **config.yaml**, the same as the
@ -43,9 +48,10 @@ model_list:
LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway
```
At boot the gateway calls into `litellm.proxy.read_model_list`, which reuses the
**real proxy config reader** (`ProxyConfig.get_config`). That means everything
the proxy supports in config.yaml works here too:
At boot `litellm-config` calls into `litellm.proxy.read_model_list` and returns
resolved deployments to the gateway, which constructs the router. The Python
backend still reuses the **real proxy config reader** (`ProxyConfig.get_config`),
so everything the proxy supports in config.yaml works here too:
- `include:` to merge in other config files,
- `os.environ/VAR` secret references (resolved via the secret manager, never
@ -82,8 +88,8 @@ stand-in built from the environment:
|---|---|---|
| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). |
This mode links no libpython and needs no config file, but it only supports one
hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
The default workspace build links no libpython and needs no config file. This
fallback mode only supports one hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
stand-in only for the leanest possible build.
## Request logging

6
litellm-rust/crates/ai-gateway/config.yaml
View file

@ -1,8 +1,8 @@
# Sample realtime config for the LiteLLM Rust AI Gateway.
#
# The gateway loads this model_list at boot via the embedded python config
# reader (litellm.proxy.read_model_list), which reuses the proxy's own reader —
# so include:, os.environ/ secrets, and DB-stored models all work here too.
# litellm-config resolves this model_list at boot through the Python config
# reader (litellm.proxy.read_model_list), then the gateway builds its router.
# Includes, environment secrets, and database-stored models still work.
#
# Secrets are referenced (never inlined) via os.environ/. A real deploy can
# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH).

58
litellm-rust/crates/ai-gateway/src/gil.rs
View file

@ -1,58 +0,0 @@
//! GIL-activity tracking.
//!
//! Every acquisition of the Python GIL is recorded here so the `/health/gil`
//! endpoint can report whether Python was touched recently. The design goal is
//! that the GIL is acquired **only at load time** (config read) and never on the
//! realtime hot path — polling this endpoint during traffic should show the
//! count holding steady and `acquired_last_30s` falling to `false`.
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
/// Window (seconds) for the "recently acquired" signal.
pub const RECENT_WINDOW_SECS: u64 = 30;
static GIL_ACQUISITIONS: AtomicU64 = AtomicU64::new(0);
/// Unix seconds of the last acquisition; `0` means "never".
static LAST_GIL_UNIX_SECS: AtomicU64 = AtomicU64::new(0);
fn now_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
/// Record that the GIL was just acquired. Call immediately before taking the GIL.
///
/// Only invoked under the `python-config` feature; without it the gateway never
/// touches Python, so the recorder is unused (and the endpoint reports zero).
#[cfg_attr(not(feature = "python-config"), allow(dead_code))]
pub fn record_acquisition() {
GIL_ACQUISITIONS.fetch_add(1, Ordering::Relaxed);
LAST_GIL_UNIX_SECS.store(now_unix_secs(), Ordering::Relaxed);
}
/// Point-in-time view of GIL activity.
pub struct GilSnapshot {
pub total_acquisitions: u64,
pub seconds_since_last: Option<u64>,
pub acquired_last_30s: bool,
}
/// Read the current GIL-activity snapshot.
pub fn snapshot() -> GilSnapshot {
let total = GIL_ACQUISITIONS.load(Ordering::Relaxed);
let last = LAST_GIL_UNIX_SECS.load(Ordering::Relaxed);
let seconds_since_last = if last == 0 {
None
} else {
Some(now_unix_secs().saturating_sub(last))
};
let acquired_last_30s = seconds_since_last.is_some_and(|secs| secs <= RECENT_WINDOW_SECS);
GilSnapshot {
total_acquisitions: total,
seconds_since_last,
acquired_last_30s,
}
}

12
litellm-rust/crates/ai-gateway/src/lib.rs
View file

@ -10,29 +10,23 @@
//! - [`io`]: compatibility exports and realtime WebSocket splice helpers.
//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling
//! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway`
//! binary turns on. The `python-config` feature additionally pulls in [`python`]
//! for the load-time config reader.
//! binary turns on.
pub mod audio_transcription;
mod client;
pub mod io;
pub mod ocr;
/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
/// the `python-config` reader, so it is available without either feature.
pub mod gil;
#[cfg(feature = "server")]
pub mod auth;
#[cfg(feature = "server")]
pub mod routes;
#[cfg(feature = "server")]
pub mod state;
#[cfg(feature = "trace-parity")]
pub mod trace_parity;
mod constants;
pub mod integrations;
#[cfg(feature = "server")]
mod realtime;
#[cfg(feature = "python-config")]
pub mod python;

10
litellm-rust/crates/ai-gateway/src/main.rs
View file

@ -14,12 +14,12 @@ use std::sync::Arc;
use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key};
use litellm_ai_gateway::routes;
use litellm_ai_gateway::state::AppState;
#[cfg(feature = "python-config")]
use litellm_config::load_model_list;
use litellm_core::router::{Deployment, LiteLLMParams, Router};
use litellm_ai_gateway::integrations::custom_logger::CustomLogger;
use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger;
#[cfg(feature = "python-config")]
use litellm_ai_gateway::python;
/// Bind to localhost by default so the gateway is not a public, unauthenticated
/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`).
@ -124,10 +124,10 @@ fn resolve_port() -> u16 {
fn build_router() -> Router {
#[cfg(feature = "python-config")]
if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") {
match python::config::load_router_from_config(&config_path) {
Ok(router) => {
match load_model_list(std::path::Path::new(&config_path)) {
Ok(deployments) => {
eprintln!("loaded model_list from {config_path} via python config reader");
return router;
return Router::new(deployments);
}
Err(err) => {
eprintln!("config load failed ({err}); falling back to env deployment");

90
litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs
View file

@ -12,6 +12,7 @@ use litellm_core::providers::azure_ai::ocr::transformation::{
AZURE_AI_OCR_CONFIG, AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG,
};
use litellm_core::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG;
use litellm_core::providers::reducto::ocr::transformation as reducto;
use litellm_core::providers::vertex_ai::ocr::transformation as vertex_ai;
use litellm_core::providers::vertex_ai::ocr::transformation::{
VERTEX_AI_DEEPSEEK_OCR_CONFIG, VERTEX_AI_OCR_CONFIG,
@ -39,6 +40,7 @@ pub(super) fn ocr_provider_config(
) -> Option<&'static dyn OcrProviderConfig> {
match provider {
"mistral" => Some(&MISTRAL_OCR_CONFIG),
"reducto" => reducto::config_for_model(model),
"azure_ai" if is_azure_document_intelligence_model(model) => {
Some(&AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG)
}
@ -334,6 +336,7 @@ fn operation_status(response_json: &Value) -> Result<&str, Error> {
}
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
pub(super) async fn poll_document_intelligence(
operation_url: &str,
original_url: &str,
@ -392,9 +395,11 @@ pub(super) async fn poll_document_intelligence(
#[cfg(test)]
mod tests {
use super::*;
use litellm_core::ocr::transformation::OcrResponseHandling;
use serde_json::json;
use super::*;
#[test]
fn blocks_private_and_metadata_ips() {
assert!(is_blocked_ip("127.0.0.1".parse().unwrap()));
@ -438,4 +443,87 @@ mod tests {
assert_eq!(transformed, document);
}
#[test]
fn truncate_error_body_passes_short_strings_through() {
let body = "Unauthorized";
assert_eq!(truncate_error_body(body), "Unauthorized");
}
#[test]
fn truncate_error_body_caps_long_payloads() {
let body = "x".repeat(306);
let truncated = truncate_error_body(&body);
assert!(truncated.ends_with("... (truncated)"));
let prefix_chars = truncated
.strip_suffix("... (truncated)")
.expect("truncated marker present")
.chars()
.count();
assert_eq!(prefix_chars, 256);
}
#[test]
fn truncate_error_body_does_not_split_multibyte_chars() {
let body = "é".repeat(266);
let truncated = truncate_error_body(&body);
assert!(truncated.is_char_boundary(truncated.len()));
}
#[test]
fn ocr_dispatch_supports_migrated_providers() {
assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
assert!(
ocr_provider_config("azure_ai", "pixtral-12b-2409")
.expect("azure ai config resolves")
.requires_data_uri_document()
);
assert_eq!(
ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
.expect("document intelligence config resolves")
.response_handling(),
OcrResponseHandling::AzureDocumentIntelligencePoll
);
assert!(
ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
.expect("vertex deepseek config resolves")
.supported_ocr_params()
.contains(&"temperature")
);
assert!(ocr_provider_config("openai", "gpt-4o").is_none());
}
#[test]
fn string_headers_accepts_string_values() {
let headers = json!({
"x-trace-id": "trace-1"
})
.as_object()
.unwrap()
.clone();
assert_eq!(
string_headers(Some(headers)).expect("string headers accepted"),
vec![("x-trace-id".to_string(), "trace-1".to_string())]
);
}
#[test]
fn string_headers_rejects_non_string_values() {
let headers = json!({
"x-retry-count": 3
})
.as_object()
.unwrap()
.clone();
let err = string_headers(Some(headers)).expect_err("non-string header rejected");
assert_eq!(
err,
Error::InvalidRequest(
"OCR extra_headers.x-retry-count must be a string, got number".to_string()
)
);
}
}

12
litellm-rust/crates/ai-gateway/src/ocr/handler.rs
View file

@ -50,7 +50,11 @@ pub(crate) async fn execute_ocr_provider_call(
.await?;
return Ok(request
.config
.transform_ocr_response(&request.model, response_json)?
.transform_ocr_response_with_params(
&request.model,
response_json,
&request.optional_params,
)?
.into_json());
}
@ -71,6 +75,10 @@ pub(crate) async fn execute_ocr_provider_call(
Ok(request
.config
.transform_ocr_response(&request.model, response_json)?
.transform_ocr_response_with_params(
&request.model,
response_json,
&request.optional_params,
)?
.into_json())
}

90
litellm-rust/crates/ai-gateway/src/ocr/hooks.rs
View file

@ -1,11 +1,15 @@
use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
use litellm_core::error::Error;
use litellm_core::providers::reducto::ocr::transformation::{
build_upload_request, extract_document_source, extract_upload_file_id,
};
use serde_json::{Map, Value, json};
use std::future::Future;
use std::pin::Pin;
use super::common_utils::{convert_document_url_to_data_uri, string_headers};
use super::common_utils::{convert_document_url_to_data_uri, string_headers, truncate_error_body};
use super::types::{PreparedOcrRequest, ProviderOcrRequest};
use crate::client::http_client;
use crate::integrations::custom_guardrail::{
CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest,
};
@ -89,22 +93,39 @@ impl OcrLifecycleHooks {
)?;
let model = request.model.clone();
let custom_llm_provider = request.custom_llm_provider.clone();
let document = if config.requires_data_uri_document() {
let is_reducto = custom_llm_provider == "reducto";
let document = if is_reducto {
let guarded_document = self
.run_during_call_guardrails(&model, &custom_llm_provider, &url, request.document)
.await?;
upload_reducto_document(
&guarded_document,
request.api_base.as_deref(),
request.timeout,
&upstream_headers,
)
.await?
} else if config.requires_data_uri_document() {
convert_document_url_to_data_uri(request.document).await?
} else {
request.document
};
let optional_params = request.optional_params;
let body = config
.transform_ocr_request(&request.model, document, request.optional_params)?
.transform_ocr_request(&request.model, document, optional_params.clone())?
.data;
let body = self
.run_during_call_guardrails(&model, &custom_llm_provider, &url, body)
.await?;
let body = if is_reducto {
body
} else {
self.run_during_call_guardrails(&model, &custom_llm_provider, &url, body)
.await?
};
Ok(ProviderOcrRequest {
model,
config,
url,
body,
optional_params,
upstream_headers,
timeout: request.timeout,
})
@ -165,6 +186,63 @@ impl OcrLifecycleHooks {
}
}
async fn upload_reducto_document(
document: &Value,
api_base: Option<&str>,
timeout: Option<std::time::Duration>,
upstream_headers: &[(String, String)],
) -> Result<Value, Error> {
let source = extract_document_source(document)?;
let Some(authorization) = upstream_headers
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case("authorization"))
.map(|(_, value)| value.as_str())
else {
return Err(Error::Auth(
"Reducto upload requires an Authorization header".to_string(),
));
};
let Some(upload) = build_upload_request(source, authorization, api_base) else {
return Ok(document.clone());
};
let part = reqwest::multipart::Part::bytes(upload.bytes)
.file_name(upload.file_name)
.mime_str(&upload.mime_type)
.map_err(|error| Error::InvalidRequest(error.to_string()))?;
let form = reqwest::multipart::Form::new().part("file", part);
let mut request_builder = http_client().post(upload.url).multipart(form);
for (name, value) in upstream_headers {
if !name.eq_ignore_ascii_case("content-type")
&& !name.eq_ignore_ascii_case("content-length")
{
request_builder = request_builder.header(name, value);
}
}
if let Some(timeout) = timeout {
request_builder = request_builder.timeout(timeout);
}
let response = request_builder
.send()
.await
.map_err(|error| Error::Network(error.to_string()))?;
let status = response.status();
let body = response
.text()
.await
.map_err(|error| Error::Network(error.to_string()))?;
if !status.is_success() {
return Err(Error::Http {
status: status.as_u16(),
body: truncate_error_body(&body),
});
}
let response_json: Value = serde_json::from_str(&body).map_err(|error| {
Error::InvalidResponse(format!("invalid Reducto upload response JSON: {error}"))
})?;
let file_id = extract_upload_file_id(&response_json)?;
Ok(json!({"type": "document_url", "document_url": file_id}))
}
impl CallLifecycleHooks<PreparedOcrRequest, PreparedOcrRequest, Value> for OcrLifecycleHooks {
type PreCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>;
type DuringCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>;

149
litellm-rust/crates/ai-gateway/src/ocr/mod.rs
View file

@ -24,4 +24,151 @@ pub async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
}
#[cfg(test)]
mod tests;
mod tests {
use serde_json::{Map, json};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use super::{OcrRequest, ocr};
use crate::integrations::types::RequestMetadata;
async fn read_http_request(socket: &mut TcpStream) -> String {
let mut request = Vec::new();
let mut buffer = [0_u8; 1024];
let header_end = loop {
let n = socket.read(&mut buffer).await.expect("reads request");
if n == 0 {
break request.len();
}
request.extend_from_slice(&buffer[..n]);
if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
break position + 4;
}
};
let headers = String::from_utf8_lossy(&request[..header_end]);
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().ok())
.flatten()
})
.unwrap_or(0);
while request.len().saturating_sub(header_end) < content_length {
let n = socket.read(&mut buffer).await.expect("reads body");
if n == 0 {
break;
}
request.extend_from_slice(&buffer[..n]);
}
String::from_utf8(request).expect("request is utf8")
}
fn base_ocr_request(model: &str) -> OcrRequest<'_> {
OcrRequest {
model,
document: json!({
"type": "document_url",
"document_url": "https://example.com/doc.pdf"
}),
api_key: Some("sk-test"),
api_base: None,
custom_llm_provider: None,
extra_headers: None,
optional_params: Map::new(),
timeout: None,
callbacks: Vec::new(),
guardrails: Vec::new(),
request_metadata: RequestMetadata::default(),
litellm_call_id: None,
}
}
#[tokio::test]
async fn reducto_file_upload_then_parse_maps_response() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("test listener binds");
let address = listener.local_addr().expect("listener has local address");
let server = tokio::spawn(async move {
let (mut upload_socket, _) = listener.accept().await.expect("accepts upload request");
let upload_request = read_http_request(&mut upload_socket).await;
let upload_body = r#"{"file_id":"reducto://uploaded.pdf"}"#;
let upload_response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
upload_body.len(),
upload_body
);
upload_socket
.write_all(upload_response.as_bytes())
.await
.expect("writes upload response");
let (mut parse_socket, _) = listener.accept().await.expect("accepts parse request");
let parse_request = read_http_request(&mut parse_socket).await;
let parse_body = r#"{"job_id":"job_123","usage":{"num_pages":3,"credits":3},"result":{"chunks":[{"content":"Page 1 block A","blocks":[{"content":"Page 1 block A","bbox":{"page":1},"kind":"text"}]},{"content":"Page 2 block A","blocks":[{"content":"Page 2 block A","bbox":{"page":2},"kind":"table"}]},{"content":"Page 1 block B","blocks":[{"content":"Page 1 block B","bbox":{"page":1},"kind":"text"}]},{"content":"Page 3 block A","blocks":[{"content":"Page 3 block A","bbox":{"page":3},"kind":"figure"}]}]}}"#;
let parse_response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
parse_body.len(),
parse_body
);
parse_socket
.write_all(parse_response.as_bytes())
.await
.expect("writes parse response");
(upload_request, parse_request)
});
let api_base = format!("http://{address}");
let mut request = base_ocr_request("reducto/parse-v3");
request.api_base = Some(&api_base);
request.api_key = None;
request.extra_headers = Some(Map::from_iter([
("Authorization".to_string(), json!("Bearer test-key")),
("x-trace-id".to_string(), json!("trace-1")),
]));
request.document = json!({
"type": "document_url",
"document_url": "data:application/pdf;base64,JVBERi0xLjQ="
});
request.optional_params = Map::from_iter([
(
"formatting".to_string(),
json!({"table_output_format": "html"}),
),
("retrieval".to_string(), json!({"chunk_mode": "section"})),
("settings".to_string(), json!({"ocr_system": "standard"})),
]);
let response = ocr(request).await.expect("Reducto OCR succeeds");
assert_eq!(response["pages"].as_array().map(Vec::len), Some(3));
assert_eq!(
response["pages"][0]["markdown"],
"Page 1 block A\n\nPage 1 block B"
);
assert_eq!(response["pages"][1]["markdown"], "Page 2 block A");
assert_eq!(response["pages"][2]["markdown"], "Page 3 block A");
assert_eq!(response["usage_info"]["pages_processed"], 3);
assert_eq!(response["usage_info"]["credits"], 3);
assert_eq!(response["provider_native_response"]["job_id"], "job_123");
let (upload_request, parse_request) = server.await.expect("server task completes");
assert!(
upload_request
.to_ascii_lowercase()
.contains("authorization: bearer test-key")
);
assert!(upload_request.contains("application/pdf"));
assert!(upload_request.contains("%PDF-1.4"));
assert!(upload_request.contains("x-trace-id: trace-1"));
assert!(
parse_request
.to_ascii_lowercase()
.contains("authorization: bearer test-key")
);
assert!(parse_request.contains(r#""input":"reducto://uploaded.pdf""#));
assert!(parse_request.contains(r#""table_output_format":"html""#));
assert!(parse_request.contains(r#""chunk_mode":"section""#));
assert!(parse_request.contains(r#""ocr_system":"standard""#));
}
}

96
litellm-rust/crates/ai-gateway/src/ocr/prepare.rs
View file

@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
use serde_json::{Map, Value};
use super::common_utils::ocr_provider_config;
use super::hooks::OcrLifecycleHooks;
@ -28,17 +29,33 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall {
let model = provider_info.model.to_string();
let custom_llm_provider = provider_info.custom_llm_provider.to_string();
let config = ocr_provider_config(&custom_llm_provider, &model)
.ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone()));
.ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone()))
.and_then(|config| {
validate_request_format(config, &request.optional_params, &custom_llm_provider)?;
Ok(config)
});
let optional_params = match &config {
Ok(config) => {
let supported = config.supported_ocr_params();
config.map_ocr_params(
let mut mapped = config.map_ocr_params(
&request
.optional_params
.into_iter()
.iter()
.filter(|(name, _)| supported.contains(&name.as_str()))
.map(|(name, value)| (name.clone(), value.clone()))
.collect(),
)
);
for name in [
"vertex_project",
"vertex_ai_project",
"vertex_location",
"vertex_ai_location",
] {
if let Some(value) = request.optional_params.get(name) {
mapped.insert(name.to_string(), value.clone());
}
}
mapped
}
Err(_) => request.optional_params,
};
@ -64,6 +81,26 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall {
}
}
fn validate_request_format(
config: &'static dyn litellm_core::ocr::transformation::OcrProviderConfig,
optional_params: &Map<String, Value>,
provider: &str,
) -> Result<(), litellm_core::Error> {
let Some(format) = optional_params.get("req_format") else {
return Ok(());
};
match format.as_str() {
Some("litellm") => Ok(()),
Some("native") if config.supported_ocr_params().contains(&"req_format") => Ok(()),
Some("native") => Err(litellm_core::Error::InvalidRequest(format!(
"`req_format=native` is not supported for provider {provider}"
))),
_ => Err(litellm_core::Error::InvalidRequest(format!(
"Invalid `req_format`: {format}. Expected `litellm` or `native`"
))),
}
}
fn new_ocr_call_id() -> String {
static COUNTER: AtomicU64 = AtomicU64::new(1);
let sequence = COUNTER.fetch_add(1, Ordering::Relaxed);
@ -73,3 +110,54 @@ fn new_ocr_call_id() -> String {
.unwrap_or(0);
format!("ocr-{timestamp}-{sequence}")
}
#[cfg(test)]
mod tests {
use litellm_core::error::Error;
use serde_json::{Map, json};
use super::{OcrRequest, prepare_ocr_call};
use crate::integrations::types::RequestMetadata;
fn base_ocr_request(model: &str) -> OcrRequest<'_> {
OcrRequest {
model,
document: json!({
"type": "document_url",
"document_url": "https://example.com/doc.pdf"
}),
api_key: Some("sk-test"),
api_base: None,
custom_llm_provider: None,
extra_headers: None,
optional_params: Map::new(),
timeout: None,
callbacks: Vec::new(),
guardrails: Vec::new(),
request_metadata: RequestMetadata::default(),
litellm_call_id: None,
}
}
fn request_with_format(format: &str) -> OcrRequest<'_> {
let mut request = base_ocr_request("mistral/mistral-ocr-latest");
request.optional_params = Map::from_iter([("req_format".to_string(), json!(format))]);
request
}
#[test]
fn native_format_rejected_for_provider_without_support_as_bad_request() {
let prepared = prepare_ocr_call(request_with_format("native"));
assert!(
matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("not supported for provider"))
);
}
#[test]
fn unknown_format_rejected_for_provider_without_support_as_bad_request() {
let prepared = prepare_ocr_call(request_with_format("raw"));
assert!(
matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("Invalid `req_format`"))
);
}
}

1
litellm-rust/crates/ai-gateway/src/ocr/types.rs
View file

@ -53,6 +53,7 @@ pub(crate) struct ProviderOcrRequest {
pub(crate) config: &'static dyn OcrProviderConfig,
pub(crate) url: String,
pub(crate) body: Value,
pub(crate) optional_params: Map<String, Value>,
pub(crate) upstream_headers: Vec<(String, String)>,
pub(crate) timeout: Option<Duration>,
}

27
litellm-rust/crates/ai-gateway/src/python/AGENTS.md
View file

@ -1,27 +0,0 @@
# ai-gateway/src/python — Python interop (load-time only)
Functions here embed the Python interpreter (pyo3) and take the GIL to call into
`litellm` (e.g. read the proxy `model_list`). Compiled only under the
`python-config` feature.
## Hard rule: non-hot-path functions only
Everything in this folder MUST run **at most once per process lifetime — at
startup / load time** (config read, warm-up). NEVER call into Python on the
request path:
- No GIL acquisition per request, per connection, or per realtime event.
- No Python call inside a route handler, the router's hot path, or any loop that
scales with traffic.
**Why:** the GIL serializes execution and would cap throughput; the realtime data
path must stay pure Rust. Every acquisition is recorded by `crate::gil` — poll
`GET /health/gil`, and `total_acquisitions` MUST stay flat under load.
## How to add one
Resolve whatever Python-derived data you need **once at boot** and hand the rest
of the gateway an owned, plain-Rust value (e.g. build a `Router` from the
resolved `model_list`). Record the acquisition via `crate::gil::record_acquisition()`
immediately before taking the GIL. If a function would need to run per request,
it does not belong here — move the work to Rust, or pre-resolve it at startup.

37
litellm-rust/crates/ai-gateway/src/python/config.rs
View file

@ -1,37 +0,0 @@
//! Build the router by calling the Python proxy config reader (load time only).
//!
//! Embeds the interpreter via pyo3 and calls
//! `litellm.proxy.read_model_list.read_model_list`, which reuses the proxy's
//! `os.environ/` + secret-manager resolution. The GIL is taken **once at boot**
//! (and recorded in [`crate::gil`]); the realtime hot path never touches Python.
//!
//! Compiled only under the `python-config` feature.
use litellm_core::error::Error;
use litellm_core::router::{Deployment, Router};
use pyo3::prelude::*;
use crate::gil;
/// Load the router's `model_list` from `config_path` via the Python reader.
pub fn load_router_from_config(config_path: &str) -> Result<Router, Error> {
gil::record_acquisition();
Python::attach(|py| {
let model_list = py
.import("litellm.proxy.read_model_list")
.and_then(|module| module.getattr("read_model_list"))
.and_then(|reader| reader.call1((config_path,)))
.map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?;
let model_list_json: String = py
.import("json")
.and_then(|json| json.getattr("dumps"))
.and_then(|dumps| dumps.call1((model_list,)))
.and_then(|encoded| encoded.extract())
.map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?;
let deployments: Vec<Deployment> = serde_json::from_str(&model_list_json)
.map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?;
Ok(Router::new(deployments))
})
}

4
litellm-rust/crates/ai-gateway/src/python/mod.rs
View file

@ -1,4 +0,0 @@
//! Python interop for the gateway. See `AGENTS.md`: **load-time / non-hot-path
//! only.** Compiled only under the `python-config` feature.
pub mod config;

2
litellm-rust/crates/ai-gateway/src/routes/AGENTS.md
View file

@ -13,7 +13,7 @@ private). This is the norm — don't split until it hurts.
pub fn router() -> Router<AppState> { Router::new().route(PATH, get(handle)) }
async fn handle(...) -> impl IntoResponse { ... }
```
`health.rs` and `gil.rs` are examples.
`health.rs` is the example.
## Split out `service` when there's real logic
When a route has business logic worth testing without axum, put it in a sibling

30
litellm-rust/crates/ai-gateway/src/routes/gil.rs
View file

@ -1,30 +0,0 @@
//! `GET /health/gil` — poll to confirm Python is only touched at load time.
//! Simple-route template: a `router()` plus its handler, in one file.
use axum::routing::get;
use axum::{Json, Router};
use serde::Serialize;
use crate::gil;
use crate::state::AppState;
/// This route's contribution to the app router.
pub fn router() -> Router<AppState> {
Router::new().route("/health/gil", get(status))
}
#[derive(Debug, Serialize)]
struct GilStatusResponse {
gil_acquired_last_30s: bool,
total_acquisitions: u64,
seconds_since_last: Option<u64>,
}
async fn status() -> Json<GilStatusResponse> {
let snapshot = gil::snapshot();
Json(GilStatusResponse {
gil_acquired_last_30s: snapshot.acquired_last_30s,
total_acquisitions: snapshot.total_acquisitions,
seconds_since_last: snapshot.seconds_since_last,
})
}

6
litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs
View file

@ -21,6 +21,12 @@ pub fn router() -> Router<AppState> {
Router::new().route(MESSAGES_ROUTE_PATH, post(handle))
}
#[tracing::instrument(
name = "messages_gateway_route",
target = "litellm::function_trace",
level = "trace",
skip_all
)]
async fn handle(
_auth: RequireMasterKey,
State(state): State<AppState>,

6
litellm-rust/crates/ai-gateway/src/routes/messages/service.rs
View file

@ -12,6 +12,12 @@ pub(crate) enum MessagesResponse {
Stream(reqwest::Response),
}
#[tracing::instrument(
name = "messages_gateway_service",
target = "litellm::function_trace",
level = "trace",
skip_all
)]
pub async fn run(
router: &Arc<Router>,
body: Value,

4
litellm-rust/crates/ai-gateway/src/routes/mod.rs
View file

@ -2,10 +2,9 @@
//!
//! **Template:** every route module exposes `pub fn router() -> Router<AppState>`
//! that mounts its own paths; [`app`] merges them. A trivial route is a single
//! file (`health.rs`, `gil.rs`); a non-trivial one is a folder (`realtime/`) with
//! file (`health.rs`); a non-trivial one is a folder (`realtime/`) with
//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md.
pub mod gil;
pub mod health;
pub mod messages;
pub mod realtime;
@ -19,7 +18,6 @@ use crate::state::AppState;
pub fn app(state: AppState) -> Router {
Router::new()
.merge(health::router())
.merge(gil::router())
.merge(messages::router())
.merge(realtime::router())
.merge(responses::router())

65
litellm-rust/crates/ai-gateway/src/trace_parity.rs Normal file
View file

@ -0,0 +1,65 @@
//! Harness-only in-process adapters. Never mounted as production routes.
use std::sync::Arc;
use axum::body::{Body, to_bytes};
use axum::http::header::{AUTHORIZATION, CONTENT_TYPE};
use axum::http::{Request, StatusCode};
use litellm_core::Error;
use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter};
use serde::Serialize;
use serde_json::Value;
use tower::ServiceExt;
use crate::io::realtime_pool::RealtimePool;
use crate::routes;
use crate::state::AppState;
#[derive(Debug, Serialize)]
pub struct GatewayResponse {
pub status: u16,
pub body: Value,
}
pub async fn messages_request(
model_alias: String,
provider_model: String,
api_base: String,
body: Value,
) -> Result<GatewayResponse, Error> {
let state = AppState {
router: Arc::new(ModelRouter::new(vec![Deployment {
model_name: model_alias,
litellm_params: LiteLLMParams {
model: provider_model,
api_key: Some("trace-provider-key".to_string()),
api_base: Some(api_base),
},
}])),
master_key: Some(Arc::from("trace-master-key")),
loggers: Arc::new(Vec::new()),
realtime_pool: RealtimePool::disabled(),
};
let request = Request::builder()
.method("POST")
.uri("/v1/messages")
.header(AUTHORIZATION, "Bearer trace-master-key")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(body.to_string()))
.map_err(|error| Error::InvalidRequest(error.to_string()))?;
let response = routes::app(state)
.oneshot(request)
.await
.map_err(|error| match error {})?;
let status: StatusCode = response.status();
let bytes = to_bytes(response.into_body(), usize::MAX)
.await
.map_err(|error| Error::InvalidResponse(error.to_string()))?;
let body = serde_json::from_slice(&bytes).map_err(|error| {
Error::InvalidResponse(format!("gateway returned invalid JSON: {error}"))
})?;
Ok(GatewayResponse {
status: status.as_u16(),
body,
})
}

196
litellm-rust/crates/ai-gateway/src/ocr/tests.rs → litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs
View file

@ -1,23 +1,19 @@
use std::sync::{Arc, Mutex};
use std::time::Duration;
use litellm_core::error::Error;
use litellm_core::http_utils::has_header;
use litellm_core::ocr::transformation::OcrResponseHandling;
use serde_json::{Map, Value, json};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use super::common_utils::{ocr_provider_config, string_headers, truncate_error_body};
use super::{OcrRequest, ocr};
use crate::integrations::custom_guardrail::{
use litellm_ai_gateway::integrations::custom_guardrail::{
CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook,
GuardrailFuture, GuardrailRequest,
};
use crate::integrations::custom_logger::{
use litellm_ai_gateway::integrations::custom_logger::{
CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails,
};
use crate::integrations::types::RequestMetadata;
use litellm_ai_gateway::integrations::types::RequestMetadata;
use litellm_ai_gateway::ocr::{OcrRequest, ocr};
use litellm_core::error::Error;
use serde_json::{Map, Value, json};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
async fn read_http_headers(socket: &mut TcpStream) -> String {
let mut request = Vec::new();
@ -136,6 +132,7 @@ struct RecordingOcrGuardrail {
hooks: Vec<GuardrailEventHook>,
events: Mutex<Vec<&'static str>>,
block_pre_call: bool,
block_during_call: bool,
}
impl RecordingOcrGuardrail {
@ -144,6 +141,7 @@ impl RecordingOcrGuardrail {
hooks,
events: Mutex::new(Vec::new()),
block_pre_call: false,
block_during_call: false,
}
}
@ -152,6 +150,16 @@ impl RecordingOcrGuardrail {
hooks: vec![GuardrailEventHook::PreCall],
events: Mutex::new(Vec::new()),
block_pre_call: true,
block_during_call: false,
}
}
fn blocking_during_call() -> Self {
Self {
hooks: vec![GuardrailEventHook::DuringCall],
events: Mutex::new(Vec::new()),
block_pre_call: false,
block_during_call: true,
}
}
@ -193,91 +201,95 @@ impl CustomGuardrail for RecordingOcrGuardrail {
) -> GuardrailFuture<'a> {
Box::pin(async move {
self.events.lock().unwrap().push("async_moderation_hook");
if self.block_during_call {
return Ok(GuardrailDecision::Block(GuardrailError::blocked(
"blocked before provider",
)));
}
request.data["body"]["guarded_during"] = json!(true);
Ok(GuardrailDecision::Mask(request))
})
}
}
#[test]
fn truncate_error_body_passes_short_strings_through() {
let body = "Unauthorized";
assert_eq!(truncate_error_body(body), "Unauthorized");
fn base_ocr_request(model: &str) -> OcrRequest<'_> {
OcrRequest {
model,
document: json!({
"type": "document_url",
"document_url": "https://example.com/doc.pdf"
}),
api_key: Some("sk-test"),
api_base: None,
custom_llm_provider: None,
extra_headers: None,
optional_params: Map::new(),
timeout: None,
callbacks: Vec::new(),
guardrails: Vec::new(),
request_metadata: RequestMetadata::default(),
litellm_call_id: None,
}
}
#[test]
fn truncate_error_body_caps_long_payloads() {
let body = "x".repeat(306);
let truncated = truncate_error_body(&body);
#[tokio::test]
async fn reducto_during_call_guardrail_blocks_before_upload() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("test listener binds");
let address = listener.local_addr().expect("listener has local address");
let api_base = format!("http://{address}");
let guardrail = Arc::new(RecordingOcrGuardrail::blocking_during_call());
let mut request = base_ocr_request("reducto/parse-v3");
request.api_base = Some(&api_base);
request.document = json!({
"type": "document_url",
"document_url": "data:application/pdf;base64,JVBERi0xLjQ="
});
request.guardrails = vec![guardrail.clone()];
assert!(truncated.ends_with("... (truncated)"));
let prefix_chars = truncated
.strip_suffix("... (truncated)")
.expect("truncated marker present")
.chars()
.count();
assert_eq!(prefix_chars, 256);
let error = ocr(request).await.expect_err("guardrail blocks upload");
assert!(matches!(error, Error::InvalidRequest(_)));
assert_eq!(guardrail.events(), vec!["async_moderation_hook"]);
let accepted = tokio::time::timeout(Duration::from_millis(100), listener.accept()).await;
assert!(accepted.is_err(), "upload socket should not be touched");
}
#[test]
fn truncate_error_body_does_not_split_multibyte_chars() {
let body = "é".repeat(266);
let truncated = truncate_error_body(&body);
assert!(truncated.is_char_boundary(truncated.len()));
}
#[tokio::test]
async fn reducto_upload_error_body_is_truncated() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("test listener binds");
let address = listener.local_addr().expect("listener has local address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accepts upload request");
let _request = read_http_request(&mut socket).await;
let body = "x".repeat(300);
let response = format!(
"HTTP/1.1 500 Internal Server Error\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
socket
.write_all(response.as_bytes())
.await
.expect("writes upload response");
});
let api_base = format!("http://{address}");
let mut request = base_ocr_request("reducto/parse-v3");
request.api_base = Some(&api_base);
request.document = json!({
"type": "document_url",
"document_url": "data:application/pdf;base64,JVBERi0xLjQ="
});
let error = ocr(request).await.expect_err("upload should fail");
#[test]
fn ocr_dispatch_supports_migrated_providers() {
assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
assert!(
ocr_provider_config("azure_ai", "pixtral-12b-2409")
.expect("azure ai config resolves")
.requires_data_uri_document()
matches!(error, Error::Http { status: 500, body } if body.chars().count() < 300 && body.ends_with("... (truncated)"))
);
assert_eq!(
ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
.expect("document intelligence config resolves")
.response_handling(),
OcrResponseHandling::AzureDocumentIntelligencePoll
);
assert!(
ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
.expect("vertex deepseek config resolves")
.supported_ocr_params()
.contains(&"temperature")
);
assert!(ocr_provider_config("openai", "gpt-4o").is_none());
}
#[test]
fn string_headers_accepts_string_values() {
let headers = json!({
"x-trace-id": "trace-1"
})
.as_object()
.unwrap()
.clone();
assert_eq!(
string_headers(Some(headers)).expect("string headers accepted"),
vec![("x-trace-id".to_string(), "trace-1".to_string())]
);
}
#[test]
fn auth_header_detection_is_case_insensitive() {
let headers = vec![
("x-trace-id".to_string(), "trace-1".to_string()),
("authorization".to_string(), "Bearer sk-test".to_string()),
];
assert!(has_header(&headers, "authorization"));
let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())];
assert!(has_header(&headers, "authorization"));
let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())];
assert!(!has_header(&headers, "authorization"));
server.await.expect("server task completes");
}
#[tokio::test]
@ -595,21 +607,3 @@ async fn document_intelligence_poll_uses_resolved_subscription_key() {
"{poll_request}"
);
}
#[test]
fn string_headers_rejects_non_string_values() {
let headers = json!({
"x-retry-count": 3
})
.as_object()
.unwrap()
.clone();
let err = string_headers(Some(headers)).expect_err("non-string header rejected");
assert_eq!(
err,
Error::InvalidRequest(
"OCR extra_headers.x-retry-count must be a string, got number".to_string()
)
);
}

16
litellm-rust/crates/config/Cargo.toml Normal file
View file

@ -0,0 +1,16 @@
[package]
name = "litellm-config"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-core.workspace = true
pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
serde_json.workspace = true
thiserror.workspace = true
[features]
default = []
python = ["dep:pyo3"]

11
litellm-rust/crates/config/src/error.rs Normal file
View file

@ -0,0 +1,11 @@
use thiserror::Error as ThisError;
#[derive(Debug, ThisError)]
pub enum Error {
#[error("read_model_list failed: {0}")]
PythonLoading(String),
#[error("serializing model_list failed: {0}")]
Serialization(String),
#[error("parsing model_list failed: {0}")]
ModelListParsing(#[source] serde_json::Error),
}

7
litellm-rust/crates/config/src/lib.rs Normal file
View file

@ -0,0 +1,7 @@
mod error;
#[cfg(feature = "python")]
mod python;
pub use error::Error;
#[cfg(feature = "python")]
pub use python::load_model_list;

76
litellm-rust/crates/config/src/python.rs Normal file
View file

@ -0,0 +1,76 @@
use std::path::Path;
use litellm_core::router::Deployment;
use pyo3::prelude::*;
use crate::Error;
pub fn load_model_list(config_path: &Path) -> Result<Vec<Deployment>, Error> {
Python::attach(|python| {
let model_list = python
.import("litellm.proxy.read_model_list")
.and_then(|module| module.getattr("read_model_list"))
.and_then(|reader| reader.call1((config_path.to_string_lossy().as_ref(),)))
.map_err(|error| Error::PythonLoading(error.to_string()))?;
let model_list_json = python
.import("json")
.and_then(|json| json.getattr("dumps"))
.and_then(|dumps| dumps.call1((model_list,)))
.and_then(|encoded| encoded.extract::<String>())
.map_err(|error| Error::Serialization(error.to_string()))?;
parse_model_list(&model_list_json)
})
}
fn parse_model_list(model_list_json: &str) -> Result<Vec<Deployment>, Error> {
serde_json::from_str(model_list_json).map_err(Error::ModelListParsing)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_resolved_model_list() {
let deployments = parse_model_list(
r#"[
{
"model_name": "realtime",
"litellm_params": {
"model": "openai/gpt-realtime",
"api_key": "resolved-secret",
"api_base": "https://api.example.test/v1"
}
},
{
"model_name": "without-optional-values",
"litellm_params": {"model": "openai/gpt-4.1"}
}
]"#,
)
.expect("resolved model list should parse");
assert_eq!(deployments.len(), 2);
assert_eq!(deployments[0].model_name, "realtime");
assert_eq!(
deployments[0].litellm_params.api_key.as_deref(),
Some("resolved-secret")
);
assert_eq!(
deployments[0].litellm_params.api_base.as_deref(),
Some("https://api.example.test/v1")
);
assert_eq!(deployments[1].litellm_params.api_key, None);
assert_eq!(deployments[1].litellm_params.api_base, None);
}
#[test]
fn malformed_model_list_returns_parsing_error() {
let error = parse_model_list(r#"[{"model_name":"missing-params"}]"#)
.expect_err("missing litellm_params should fail");
assert!(matches!(error, Error::ModelListParsing(_)));
}
}

5
litellm-rust/crates/core/Cargo.toml
View file

@ -6,12 +6,14 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
rand.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tracing.workspace = true
tracing-subscriber = { workspace = true, optional = true }
sha2.workspace = true
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true }
@ -30,6 +32,9 @@ bedrock-auth = [
"dep:aws-types",
"dep:aws-smithy-runtime-api",
]
observability = ["dep:tracing-subscriber"]
[dev-dependencies]
rstest.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
tracing-subscriber.workspace = true

2
litellm-rust/crates/core/src/constants.rs
View file

@ -41,3 +41,5 @@ pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion";
/// `litellm/litellm_core_utils/prompt_templates/factory.py`.
pub const EMPTY_TEXT_PLACEHOLDER: &str =
"[System: Empty message content sanitised to satisfy protocol]";
pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";

17
litellm-rust/crates/core/src/http_utils.rs
View file

@ -101,6 +101,23 @@ mod tests {
assert!(!has_header(&headers, "authorization"));
}
#[test]
fn auth_header_detection_is_case_insensitive() {
let headers = vec![
("x-trace-id".to_string(), "trace-1".to_string()),
("authorization".to_string(), "Bearer sk-test".to_string()),
];
assert!(has_header(&headers, "authorization"));
let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())];
assert!(has_header(&headers, "authorization"));
let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())];
assert!(!has_header(&headers, "authorization"));
}
#[test]
fn bearer_detection_requires_a_non_empty_token() {
assert!(has_bearer_auth(&[(

2
litellm-rust/crates/core/src/lib.rs
View file

@ -6,6 +6,8 @@ pub mod constants;
pub mod error;
pub mod http_utils;
pub mod messages;
#[cfg(any(feature = "observability", test))]
pub mod observability;
pub mod ocr;
pub mod providers;
pub mod realtime;

215
litellm-rust/crates/core/src/observability/function_trace.rs Normal file
View file

@ -0,0 +1,215 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use serde::Serialize;
use tracing::span::{Attributes, Id};
use tracing::{Dispatch, Subscriber};
use tracing_subscriber::layer::Context;
use tracing_subscriber::prelude::*;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::{Layer, Registry};
use super::function_trace_filter;
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct FunctionTraceEvent {
pub id: usize,
pub parent_id: Option<usize>,
pub function: &'static str,
pub module_path: Option<&'static str>,
pub file: Option<&'static str>,
pub line: Option<u32>,
}
#[derive(Clone, Default)]
pub struct FunctionTrace {
events: Arc<Mutex<Vec<FunctionTraceEvent>>>,
span_events: Arc<Mutex<HashMap<Id, usize>>>,
}
impl FunctionTrace {
pub fn dispatcher(&self) -> Dispatch {
Dispatch::new(
Registry::default().with(
FunctionTraceLayer {
trace: self.clone(),
}
.with_filter(function_trace_filter()),
),
)
}
pub fn events(&self) -> Vec<FunctionTraceEvent> {
self.events
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone()
}
}
struct FunctionTraceLayer {
trace: FunctionTrace,
}
impl<S> Layer<S> for FunctionTraceLayer
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) {
let parent_id = context.span(id).and_then(|span| {
let span_events = self
.trace
.span_events
.lock()
.unwrap_or_else(|error| error.into_inner());
span.scope()
.skip(1)
.find_map(|ancestor| span_events.get(&ancestor.id()).copied())
});
let mut events = self
.trace
.events
.lock()
.unwrap_or_else(|error| error.into_inner());
let event_id = events.len();
events.push(FunctionTraceEvent {
id: event_id,
parent_id,
function: attributes.metadata().name(),
module_path: attributes.metadata().module_path(),
file: attributes.metadata().file(),
line: attributes.metadata().line(),
});
self.trace
.span_events
.lock()
.unwrap_or_else(|error| error.into_inner())
.insert(id.clone(), event_id);
}
}
#[cfg(test)]
mod tests {
use crate::constants::FUNCTION_TRACE_TARGET;
use super::*;
fn event(
id: usize,
parent_id: Option<usize>,
function: &'static str,
) -> (usize, Option<usize>, &'static str) {
(id, parent_id, function)
}
fn structural_events(
events: &[FunctionTraceEvent],
) -> Vec<(usize, Option<usize>, &'static str)> {
events
.iter()
.map(|event| (event.id, event.parent_id, event.function))
.collect()
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn outer() {
tokio::task::yield_now().await;
inner().await;
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn inner() {
tokio::task::yield_now().await;
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn concurrent_parent() {
tokio::join!(inner(), inner());
}
#[tokio::test]
async fn concurrent_futures_keep_separate_traces_across_yields() {
use tracing::instrument::WithSubscriber;
let first = FunctionTrace::default();
let second = FunctionTrace::default();
let outside = FunctionTrace::default();
async {
tokio::join!(
outer().with_subscriber(first.dispatcher()),
inner().with_subscriber(second.dispatcher()),
);
inner().await;
}
.with_subscriber(outside.dispatcher())
.await;
assert_eq!(
structural_events(&first.events()),
vec![event(0, None, "outer"), event(1, Some(0), "inner")],
);
assert_eq!(
structural_events(&second.events()),
vec![event(0, None, "inner")],
);
assert_eq!(
structural_events(&outside.events()),
vec![event(0, None, "inner")],
);
}
#[tokio::test]
async fn concurrent_siblings_keep_the_same_parent() {
use tracing::instrument::WithSubscriber;
let trace = FunctionTrace::default();
concurrent_parent()
.with_subscriber(trace.dispatcher())
.await;
assert_eq!(
structural_events(&trace.events()),
vec![
event(0, None, "concurrent_parent"),
event(1, Some(0), "inner"),
event(2, Some(0), "inner"),
]
);
}
#[test]
fn records_matching_spans_in_creation_order() {
let trace = FunctionTrace::default();
let dispatch = trace.dispatcher();
tracing::dispatcher::with_default(&dispatch, || {
let _ignored = tracing::trace_span!(target: "other", "ignored");
let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level");
let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
});
assert_eq!(
structural_events(&trace.events()),
vec![event(0, None, "same_name"), event(1, None, "same_name")]
);
}
#[test]
fn records_matching_span_nesting_depth() {
let trace = FunctionTrace::default();
let dispatch = trace.dispatcher();
tracing::dispatcher::with_default(&dispatch, || {
let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer");
let _outer_guard = outer.enter();
let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner");
});
assert_eq!(
structural_events(&trace.events()),
vec![event(0, None, "outer"), event(1, Some(0), "inner")]
);
}
}

59
litellm-rust/crates/core/src/observability/mod.rs Normal file
View file

@ -0,0 +1,59 @@
use tracing::span::Id;
use tracing::{Level, Metadata, Subscriber};
use tracing_subscriber::filter::{FilterFn, LevelFilter, filter_fn};
use tracing_subscriber::layer::Context;
use tracing_subscriber::registry::LookupSpan;
use crate::constants::FUNCTION_TRACE_TARGET;
pub mod function_trace;
pub use function_trace::{FunctionTrace, FunctionTraceEvent};
pub fn function_trace_filter() -> FilterFn<impl Fn(&Metadata<'_>) -> bool> {
filter_fn(|metadata| {
metadata.is_span()
&& metadata.target() == FUNCTION_TRACE_TARGET
&& *metadata.level() == Level::TRACE
})
.with_max_level_hint(LevelFilter::TRACE)
}
pub fn span_depth<S>(context: &Context<'_, S>, id: &Id) -> usize
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
context
.span(id)
.map(|span| span.scope().skip(1).count())
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use tracing::instrument::WithSubscriber;
use super::*;
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn instrumented_with_literal_target() {}
#[tokio::test]
async fn literal_instrument_target_matches_filter_constant() {
assert_eq!(FUNCTION_TRACE_TARGET, "litellm::function_trace");
let trace = FunctionTrace::default();
instrumented_with_literal_target()
.with_subscriber(trace.dispatcher())
.await;
let events = trace.events();
assert_eq!(events.len(), 1);
assert_eq!(events[0].id, 0);
assert_eq!(events[0].parent_id, None);
assert_eq!(events[0].function, "instrumented_with_literal_target");
assert_eq!(events[0].module_path, Some(module_path!()));
assert_eq!(events[0].file, Some(file!()));
assert!(events[0].line.is_some());
}
}

9
litellm-rust/crates/core/src/ocr/transformation.rs
View file

@ -51,6 +51,15 @@ pub trait OcrProviderConfig: Sync {
response_json: Value,
) -> Result<OcrResponseData, Error>;
fn transform_ocr_response_with_params(
&self,
model: &str,
response_json: Value,
_optional_params: &Map<String, Value>,
) -> Result<OcrResponseData, Error> {
self.transform_ocr_response(model, response_json)
}
fn complete_url(
&self,
api_base: Option<&str>,

15
litellm-rust/crates/core/src/ocr/types.rs
View file

@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_json::{Map, Value};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct OcrRequestData {
@ -14,16 +14,25 @@ pub struct OcrResponseData {
pub document_annotation: Option<Value>,
pub usage_info: Option<Value>,
pub object: String,
pub extra_fields: Map<String, Value>,
pub provider_native_response: Option<Value>,
}
impl OcrResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
let mut response = serde_json::json!({
"pages": self.pages,
"model": self.model,
"document_annotation": self.document_annotation,
"usage_info": self.usage_info,
"object": self.object,
})
});
if let Value::Object(object) = &mut response {
object.extend(self.extra_fields);
if let Some(native_response) = self.provider_native_response {
object.insert("provider_native_response".to_string(), native_response);
}
}
response
}
}

887
litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs
View file

File diff suppressed because it is too large Load diff

2
litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs
View file

@ -134,6 +134,8 @@ impl OcrProviderConfig for MistralOcrConfig {
document_annotation,
usage_info,
object: "ocr".to_string(),
extra_fields: Map::new(),
provider_native_response: None,
})
}

1
litellm-rust/crates/core/src/providers/mod.rs
View file

@ -4,4 +4,5 @@ pub mod azure_ai;
pub mod bedrock;
pub mod mistral;
pub mod openai;
pub mod reducto;
pub mod vertex_ai;

1
litellm-rust/crates/core/src/providers/reducto/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod ocr;

4
litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs Normal file
View file

@ -0,0 +1,4 @@
pub mod transformation;
#[cfg(test)]
mod tests;

202
litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs Normal file
View file

@ -0,0 +1,202 @@
use rstest::{fixture, rstest};
use serde_json::{Value, json};
use super::transformation::*;
use crate::ocr::transformation::OcrProviderConfig;
#[fixture]
fn parse_response() -> Value {
json!({
"job_id": "job_123",
"usage": {"num_pages": 3, "credits": 3},
"result": {
"chunks": [
{
"content": "Page 1 block A",
"blocks": [{
"content": "Page 1 block A",
"bbox": {"page": 1},
"kind": "text",
}],
},
{
"content": "Page 2 block A",
"blocks": [{
"content": "Page 2 block A",
"bbox": {"page": 2},
"kind": "table",
}],
},
{
"content": "Page 1 block B",
"blocks": [{
"content": "Page 1 block B",
"bbox": {"page": 1},
"kind": "text",
}],
},
{
"content": "Page 3 block A",
"blocks": [{
"content": "Page 3 block A",
"bbox": {"page": 3},
"kind": "figure",
}],
},
],
},
})
}
#[rstest]
fn test_parse_v3_file_upload_and_response_mapping(parse_response: Value) {
let source = classify_document_source("data:application/pdf;base64,JVBERi0xLjQ=")
.expect("PDF data URI should be valid");
let upload = build_upload_request(
source,
"Bearer test-key",
Some("https://platform.reducto.ai"),
)
.expect("data URI should require upload");
assert_eq!(upload.url, "https://platform.reducto.ai/upload");
assert_eq!(upload.authorization, "Bearer test-key");
assert_eq!(upload.file_name, "document");
assert_eq!(upload.mime_type, "application/pdf");
assert_eq!(upload.bytes, b"%PDF-1.4");
let optional_params = json!({
"formatting": {"table_output_format": "html"},
"retrieval": {"chunk_mode": "section"},
"settings": {"ocr_system": "standard"},
})
.as_object()
.expect("params should be an object")
.clone();
let request = build_parse_v3_request("reducto://uploaded.pdf", optional_params);
assert_eq!(
request.data,
json!({
"input": "reducto://uploaded.pdf",
"formatting": {"table_output_format": "html"},
"retrieval": {"chunk_mode": "section"},
"settings": {"ocr_system": "standard"},
})
);
let transformed = transform_reducto_response("parse-v3", parse_response.clone())
.expect("response should transform");
assert_eq!(
transformed.usage_info,
Some(json!({"pages_processed": 3, "credits": 3}))
);
assert_eq!(transformed.pages.len(), 3);
assert_eq!(
transformed.pages[0],
json!({
"index": 0,
"markdown": "Page 1 block A\n\nPage 1 block B",
"blocks": [
{"content": "Page 1 block A", "bbox": {"page": 1}, "kind": "text"},
{"content": "Page 1 block B", "bbox": {"page": 1}, "kind": "text"},
],
})
);
assert_eq!(transformed.pages[1]["markdown"], "Page 2 block A");
assert_eq!(transformed.pages[2]["markdown"], "Page 3 block A");
assert_eq!(transformed.provider_native_response, Some(parse_response));
}
#[rstest]
fn test_parse_v3_reducto_id_passthrough_skips_upload(parse_response: Value) {
let document = json!({
"type": "document_url",
"document_url": "reducto://already-uploaded.pdf",
});
let source = extract_document_source(&document).expect("Reducto ID should be valid");
assert!(build_upload_request(source.clone(), "Bearer test-key", None).is_none());
assert_eq!(
source,
ReductoDocumentSource::FileId("reducto://already-uploaded.pdf".to_string())
);
let request = REDUCTO_PARSE_V3_CONFIG
.transform_ocr_request(
"parse-v3",
document,
json!({"retrieval": {"chunk_mode": "section"}})
.as_object()
.expect("params should be object")
.clone(),
)
.expect("direct ID should transform");
assert_eq!(request.data["input"], "reducto://already-uploaded.pdf");
assert_eq!(request.data["retrieval"]["chunk_mode"], "section");
let response = REDUCTO_PARSE_V3_CONFIG
.transform_ocr_response("parse-v3", parse_response)
.expect("response should transform");
assert!(
response.pages[0]["markdown"]
.as_str()
.expect("markdown should be string")
.starts_with("Page 1 block A")
);
}
#[rstest]
fn test_parse_legacy_wraps_enhance_under_options() {
let request = build_parse_legacy_request(
"reducto://legacy.pdf",
json!({"enhance": {"agentic": [{"type": "table"}]}})
.as_object()
.expect("params should be object"),
);
assert_eq!(
request.data,
json!({
"document_url": "reducto://legacy.pdf",
"options": {"enhance": {"agentic": [{"type": "table"}]}},
})
);
}
#[rstest]
fn test_parse_v3_image_data_uri_upload_uses_image_mime() {
let source = classify_document_source("data:image/png;base64,iVBORw0KGgo=")
.expect("PNG data URI should be valid");
let upload = build_upload_request(
source,
"Bearer programmatic-key",
Some("https://custom.reducto.test/"),
)
.expect("data URI should require upload");
assert_eq!(upload.url, "https://custom.reducto.test/upload");
assert_eq!(upload.authorization, "Bearer programmatic-key");
assert_eq!(upload.mime_type, "image/png");
assert_eq!(upload.bytes, b"\x89PNG\r\n\x1a\n");
}
#[rstest]
#[case::http("http://example.com/document.pdf")]
#[case::https("https://example.com/document.pdf")]
fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) {
let error = classify_document_source(source).expect_err("plain URL should be rejected");
assert!(error.to_string().contains("upload the file first"));
}
#[rstest]
fn test_parse_v3_uses_programmatic_api_key_over_env() {
let key = resolve_api_key(Some("passed-key"), &|_| Some("env-reducto-key".to_string()))
.expect("explicit key should resolve");
assert_eq!(key, "passed-key");
let headers = REDUCTO_PARSE_V3_CONFIG
.validate_environment(Vec::new(), Some("passed-key"), &|_| {
Some("env-reducto-key".to_string())
})
.expect("headers should validate");
assert_eq!(
headers,
vec![("Authorization".to_string(), "Bearer passed-key".to_string())]
);
}

407
litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs Normal file
View file

@ -0,0 +1,407 @@
use std::collections::BTreeMap;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use serde_json::{Map, Value, json};
use crate::error::{Error, json_type_name};
use crate::ocr::transformation::OcrProviderConfig;
use crate::ocr::types::{OcrRequestData, OcrResponseData};
pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai";
pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY";
pub const REDUCTO_ID_PREFIX: &str = "reducto://";
const PARSE_V3_SUPPORTED_OCR_PARAMS: &[&str] = &["formatting", "retrieval", "settings"];
const PARSE_LEGACY_SUPPORTED_OCR_PARAMS: &[&str] = &["enhance"];
const MISSING_KEY_MESSAGE: &str = "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()";
const DATA_URI_UPLOAD_REQUIRED: &str =
"Reducto data URI upload must complete before OCR request transformation";
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ReductoDocumentSource {
FileId(String),
Upload { bytes: Vec<u8>, mime_type: String },
}
#[derive(Clone, PartialEq, Eq)]
pub struct ReductoUploadRequest {
pub url: String,
pub authorization: String,
pub file_name: &'static str,
pub bytes: Vec<u8>,
pub mime_type: String,
}
pub struct ReductoParseV3Config;
pub struct ReductoParseLegacyConfig;
pub const REDUCTO_PARSE_V3_CONFIG: ReductoParseV3Config = ReductoParseV3Config;
pub const REDUCTO_PARSE_LEGACY_CONFIG: ReductoParseLegacyConfig = ReductoParseLegacyConfig;
pub fn config_for_model(model: &str) -> Option<&'static dyn OcrProviderConfig> {
match model {
"parse-v3" => Some(&REDUCTO_PARSE_V3_CONFIG),
"parse-legacy" => Some(&REDUCTO_PARSE_LEGACY_CONFIG),
_ => None,
}
}
pub fn normalize_api_base(api_base: Option<&str>) -> String {
api_base
.map(str::trim)
.filter(|base| !base.is_empty())
.unwrap_or(REDUCTO_API_BASE)
.trim_end_matches('/')
.to_string()
}
pub fn parse_url(api_base: Option<&str>) -> String {
format!("{}/parse", normalize_api_base(api_base))
}
pub fn upload_url(api_base: Option<&str>) -> String {
format!("{}/upload", normalize_api_base(api_base))
}
pub fn resolve_api_key(
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
api_key
.map(str::trim)
.filter(|key| !key.is_empty())
.map(str::to_string)
.or_else(|| {
env_lookup(REDUCTO_API_KEY_ENV)
.map(|key| key.trim().to_string())
.filter(|key| !key.is_empty())
})
.ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string()))
}
pub fn extract_document_source(document: &Value) -> Result<ReductoDocumentSource, Error> {
let document = document.as_object().ok_or_else(|| Error::InvalidType {
expected: "object",
actual: json_type_name(document),
})?;
let source = document
.get("document_url")
.and_then(Value::as_str)
.filter(|source| !source.is_empty())
.or_else(|| document.get("image_url").and_then(Value::as_str))
.ok_or_else(|| {
Error::InvalidRequest(
"Reducto expected OCR preprocessing to produce document_url or image_url"
.to_string(),
)
})?;
classify_document_source(source)
}
pub fn classify_document_source(source: &str) -> Result<ReductoDocumentSource, Error> {
if source.starts_with(REDUCTO_ID_PREFIX) {
return Ok(ReductoDocumentSource::FileId(source.to_string()));
}
if source.starts_with("http://") || source.starts_with("https://") {
return Err(Error::InvalidRequest(
"Reducto requires type='file' (auto-uploaded) or a reducto:// id. Plain http(s) URLs are not supported; upload the file first."
.to_string(),
));
}
if !source.starts_with("data:") {
return Err(Error::InvalidRequest(
"Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing."
.to_string(),
));
}
let (header, encoded) = source
.split_once(',')
.ok_or_else(|| Error::InvalidRequest("Invalid Reducto data URI provided.".to_string()))?;
if !header.split(';').any(|part| part == "base64") {
return Err(Error::InvalidRequest(
"Reducto only supports base64-encoded data URIs.".to_string(),
));
}
let mime_type = header
.strip_prefix("data:")
.and_then(|header| header.split(';').next())
.filter(|mime| !mime.is_empty())
.unwrap_or("application/octet-stream")
.to_string();
let bytes = BASE64_STANDARD.decode(encoded).map_err(|_| {
Error::InvalidRequest("Invalid Reducto base64 payload provided.".to_string())
})?;
Ok(ReductoDocumentSource::Upload { bytes, mime_type })
}
pub fn build_upload_request(
source: ReductoDocumentSource,
authorization: &str,
api_base: Option<&str>,
) -> Option<ReductoUploadRequest> {
let ReductoDocumentSource::Upload { bytes, mime_type } = source else {
return None;
};
Some(ReductoUploadRequest {
url: upload_url(api_base),
authorization: authorization.to_string(),
file_name: "document",
bytes,
mime_type,
})
}
pub fn extract_upload_file_id(response_json: &Value) -> Result<&str, Error> {
response_json
.as_object()
.and_then(|response| response.get("file_id"))
.and_then(Value::as_str)
.filter(|file_id| !file_id.is_empty())
.ok_or_else(|| {
Error::InvalidResponse(format!(
"Reducto /upload returned 200 without a file_id; got payload={response_json}"
))
})
}
pub fn build_parse_v3_request(
file_id: &str,
optional_params: Map<String, Value>,
) -> OcrRequestData {
let data = std::iter::once(("input".to_string(), Value::String(file_id.to_string())))
.chain(optional_params)
.collect();
OcrRequestData {
data: Value::Object(data),
files: None,
}
}
pub fn build_parse_legacy_request(
file_id: &str,
optional_params: &Map<String, Value>,
) -> OcrRequestData {
let options = optional_params
.get("enhance")
.filter(|enhance| !enhance.is_null())
.map(|enhance| json!({"options": {"enhance": enhance}}));
let data = match options {
Some(Value::Object(options)) => std::iter::once((
"document_url".to_string(),
Value::String(file_id.to_string()),
))
.chain(options)
.collect(),
_ => Map::from_iter([(
"document_url".to_string(),
Value::String(file_id.to_string()),
)]),
};
OcrRequestData {
data: Value::Object(data),
files: None,
}
}
fn source_file_id(document: &Value) -> Result<String, Error> {
match extract_document_source(document)? {
ReductoDocumentSource::FileId(file_id) => Ok(file_id),
ReductoDocumentSource::Upload { .. } => Err(Error::Unsupported(DATA_URI_UPLOAD_REQUIRED)),
}
}
fn page_number(block: &Map<String, Value>) -> Option<i64> {
let page = block.get("bbox")?.as_object()?.get("page")?;
page.as_i64()
.or_else(|| page.as_u64().and_then(|page| i64::try_from(page).ok()))
.or_else(|| page.as_str().and_then(|page| page.parse().ok()))
}
fn chunks(result: &Map<String, Value>) -> &[Value] {
result
.get("chunks")
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default()
}
fn build_pages(result: &Map<String, Value>) -> Vec<Value> {
let blocks_by_page = chunks(result)
.iter()
.filter_map(Value::as_object)
.filter_map(|chunk| chunk.get("blocks").and_then(Value::as_array))
.flatten()
.filter_map(|block| block.as_object().map(|object| (block, object)))
.filter_map(|(block, object)| page_number(object).map(|page| (page, block.clone())))
.fold(
BTreeMap::<i64, Vec<Value>>::new(),
|mut pages, (page, block)| {
pages.entry(page).or_default().push(block);
pages
},
);
if blocks_by_page.is_empty() {
let markdown = chunks(result)
.iter()
.filter_map(Value::as_object)
.filter_map(|chunk| chunk.get("content").and_then(Value::as_str))
.filter(|content| !content.is_empty())
.collect::<Vec<_>>()
.join("\n\n");
return if markdown.is_empty() {
Vec::new()
} else {
vec![json!({"index": 0, "markdown": markdown})]
};
}
blocks_by_page
.into_iter()
.map(|(page, blocks)| {
let markdown = blocks
.iter()
.filter_map(Value::as_object)
.filter_map(|block| block.get("content").and_then(Value::as_str))
.filter(|content| !content.is_empty())
.collect::<Vec<_>>()
.join("\n\n");
json!({
"index": page.saturating_sub(1).max(0),
"markdown": markdown,
"blocks": blocks,
})
})
.collect()
}
pub fn transform_reducto_response(
model: &str,
response_json: Value,
) -> Result<OcrResponseData, Error> {
let response = response_json
.as_object()
.ok_or_else(|| Error::InvalidType {
expected: "object",
actual: json_type_name(&response_json),
})?;
let empty_result = Map::new();
let result = match response.get("result") {
Some(Value::Object(result)) => result,
Some(Value::Null) => &empty_result,
Some(_) => {
return Err(Error::InvalidResponse(
"Reducto result must be an object".to_string(),
));
}
None => response,
};
let usage = response
.get("usage")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
let usage_info = Some(json!({
"pages_processed": usage.get("num_pages").cloned().unwrap_or(Value::Null),
"credits": usage.get("credits").cloned().unwrap_or(Value::Null),
}));
Ok(OcrResponseData {
pages: build_pages(result),
model: model.to_string(),
document_annotation: None,
usage_info,
object: "ocr".to_string(),
extra_fields: Map::new(),
provider_native_response: Some(response_json),
})
}
impl OcrProviderConfig for ReductoParseV3Config {
fn supported_ocr_params(&self) -> &'static [&'static str] {
PARSE_V3_SUPPORTED_OCR_PARAMS
}
fn transform_ocr_request(
&self,
_model: &str,
document: Value,
optional_params: Map<String, Value>,
) -> Result<OcrRequestData, Error> {
let file_id = source_file_id(&document)?;
Ok(build_parse_v3_request(&file_id, optional_params))
}
fn transform_ocr_response(
&self,
model: &str,
response_json: Value,
) -> Result<OcrResponseData, Error> {
transform_reducto_response(model, response_json)
}
fn complete_url(
&self,
api_base: Option<&str>,
_model: &str,
_optional_params: &Map<String, Value>,
_env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
Ok(parse_url(api_base))
}
fn resolve_api_key(
&self,
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
resolve_api_key(api_key, env_lookup)
}
}
impl OcrProviderConfig for ReductoParseLegacyConfig {
fn supported_ocr_params(&self) -> &'static [&'static str] {
PARSE_LEGACY_SUPPORTED_OCR_PARAMS
}
fn transform_ocr_request(
&self,
_model: &str,
document: Value,
optional_params: Map<String, Value>,
) -> Result<OcrRequestData, Error> {
let file_id = source_file_id(&document)?;
Ok(build_parse_legacy_request(&file_id, &optional_params))
}
fn transform_ocr_response(
&self,
model: &str,
response_json: Value,
) -> Result<OcrResponseData, Error> {
transform_reducto_response(model, response_json)
}
fn complete_url(
&self,
api_base: Option<&str>,
_model: &str,
_optional_params: &Map<String, Value>,
_env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
Ok(parse_url(api_base))
}
fn resolve_api_key(
&self,
api_key: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
resolve_api_key(api_key, env_lookup)
}
}

34
litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs
View file

@ -212,6 +212,7 @@ impl OcrProviderConfig for VertexAiOcrConfig {
MISTRAL_OCR_CONFIG.supported_ocr_params()
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn transform_ocr_request(
&self,
model: &str,
@ -229,6 +230,7 @@ impl OcrProviderConfig for VertexAiOcrConfig {
MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json)
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn complete_url(
&self,
api_base: Option<&str>,
@ -253,10 +255,21 @@ impl OcrProviderConfig for VertexAiOcrConfig {
}
impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn supported_ocr_params(&self) -> &'static [&'static str] {
DEEPSEEK_SUPPORTED_OCR_PARAMS
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn map_ocr_params(&self, non_default_params: &Map<String, Value>) -> Map<String, Value> {
non_default_params
.iter()
.filter(|(name, _)| DEEPSEEK_SUPPORTED_OCR_PARAMS.contains(&name.as_str()))
.map(|(name, value)| (name.clone(), value.clone()))
.collect()
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn transform_ocr_request(
&self,
model: &str,
@ -283,6 +296,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
})
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn transform_ocr_response(
&self,
model: &str,
@ -335,9 +349,12 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
document_annotation: object.get("document_annotation").cloned(),
usage_info,
object: "ocr".to_string(),
extra_fields: Map::new(),
provider_native_response: None,
})
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn complete_url(
&self,
api_base: Option<&str>,
@ -360,6 +377,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
#[test]
fn vertex_mistral_url_uses_project_location_and_model() {
@ -411,6 +429,22 @@ mod tests {
);
}
#[rstest]
#[case::bare_model("deepseek-ocr-maas")]
#[case::namespaced_model("deepseek-ai/deepseek-ocr-maas")]
fn vertex_deepseek_request_uses_single_provider_namespace(#[case] model: &str) {
let body = VERTEX_AI_DEEPSEEK_OCR_CONFIG
.transform_ocr_request(
model,
json!({"type": "image_url", "image_url": "data:image/png;base64,AA=="}),
Map::new(),
)
.expect("request transforms")
.data;
assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas");
}
#[test]
fn vertex_deepseek_response_wraps_markdown_content() {
let response = VERTEX_AI_DEEPSEEK_OCR_CONFIG

14
litellm-rust/crates/core/tests/workspace_crate_allowlist.rs
View file

@ -1,6 +1,7 @@
//! Enforcement: the litellm-rust workspace has exactly four crates.
//! Enforcement: the litellm-rust workspace has exactly five crates.
//!
//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host),
//! `core` (the Rust SDK), `config` (the config-loading boundary),
//! `ai-gateway` (the HTTP/WebSocket host),
//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the
//! PyO3 cdylib). Adding or removing a crate must be a
//! deliberate act: this test fails until the allowlist here is updated, forcing
@ -19,13 +20,20 @@ use std::path::{Path, PathBuf};
/// workspace legitimately gains or loses a crate.
const EXPECTED_MEMBERS: &[&str] = &[
"crates/core",
"crates/config",
"crates/ai-gateway",
"crates/python-interop",
"crates/python-bridge",
];
/// The crate subdirectory names that must exist under `crates/`.
const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"];
const EXPECTED_CRATE_DIRS: &[&str] = &[
"core",
"config",
"ai-gateway",
"python-interop",
"python-bridge",
];
const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact).";

9
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -14,11 +14,15 @@ default = ["abi3"]
abi3 = ["pyo3/abi3-py310"]
extension-module = ["pyo3/extension-module"]
panic-test = []
trace-parity = [
"dep:tracing",
"litellm-core/observability",
"litellm-ai-gateway/trace-parity",
]
[dependencies]
futures-util.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
tracing = { workspace = true, optional = true }
litellm-core = { workspace = true, features = ["bedrock-auth"] }
litellm-ai-gateway = { workspace = true, default-features = false }
litellm-python-interop.workspace = true
@ -31,6 +35,7 @@ tokio.workspace = true
[dev-dependencies]
criterion = "0.8.2"
tokio-tungstenite.workspace = true
tracing.workspace = true
[[bench]]
name = "serialization"

1
litellm-rust/crates/python-bridge/src/constants.rs
View file

@ -1 +0,0 @@
pub(crate) const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";

208
litellm-rust/crates/python-bridge/src/function_trace.rs
View file

@ -1,216 +1,22 @@
use std::future::Future;
use std::sync::{Arc, Mutex};
use litellm_core::observability::{FunctionTrace, FunctionTraceEvent};
use serde::Serialize;
use tracing::instrument::WithSubscriber;
use tracing::span::{Attributes, Id};
use tracing::{Dispatch, Level, Subscriber};
use tracing_subscriber::filter::{LevelFilter, filter_fn};
use tracing_subscriber::layer::Context;
use tracing_subscriber::prelude::*;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::{Layer, Registry};
use crate::constants::FUNCTION_TRACE_TARGET;
#[derive(Serialize)]
#[serde(untagged)]
pub(crate) enum TraceResponse<T> {
Plain(T),
Traced {
response: T,
trace: Vec<FunctionTraceEvent>,
},
pub(crate) struct TracedResponse<T> {
response: T,
trace: Vec<FunctionTraceEvent>,
}
pub(crate) async fn trace_call<T, E>(
pub(crate) async fn capture<T, E>(
future: impl Future<Output = Result<T, E>>,
enabled: bool,
) -> Result<TraceResponse<T>, E> {
if !enabled {
return future.await.map(TraceResponse::Plain);
}
) -> Result<TracedResponse<T>, E> {
let trace = FunctionTrace::default();
let response = future.with_subscriber(trace.dispatcher()).await?;
Ok(TraceResponse::Traced {
Ok(TracedResponse {
response,
trace: trace.events(),
})
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct FunctionTraceEvent {
pub function: &'static str,
pub depth: usize,
}
#[derive(Clone, Default)]
pub struct FunctionTrace {
events: Arc<Mutex<Vec<FunctionTraceEvent>>>,
}
impl FunctionTrace {
pub fn dispatcher(&self) -> Dispatch {
let filter = filter_fn(|metadata| {
metadata.is_span()
&& metadata.target() == FUNCTION_TRACE_TARGET
&& *metadata.level() == Level::TRACE
})
.with_max_level_hint(LevelFilter::TRACE);
Dispatch::new(
Registry::default().with(
FunctionTraceLayer {
trace: self.clone(),
}
.with_filter(filter),
),
)
}
pub fn events(&self) -> Vec<FunctionTraceEvent> {
self.events
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone()
}
}
struct FunctionTraceLayer {
trace: FunctionTrace,
}
impl<S> Layer<S> for FunctionTraceLayer
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) {
let depth = context
.span(id)
.map(|span| span.scope().skip(1).count())
.unwrap_or_default();
self.trace
.events
.lock()
.unwrap_or_else(|error| error.into_inner())
.push(FunctionTraceEvent {
function: attributes.metadata().name(),
depth,
});
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn outer() {
tokio::task::yield_now().await;
inner().await;
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn inner() {
tokio::task::yield_now().await;
}
#[tokio::test]
async fn concurrent_futures_keep_separate_traces_across_yields() {
use tracing::instrument::WithSubscriber;
let first = FunctionTrace::default();
let second = FunctionTrace::default();
let outside = FunctionTrace::default();
async {
tokio::join!(
outer().with_subscriber(first.dispatcher()),
inner().with_subscriber(second.dispatcher()),
);
inner().await;
}
.with_subscriber(outside.dispatcher())
.await;
assert_eq!(
first.events(),
vec![
FunctionTraceEvent {
function: "outer",
depth: 0
},
FunctionTraceEvent {
function: "inner",
depth: 1
},
],
);
assert_eq!(
second.events(),
vec![FunctionTraceEvent {
function: "inner",
depth: 0
}],
);
assert_eq!(
outside.events(),
vec![FunctionTraceEvent {
function: "inner",
depth: 0
}],
);
}
#[test]
fn records_matching_spans_in_creation_order() {
let trace = FunctionTrace::default();
let dispatch = trace.dispatcher();
tracing::dispatcher::with_default(&dispatch, || {
let _ignored = tracing::trace_span!(target: "other", "ignored");
let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level");
let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name");
});
assert_eq!(
trace.events(),
vec![
FunctionTraceEvent {
function: "same_name",
depth: 0,
},
FunctionTraceEvent {
function: "same_name",
depth: 0,
},
]
);
}
#[test]
fn records_matching_span_nesting_depth() {
let trace = FunctionTrace::default();
let dispatch = trace.dispatcher();
tracing::dispatcher::with_default(&dispatch, || {
let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer");
let _outer_guard = outer.enter();
let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner");
});
assert_eq!(
trace.events(),
vec![
FunctionTraceEvent {
function: "outer",
depth: 0,
},
FunctionTraceEvent {
function: "inner",
depth: 1,
},
]
);
}
}

40
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -1,8 +1,8 @@
mod constants;
mod diagnostics;
mod errors;
mod execution;
pub mod function_trace;
#[cfg(feature = "trace-parity")]
mod function_trace;
mod marshal;
mod routes;
@ -115,9 +115,43 @@ mod tests {
.extract::<Vec<String>>()
.expect("module names should be strings")
.into_iter()
.filter(|name| !name.starts_with("__"))
.filter(|name| !name.starts_with('_'))
.collect();
assert_eq!(public_names, expected);
#[cfg(not(feature = "trace-parity"))]
assert!(!module.hasattr("_trace").expect("module lookup should work"));
#[cfg(feature = "trace-parity")]
{
let trace = module
.getattr("_trace")
.expect("trace build should expose its diagnostic namespace");
let trace_names: Vec<String> = trace
.cast::<PyModule>()
.expect("trace namespace should be a module")
.dict()
.keys()
.extract::<Vec<String>>()
.expect("trace names should be strings")
.into_iter()
.filter(|name| !name.starts_with("__"))
.collect();
assert_eq!(
trace_names,
[
"ocr",
"aocr",
"transcription",
"atranscription",
"messages",
"amessages",
"chat_completions",
"achat_completions",
"gateway_messages",
]
);
}
});
}

6
litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs
View file

@ -54,16 +54,16 @@ bridge_route! {
required = {
model: String,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
audio: Value,
audio: serde_json::Value,
},
optional = {
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
extra_headers: Option<Value>,
extra_headers: Option<serde_json::Value>,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
optional_params: Option<Value>,
optional_params: Option<serde_json::Value>,
timeout_seconds: Option<f64>,
},
prepare = prepare_transcription,

6
litellm-rust/crates/python-bridge/src/routes/chat_completions.rs
View file

@ -73,16 +73,16 @@ bridge_route! {
required = {
model: String,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
messages: Value,
messages: serde_json::Value,
},
optional = {
#[pyo3(from_py_with = litellm_python_interop::from_py)]
optional_params: Option<Value>,
optional_params: Option<serde_json::Value>,
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
extra_headers: Option<Value>,
extra_headers: Option<serde_json::Value>,
timeout_seconds: Option<f64>,
},
prepare = prepare_chat_completions,

149
litellm-rust/crates/python-bridge/src/routes/definition.rs
View file

@ -20,43 +20,33 @@ macro_rules! bridge_route {
}
#[pyfunction]
#[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))]
#[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
#[allow(clippy::too_many_arguments)]
fn $sync_name(
py: pyo3::Python<'_>,
$($(#[$required_attr])* $required_name: $required_type,)*
$($(#[$optional_attr])* $optional_name: $optional_type,)*
trace: bool,
) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
let future = $prepare($inputs {
$($required_name,)*
$($optional_name),*
})?;
$crate::execution::run_sync(
py,
$crate::function_trace::trace_call(future, trace),
$map_error,
)
$crate::execution::run_sync(py, future, $map_error)
}
#[pyfunction]
#[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))]
#[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
#[allow(clippy::too_many_arguments)]
fn $async_name(
py: pyo3::Python<'_>,
$($(#[$required_attr])* $required_name: $required_type,)*
$($(#[$optional_attr])* $optional_name: $optional_type,)*
trace: bool,
) -> pyo3::PyResult<pyo3::Bound<'_, pyo3::PyAny>> {
let future = $prepare($inputs {
$($required_name,)*
$($optional_name),*
})?;
$crate::execution::run_async(
py,
$crate::function_trace::trace_call(future, trace),
$map_error,
)
$crate::execution::run_async(py, future, $map_error)
}
pub(super) fn register(
@ -67,6 +57,71 @@ macro_rules! bridge_route {
$crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($async_name, module)?)?;
Ok(())
}
#[cfg(feature = "trace-parity")]
mod trace {
use pyo3::prelude::*;
use super::{$inputs, $map_error, $prepare};
#[pyfunction]
#[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
#[allow(clippy::too_many_arguments)]
fn $sync_name(
py: pyo3::Python<'_>,
$($(#[$required_attr])* $required_name: $required_type,)*
$($(#[$optional_attr])* $optional_name: $optional_type,)*
) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
let future = $prepare($inputs {
$($required_name,)*
$($optional_name),*
})?;
$crate::execution::run_sync(
py,
$crate::function_trace::capture(future),
$map_error,
)
}
#[pyfunction]
#[pyo3(signature = ($($required_name),*, $($optional_name=None),*))]
#[allow(clippy::too_many_arguments)]
fn $async_name(
py: pyo3::Python<'_>,
$($(#[$required_attr])* $required_name: $required_type,)*
$($(#[$optional_attr])* $optional_name: $optional_type,)*
) -> pyo3::PyResult<pyo3::Bound<'_, pyo3::PyAny>> {
let future = $prepare($inputs {
$($required_name,)*
$($optional_name),*
})?;
$crate::execution::run_async(
py,
$crate::function_trace::capture(future),
$map_error,
)
}
pub(super) fn register(
module: &pyo3::Bound<'_, pyo3::types::PyModule>,
) -> pyo3::PyResult<()> {
$crate::routes::definition::add_function(
module,
pyo3::wrap_pyfunction!($sync_name, module)?,
)?;
$crate::routes::definition::add_function(
module,
pyo3::wrap_pyfunction!($async_name, module)?,
)?;
Ok(())
}
}
#[cfg(feature = "trace-parity")]
pub(super) fn register_trace(
module: &pyo3::Bound<'_, pyo3::types::PyModule>,
) -> pyo3::PyResult<()> {
trace::register(module)
}
};
}
@ -130,20 +185,26 @@ mod tests {
) -> PyResult<impl Future<Output = Result<String, Error>> + Send + 'static> {
FUTURE_DROPPED.store(false, Ordering::SeqCst);
let drop_guard = (inputs.value == "pending").then_some(DropGuard);
Ok(async move {
let _drop_guard = drop_guard;
tokio::task::yield_now().await;
match inputs.value.as_str() {
"error" => Err(Error::InvalidRequest("synthetic error".to_string())),
"map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())),
"panic" => panic!("synthetic panic"),
"pending" => {
pending::<()>().await;
unreachable!()
}
_ => Ok(inputs.value),
Ok(execute_echo(inputs, drop_guard))
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn execute_echo(
inputs: EchoInputs,
drop_guard: Option<DropGuard>,
) -> Result<String, Error> {
let _drop_guard = drop_guard;
tokio::task::yield_now().await;
match inputs.value.as_str() {
"error" => Err(Error::InvalidRequest("synthetic error".to_string())),
"map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())),
"panic" => panic!("synthetic panic"),
"pending" => {
pending::<()>().await;
unreachable!()
}
})
_ => Ok(inputs.value),
}
}
fn map_error(error: Error) -> PyErr {
@ -164,22 +225,22 @@ mod tests {
(
"ocr",
"aocr",
"(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)",
"(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
),
(
"transcription",
"atranscription",
"(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)",
"(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)",
),
(
"messages",
"amessages",
"(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)",
"(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
),
(
"chat_completions",
"achat_completions",
"(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)",
"(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None)",
),
];
@ -411,6 +472,32 @@ asyncio.run(exercise())
});
}
#[cfg(feature = "trace-parity")]
#[test]
fn diagnostic_route_returns_the_response_and_filtered_trace() {
Python::initialize();
Python::attach(|py| {
let module = PyModule::new(py, "synthetic").expect("module should be created");
synthetic::register_trace(&module).expect("trace routes should register");
let locals = PyDict::new(py);
locals
.set_item("routes", &module)
.expect("module should enter Python locals");
let code = CString::new(
r#"
result = routes.echo("traced")
assert result == {
"response": "traced",
"trace": [{"function": "execute_echo", "depth": 0}],
}
"#,
)
.expect("Python source should not contain null bytes");
py.run(&code, Some(&locals), Some(&locals))
.expect("diagnostic route should return its response and trace");
});
}
#[test]
fn route_registration_rejects_duplicate_python_names() {
Python::initialize();

29
litellm-rust/crates/python-bridge/src/routes/gateway_messages.rs Normal file
View file

@ -0,0 +1,29 @@
use pyo3::prelude::*;
use serde_json::Value;
use crate::errors::core_error_to_pyerr;
#[pyfunction]
fn gateway_messages<'py>(
py: Python<'py>,
model_alias: String,
provider_model: String,
api_base: String,
#[pyo3(from_py_with = litellm_python_interop::from_py)] body: Value,
) -> PyResult<Bound<'py, PyAny>> {
let future = litellm_ai_gateway::trace_parity::messages_request(
model_alias,
provider_model,
api_base,
body,
);
crate::execution::run_async(
py,
crate::function_trace::capture(future),
core_error_to_pyerr,
)
}
pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> {
super::definition::add_function(module, wrap_pyfunction!(gateway_messages, module)?)
}

4
litellm-rust/crates/python-bridge/src/routes/messages.rs
View file

@ -50,14 +50,14 @@ bridge_route! {
required = {
model: String,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
body: Value,
body: serde_json::Value,
},
optional = {
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
extra_headers: Option<Value>,
extra_headers: Option<serde_json::Value>,
timeout_seconds: Option<f64>,
},
prepare = prepare_messages,

16
litellm-rust/crates/python-bridge/src/routes/mod.rs
View file

@ -3,6 +3,9 @@ use pyo3::prelude::*;
#[macro_use]
mod definition;
#[cfg(feature = "trace-parity")]
mod gateway_messages;
mod audio_transcription;
mod chat_completions;
mod messages;
@ -12,5 +15,16 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
ocr::register(module)?;
audio_transcription::register(module)?;
messages::register(module)?;
chat_completions::register(module)
chat_completions::register(module)?;
#[cfg(feature = "trace-parity")]
{
let trace = PyModule::new(module.py(), "_trace")?;
ocr::register_trace(&trace)?;
audio_transcription::register_trace(&trace)?;
messages::register_trace(&trace)?;
chat_completions::register_trace(&trace)?;
gateway_messages::register_trace(&trace)?;
module.add_submodule(&trace)?;
}
Ok(())
}

6
litellm-rust/crates/python-bridge/src/routes/ocr.rs
View file

@ -56,16 +56,16 @@ bridge_route! {
required = {
model: String,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
document: Value,
document: serde_json::Value,
},
optional = {
api_key: Option<String>,
api_base: Option<String>,
custom_llm_provider: Option<String>,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
extra_headers: Option<Value>,
extra_headers: Option<serde_json::Value>,
#[pyo3(from_py_with = litellm_python_interop::from_py)]
optional_params: Option<Value>,
optional_params: Option<serde_json::Value>,
timeout_seconds: Option<f64>,
},
prepare = prepare_ocr,

423
litellm-rust/crates/python-bridge/src/routes/runtime.rs
View file

@ -1,423 +0,0 @@
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::time::Duration;
use futures_util::FutureExt;
use litellm_core::error::Error;
use litellm_python_interop::{Pythonized, panic_to_pyerr, release_gil};
use pyo3::exceptions::PyRuntimeError;
use pyo3::prelude::*;
use serde::Serialize;
use tokio::runtime::{Handle, Runtime};
use tokio::time::{self, MissedTickBehavior};
pub(super) fn run_sync<T, F>(
py: Python<'_>,
future: F,
map_error: fn(Error) -> PyErr,
) -> PyResult<Py<PyAny>>
where
T: Serialize + Send + 'static,
F: Future<Output = Result<T, Error>> + Send + 'static,
{
run_sync_on(
py,
pyo3_async_runtimes::tokio::get_runtime(),
future,
map_error,
)
}
fn run_sync_on<T, F>(
py: Python<'_>,
runtime: &Runtime,
future: F,
map_error: fn(Error) -> PyErr,
) -> PyResult<Py<PyAny>>
where
T: Serialize + Send + 'static,
F: Future<Output = Result<T, Error>> + Send + 'static,
{
if Handle::try_current().is_ok() {
return Err(PyRuntimeError::new_err(
"synchronous native routes cannot run from a Tokio context; use the async route",
));
}
let result = release_gil(py, move || runtime.block_on(wait_for_sync_result(future)))?;
let result = map_core_result(result, map_error)?;
Pythonized(result).into_pyobject(py).map(Bound::unbind)
}
pub(super) fn run_async<T, F>(
py: Python<'_>,
future: F,
map_error: fn(Error) -> PyErr,
) -> PyResult<Bound<'_, PyAny>>
where
T: Serialize + Send + 'static,
F: Future<Output = Result<T, Error>> + Send + 'static,
{
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let result = catch_route_panic(future).await?;
let result = map_core_result(result, map_error)?;
Ok(Pythonized(result))
})
}
fn map_core_result<T>(result: Result<T, Error>, map_error: fn(Error) -> PyErr) -> PyResult<T> {
match result {
Ok(value) => Ok(value),
Err(error) => Err(
std::panic::catch_unwind(AssertUnwindSafe(|| map_error(error)))
.map_err(panic_to_pyerr)?,
),
}
}
async fn catch_route_panic<T, F>(future: F) -> PyResult<Result<T, Error>>
where
F: Future<Output = Result<T, Error>>,
{
AssertUnwindSafe(future)
.catch_unwind()
.await
.map_err(panic_to_pyerr)
}
async fn wait_for_sync_result<T, F>(future: F) -> PyResult<Result<T, Error>>
where
F: Future<Output = Result<T, Error>>,
{
let future = catch_route_panic(future);
tokio::pin!(future);
let signal_interval = Duration::from_millis(50);
let mut signal_checks =
time::interval_at(time::Instant::now() + signal_interval, signal_interval);
signal_checks.set_missed_tick_behavior(MissedTickBehavior::Delay);
loop {
tokio::select! {
result = &mut future => return result,
_ = signal_checks.tick() => Python::attach(|py| py.check_signals())?,
}
}
}
#[cfg(test)]
mod tests {
use std::ffi::CString;
use std::future::poll_fn;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, mpsc};
use std::task::Poll;
use std::thread;
use std::time::Instant;
use pyo3::panic::PanicException;
use pyo3::types::{PyDict, PyModule};
use serde::Serializer;
use tokio::runtime::Builder;
use super::*;
fn runtime_error(error: Error) -> PyErr {
PyRuntimeError::new_err(error.to_string())
}
fn panicking_error_mapper(_error: Error) -> PyErr {
panic!("error mapper panicked")
}
struct PanickingOutput;
static ASYNC_PROBE_COMPLETED: AtomicUsize = AtomicUsize::new(0);
impl Serialize for PanickingOutput {
fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
panic!("serializer panicked")
}
}
#[pyfunction]
fn async_serialization_panic(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
run_async(py, async { Ok(PanickingOutput) }, runtime_error)
}
#[pyfunction]
fn async_runtime_probe(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
run_async(
py,
async {
ASYNC_PROBE_COMPLETED.fetch_add(1, Ordering::SeqCst);
Ok(true)
},
runtime_error,
)
}
#[pyfunction]
fn runtime_worker_count() -> usize {
pyo3_async_runtimes::tokio::get_runtime()
.metrics()
.num_workers()
}
#[pyfunction]
fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool {
let completion_deadline = Instant::now() + Duration::from_secs(2);
while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions {
if Instant::now() >= completion_deadline {
return false;
}
thread::sleep(Duration::from_millis(1));
}
let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1);
pyo3_async_runtimes::tokio::get_runtime().spawn(async move {
let _ = heartbeat_tx.send(());
});
heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok()
}
fn extract_bool(py: Python<'_>, result: PyResult<Py<PyAny>>) -> bool {
result
.expect("route should complete")
.bind(py)
.extract()
.expect("result should convert")
}
#[test]
fn sync_runner_polls_future_on_the_caller_thread() {
Python::initialize();
Python::attach(|py| {
let caller_thread = std::thread::current().id();
let result = run_sync(
py,
async move { Ok(std::thread::current().id() == caller_thread) },
runtime_error,
);
assert!(extract_bool(py, result));
});
}
#[test]
fn sync_runner_releases_gil_while_waiting() {
Python::initialize();
Python::attach(|py| {
let result = run_sync(
py,
async {
let gil_acquired = tokio::time::timeout(
Duration::from_secs(2),
tokio::task::spawn_blocking(|| Python::attach(|_| true)),
)
.await;
Ok(matches!(gil_acquired, Ok(Ok(true))))
},
runtime_error,
);
assert!(extract_bool(py, result));
});
}
#[test]
fn sync_runner_rejects_calls_from_a_tokio_context() {
Python::initialize();
let runtime = Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime should build");
let error = runtime.block_on(async {
Python::attach(|py| {
run_sync::<bool, _>(py, async { Ok(true) }, runtime_error)
.expect_err("sync route should reject a nested Tokio runtime")
})
});
assert_eq!(
error.to_string(),
"RuntimeError: synchronous native routes cannot run from a Tokio context; use the async route"
);
}
#[test]
fn sync_runner_can_drive_a_current_thread_runtime() {
Python::initialize();
let runtime = Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime should build");
Python::attach(|py| {
let result = run_sync_on(
py,
&runtime,
async {
tokio::task::yield_now().await;
Ok(true)
},
runtime_error,
);
assert!(extract_bool(py, result));
});
}
#[test]
fn sync_runner_maps_a_panicked_future() {
Python::initialize();
Python::attach(|py| {
let error = run_sync::<bool, _>(
py,
poll_fn(|_| -> Poll<Result<bool, Error>> { panic!("route future panicked") }),
runtime_error,
)
.expect_err("panicked route should become a Python exception");
assert!(error.is_instance_of::<PanicException>(py));
assert_eq!(error.to_string(), "PanicException: route future panicked");
});
}
#[test]
fn sync_runner_maps_a_panicked_error_mapper() {
Python::initialize();
Python::attach(|py| {
let error = run_sync::<bool, _>(
py,
async { Err(Error::InvalidRequest("invalid".to_string())) },
panicking_error_mapper,
)
.expect_err("panicked mapper should become a Python exception");
assert!(error.is_instance_of::<PanicException>(py));
assert_eq!(error.to_string(), "PanicException: error mapper panicked");
});
}
#[test]
fn sync_runner_surfaces_serializer_panics() {
Python::initialize();
Python::attach(|py| {
let error = run_sync(py, async { Ok(PanickingOutput) }, runtime_error)
.expect_err("serializer panic should become a Python exception");
assert!(error.is_instance_of::<PanicException>(py));
assert_eq!(error.to_string(), "PanicException: serializer panicked");
});
}
#[test]
fn sync_runner_supports_concurrent_callers_on_the_shared_runtime() {
Python::initialize();
let barrier = Arc::new(tokio::sync::Barrier::new(2));
let callers: Vec<_> = (0..2)
.map(|_| {
let barrier = Arc::clone(&barrier);
thread::spawn(move || {
Python::attach(|py| {
extract_bool(
py,
run_sync(
py,
async move {
Ok(tokio::time::timeout(Duration::from_secs(2), barrier.wait())
.await
.is_ok())
},
runtime_error,
),
)
})
})
})
.collect();
let results: Vec<_> = callers
.into_iter()
.map(|caller| caller.join().expect("caller should not panic"))
.collect();
assert_eq!(results, vec![true, true]);
}
#[test]
fn async_runner_surfaces_serializer_panics() {
Python::initialize();
Python::attach(|py| {
let module = PyModule::new(py, "runtime").expect("module should be created");
module
.add_function(
wrap_pyfunction!(async_serialization_panic, &module)
.expect("function should wrap"),
)
.expect("function should register");
let locals = PyDict::new(py);
locals
.set_item("runtime", &module)
.expect("module should enter Python locals");
let code = CString::new(
r#"
import asyncio
async def exercise():
try:
await runtime.async_serialization_panic()
except BaseException as error:
assert type(error).__name__ == "PanicException"
assert str(error) == "serializer panicked"
else:
raise AssertionError("serializer panic was not raised")
asyncio.run(exercise())
"#,
)
.expect("Python source should not contain null bytes");
py.run(&code, Some(&locals), Some(&locals))
.expect("serializer panic should reach the Python awaiter");
});
}
#[test]
fn async_result_delivery_does_not_stall_tokio_workers() {
Python::initialize();
ASYNC_PROBE_COMPLETED.store(0, Ordering::SeqCst);
Python::attach(|py| {
let module = PyModule::new(py, "runtime").expect("module should be created");
for function in [
wrap_pyfunction!(async_runtime_probe, &module).expect("function should wrap"),
wrap_pyfunction!(runtime_worker_count, &module).expect("function should wrap"),
wrap_pyfunction!(runtime_is_responsive, &module).expect("function should wrap"),
] {
module
.add_function(function)
.expect("function should register");
}
let locals = PyDict::new(py);
locals
.set_item("runtime", &module)
.expect("module should enter Python locals");
let code = CString::new(
r#"
import asyncio
async def exercise():
worker_count = runtime.runtime_worker_count()
awaitables = [runtime.async_runtime_probe() for _ in range(worker_count)]
assert runtime.runtime_is_responsive(worker_count)
assert await asyncio.gather(*awaitables) == [True] * worker_count
asyncio.run(exercise())
"#,
)
.expect("Python source should not contain null bytes");
py.run(&code, Some(&locals), Some(&locals))
.expect("result delivery should leave Tokio workers responsive");
});
}
}

2
litellm/__init__.py
View file

@ -1421,7 +1421,7 @@ from .skills.main import (
)
from .containers.main import *
from .ocr.main import *
from .rust_bridge import use_litellm_rust
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .search.main import *

4
litellm/anthropic_interface/exceptions/exceptions.py
View file

@ -1,8 +1,9 @@
"""Anthropic error format type definitions."""
from collections.abc import Mapping
from typing import Literal
from typing_extensions import Required, TypedDict
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
# Known Anthropic error types
# Source: https://docs.anthropic.com/en/api/errors
@ -23,6 +24,7 @@ class AnthropicErrorDetail(TypedDict):
type: AnthropicErrorType
message: str
provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
class AnthropicErrorResponse(TypedDict, total=False):

30
litellm/caching/caching.py
View file

@ -100,6 +100,7 @@ class Cache:
qdrant_semantic_cache_vector_size: int | None = None,
semantic_cache_embedding_max_input_tokens: int | None = None,
semantic_cache_embedding_timeout: float | None = None,
semantic_cache_scope: str = SemanticCacheScope.KEY.value,
# GCP IAM authentication parameters
gcp_service_account: str | None = None,
gcp_ssl_ca_certs: str | None = None,
@ -127,6 +128,7 @@ class Cache:
similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
# Disk Cache Args
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@ -274,6 +276,7 @@ class Cache:
self.redis_flush_size = redis_flush_size
self.ttl = ttl
self.mode: CacheMode = mode or CacheMode.default_on
self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
self.ttl = default_in_memory_ttl
@ -301,6 +304,7 @@ class Cache:
"user_api_key_team_id",
"user_api_key_org_id",
)
_SEMANTIC_CACHE_END_USER_SCOPE_FIELD: Final = "user_api_key_end_user_id"
def _is_semantic_cache(self) -> bool:
return self.type in (
@ -309,19 +313,21 @@ class Cache:
LiteLLMCacheType.VALKEY_SEMANTIC,
)
def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
metadata: Final[dict] = kwargs.get("metadata") or {}
litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
metadata_in_litellm_params: Final[dict] = litellm_params.get("metadata") or {}
def _semantic_cache_scope_fields(self) -> tuple[str, ...]:
if self.semantic_cache_scope == SemanticCacheScope.END_USER:
return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
scope = ""
for field in self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS:
value = metadata.get(field)
if value is None:
value = metadata_in_litellm_params.get(field)
if value is not None:
scope += f"{field}: {value}"
return scope
def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
metadata_sources: Final[tuple[dict, ...]] = tuple(
source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
)
scope_values: Final = (
(field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
for field in self._semantic_cache_scope_fields()
)
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
def get_cache_key(self, **kwargs) -> str:
"""

93
litellm/caching/redis_cluster_node_isolation.py
View file

@ -19,13 +19,13 @@ connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-cov
retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
topology changed.
redis-py 8.x fixed this upstream with gentler machinery than this override's
``node.disconnect()`` (which also kills connections other coroutines are mid-operation
on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
killed connection): it marks in-use connections for reconnect only after their current
operation completes, disconnects only the idle pooled ones, and defers reinitialization
to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
recovery API, the factory returns the base ``RedisCluster`` unmodified.
redis-py 8.x recovers connections per-connection, so the copied override is not used. Upstream
still flips the shared ``_initialize`` flag on any node's timeout, funneling every concurrent
caller through the reinit lock and, if ``CLUSTER SLOTS`` lands on the slow node, into a full
teardown. For those versions the factory returns a thin wrapper around upstream's
``_execute_command`` that clears the flag again after an isolated timeout (a ConnectionError,
a third consecutive timeout on the same node, or a concurrent request from any other command
or ``aclose()`` still reinits).
"""
import asyncio
@ -44,6 +44,8 @@ class _ClusterNodeAttrs(Protocol):
mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
own logic fully typed without a banned ``typing.cast``."""
name: str
async def execute_command(
self,
*args: object,
@ -78,18 +80,20 @@ class _ClusterAttrs(Protocol):
#: this override can't see (Python won't error -- it'll just run our now-stale copy), so
#: construction logs a loud warning rather than silently trusting an unverified copy.
_VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
_CONSECUTIVE_TIMEOUTS_BEFORE_REINIT: Final = 3
def get_litellm_async_redis_cluster_class(
def get_litellm_async_redis_cluster_class( # noqa: C901 # supports redis-py version-specific cluster implementations
cluster_node_class: type | None = None,
base_cluster_class: type | None = None,
) -> type["_AsyncRedisClusterType"]:
"""Returns the base ``RedisCluster`` when the installed redis-py already recovers a
node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
subclass with the per-node isolation fix for older versions whose upstream branch
tears down the whole cluster client.
"""Returns a timeout-tolerant ``RedisCluster`` subclass when installed redis-py already
recovers node-level connections per-connection (8.x+), else builds the ``RedisCluster``
subclass with the per-node isolation fix for older versions whose upstream branch tears
down the whole cluster client.
``cluster_node_class`` exists for dependency injection in tests; production callers
leave it unset and the installed ``ClusterNode`` is used.
``cluster_node_class`` and ``base_cluster_class`` exist for dependency injection in tests;
production callers leave them unset and the installed redis-py classes are used.
Imported lazily because this module is reachable from a base ``import litellm`` while
redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@ -118,13 +122,68 @@ def get_litellm_async_redis_cluster_class(
from redis.exceptions import TimeoutError as _RedisTimeoutError
node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
base_class: Final = base_cluster_class if base_cluster_class is not None else _BaseAsyncRedisCluster
if hasattr(node_class, "update_active_connections_for_reconnect"):
verbose_logger.debug(
"redis-py %s recovers a node-level connection error per-connection upstream; "
"using the base RedisCluster without litellm's node-isolation override.",
"redis-py %s recovers node connections per-connection upstream; using "
"LiteLLM's timeout-tolerant RedisCluster wrapper.",
redis.__version__,
)
return _BaseAsyncRedisCluster
class LiteLLMAsyncRedisClusterTimeoutTolerant(
base_class # pyright: ignore[reportGeneralTypeIssues, reportUntypedBaseClass] # the injected base class is selected at runtime
):
def __init__(
self,
*args: object,
**kwargs: object, # kwargs-ok: passes redis-py's constructor kwargs through untouched
) -> None:
self._litellm_initialize = False
self._litellm_reinit_requests = 0
self._litellm_tolerated_timeouts = 0
super().__init__(*args, **kwargs)
self._litellm_consecutive_timeouts: dict[ # mutable-ok: per-node counter updated on the command hot path
str, int
] = {}
@property
def _initialize(self) -> bool:
return self._litellm_initialize
@_initialize.setter
def _initialize(self, value: bool) -> None:
if value:
self._litellm_reinit_requests += 1
self._litellm_initialize = value
async def _execute_command(
self,
target_node: _ClusterNodeAttrs,
*args: object,
**kwargs: object, # kwargs-ok: matches redis-py's own command dispatch signature
) -> object:
outstanding_before: Final = self._litellm_reinit_requests - self._litellm_tolerated_timeouts
pending_before: Final = self._litellm_initialize
try:
result: Final = await super()._execute_command(target_node, *args, **kwargs)
except _RedisTimeoutError:
timeouts: Final = self._litellm_consecutive_timeouts.get(target_node.name, 0) + 1
if timeouts >= _CONSECUTIVE_TIMEOUTS_BEFORE_REINIT:
self._litellm_consecutive_timeouts.pop(target_node.name, None)
raise
self._litellm_consecutive_timeouts[target_node.name] = timeouts
self._litellm_tolerated_timeouts += 1
if (
not pending_before
and self._litellm_reinit_requests - self._litellm_tolerated_timeouts == outstanding_before
):
self._initialize = False
raise
if self._litellm_consecutive_timeouts:
self._litellm_consecutive_timeouts.pop(target_node.name, None)
return result
return LiteLLMAsyncRedisClusterTimeoutTolerant
if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
verbose_logger.warning(

1
litellm/constants.py
View file

@ -151,6 +151,7 @@ DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_SEMANTIC_
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60"))
MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200"))
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600"))
MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = int(os.getenv("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", "60"))
# Default npm cache directory for STDIO MCP servers.
# npm/npx needs a writable cache dir; in containers the default (~/.npm)

2
litellm/integrations/azure_storage/azure_storage.py
View file

@ -36,7 +36,7 @@ AZURE_STORAGE_TOKEN_SCOPE: Final = "https://storage.azure.com/.default"
def _cached_credential_chain_token_provider() -> Callable[[], str]:
return get_azure_ad_token_provider(
azure_scope=AZURE_STORAGE_TOKEN_SCOPE,
azure_credential=AzureCredentialType.DefaultAzureCredential,
azure_credential=AzureCredentialType.DeploymentIdentityCredential,
)

16
litellm/integrations/cloudzero/database.py
View file

@ -94,8 +94,18 @@ class LiteLLMDatabase:
try:
db_response: Final = await client.db.query_raw(query, *params)
# Convert the response to polars DataFrame with full schema inference
# This prevents schema mismatch errors when data types vary across rows
return pl.DataFrame(db_response, infer_schema_length=None)
from litellm.proxy.spend_tracking.key_metadata_recovery import (
fill_missing_api_key_aliases,
)
usage_rows: Final = (
db_response.to_dicts()
if isinstance(db_response, pl.DataFrame)
else db_response
if isinstance(db_response, list)
else []
)
recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
except Exception as e:
raise Exception(f"Error retrieving usage data: {e}")

14
litellm/integrations/focus/database.py
View file

@ -96,7 +96,19 @@ class FocusLiteLLMDatabase:
try:
db_response: Final = await client.db.query_raw(query, *query_params)
return pl.DataFrame(db_response, infer_schema_length=None)
from litellm.proxy.spend_tracking.key_metadata_recovery import (
fill_missing_api_key_aliases,
)
usage_rows: Final = (
db_response.to_dicts()
if isinstance(db_response, pl.DataFrame)
else db_response
if isinstance(db_response, list)
else []
)
recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
except Exception as exc:
raise RuntimeError(f"Error retrieving usage data: {exc}") from exc

32
litellm/litellm_core_utils/litellm_logging.py
View file

@ -10,7 +10,7 @@ import subprocess
import sys
import time
import traceback
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from datetime import datetime as dt_object
from functools import lru_cache
from types import MappingProxyType, TracebackType
@ -5897,14 +5897,28 @@ def _get_status_fields(
#########################################################
# Map - guardrail_information.guardrail_status to guardrail_status
#########################################################
guardrail_status: GuardrailStatus = "not_run"
if guardrail_information and isinstance(guardrail_information, list):
for information in guardrail_information:
if isinstance(information, dict):
raw_status = information.get("guardrail_status", "not_run")
if raw_status != "not_run":
guardrail_status = GUARDRAIL_STATUS_MAP.get(raw_status, "not_run")
break
# Severity order, least severe first. The status aggregates across ALL
# guardrail entries rather than taking the first non-"not_run" one: a
# pre_call guardrail that passed (e.g. a mask) records its entry before a
# later guardrail's block, and first-wins would report a blocked request
# as "success".
GUARDRAIL_STATUS_SEVERITY: Final[tuple[GuardrailStatus, ...]] = (
"not_run",
"success",
"guardrail_failed_to_respond",
"guardrail_intervened",
)
entries: Final[Sequence[object]] = guardrail_information if isinstance(guardrail_information, list) else ()
raw_statuses: Final[Iterator[object]] = (
entry.get("guardrail_status", "not_run") for entry in entries if isinstance(entry, dict)
)
# A guardrail is free to write any value here, and an unhashable one would
# raise TypeError on the mapping lookup and drop the whole payload.
guardrail_status: Final[GuardrailStatus] = max(
(GUARDRAIL_STATUS_MAP.get(raw_status, "not_run") for raw_status in raw_statuses if isinstance(raw_status, str)),
key=GUARDRAIL_STATUS_SEVERITY.index,
default="not_run",
)
return StandardLoggingPayloadStatusFields(llm_api_status=llm_api_status, guardrail_status=guardrail_status)

9
litellm/llms/base_llm/ocr/transformation.py
View file

@ -2,6 +2,7 @@
Base OCR transformation configuration.
"""
import builtins
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal
@ -93,8 +94,8 @@ class OCRResponse(LiteLLMPydanticObjectBase):
document_annotation: Any | None = None
usage_info: OCRUsageInfo | None = None
content: str | None = None
tables: list[dict[str, object]] | None = None
keyValuePairs: list[dict[str, object]] | None = None
tables: list[dict[str, builtins.object]] | None = None
keyValuePairs: list[dict[str, builtins.object]] | None = None
object: str = "ocr"
model_config = {"extra": "allow"}
@ -102,11 +103,11 @@ class OCRResponse(LiteLLMPydanticObjectBase):
# Define private attributes using PrivateAttr
_hidden_params: dict = PrivateAttr(default_factory=dict)
def set_provider_native_response(self, native_response: Mapping[str, object]) -> None:
def set_provider_native_response(self, native_response: Mapping[str, builtins.object]) -> None:
"""Keep the provider's own response payload alongside the normalized one."""
self._hidden_params[PROVIDER_NATIVE_RESPONSE_KEY] = native_response
def get_provider_native_response(self) -> Mapping[str, object] | None:
def get_provider_native_response(self) -> Mapping[str, builtins.object] | None:
"""The provider's own response payload, when `req_format=native` was requested."""
native_response: Final = self._hidden_params.get(PROVIDER_NATIVE_RESPONSE_KEY)
return native_response if isinstance(native_response, dict) else None

383
litellm/llms/snowflake/chat/transformation.py
View file

@ -8,23 +8,31 @@ Routes to native Cortex REST API endpoints based on model:
Ref: https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api
"""
import copy
import json
import re
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict
import httpx
from typing_extensions import ReadOnly
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_process_openai_file_message,
convert_to_anthropic_tool_result,
create_anthropic_image_param,
select_anthropic_content_block_type_for_file,
)
from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicStreamParser
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolMessage
from litellm.types.utils import (
ChatCompletionMessageToolCall,
ChatCompletionUsageBlock,
Choices,
Function,
GenericStreamingChunk,
Message,
ModelResponse,
Usage,
ModelResponseStream,
)
from ...base_llm.base_model_iterator import BaseModelResponseIterator
@ -93,6 +101,103 @@ def _is_claude_model(model: str) -> bool:
return any(name.startswith(p) for p in _CLAUDE_MODEL_PREFIXES)
def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object:
"""One OpenAI ``image_url`` block in the native shape Cortex accepts.
Cortex documents base64 sources only, so remote URLs are inlined the way every
other base64-only Anthropic dialect (Bedrock invoke, Vertex) inlines them, and
pdf/text data URIs become document blocks rather than malformed image blocks.
"""
image_url: Final = block.get("image_url")
url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url")
if not url:
return block
converted: Final = (
anthropic_process_openai_file_message({"type": "file", "file": {"file_data": url}})
if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document"
else create_anthropic_image_param(
image_url if isinstance(image_url, dict) else url, # mutable-ok: caller's JSON block
format=_image_url_field(image_url, "format"),
is_bedrock_invoke=True,
)
)
cache_control: Final = block.get("cache_control")
if cache_control is None:
return converted
return {**converted, "cache_control": cache_control} # mutable-ok: JSON wire block
def _image_url_field(image_url: object, key: str) -> str | None:
value: Final = image_url.get(key) if isinstance(image_url, dict) else None
return value if isinstance(value, str) else None
def _data_uri_media_type(url: str) -> str:
match: Final = re.match(r"data:([^;,]+)", url)
return match.group(1) if match else ""
def _convert_image_url_blocks_to_anthropic(content: object) -> object:
if not isinstance(content, list):
return content
return [ # mutable-ok: JSON wire blocks
_convert_image_url_to_anthropic(block)
if isinstance(block, Mapping) and block.get("type") == "image_url"
else block
for block in content
]
def _convert_tool_result_to_anthropic(
content: object, tool_call_id: str, cache_control: object
) -> Mapping[str, object]:
"""The Anthropic ``tool_result`` block for one OpenAI tool message.
Delegating to the shared converter keeps image, document and per-block cache
breakpoints identical to every other Anthropic dialect; only the plain-string
and non-list shapes it does not model are handled here.
"""
if not isinstance(content, list):
plain: Final[dict[str, object]] = { # mutable-ok: JSON wire block
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": content if isinstance(content, str) else json.dumps(content),
}
return {**plain, "cache_control": cache_control} if cache_control is not None else plain
converted: Final = convert_to_anthropic_tool_result(
ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content),
force_base64=True,
)
if cache_control is None:
return converted
return {**converted, "cache_control": cache_control} # mutable-ok: JSON wire block
def _signed_thinking_blocks(msg: object) -> list[dict[str, object]]: # mutable-ok: JSON wire blocks
"""The assistant turn's thinking blocks that can legally be echoed back.
Only signed blocks round-trip: Cortex rejects a thinking block whose signature is
missing, which is what an unsigned block from a non-thinking turn would produce.
"""
blocks: Final = msg.get("thinking_blocks") if isinstance(msg, dict) else getattr(msg, "thinking_blocks", None)
if not isinstance(blocks, list):
return [] # mutable-ok: JSON wire blocks
return [ # mutable-ok: JSON wire blocks
dict(block)
for block in blocks
if isinstance(block, Mapping) and (block.get("signature") or block.get("type") == "redacted_thinking")
]
def _clean_input_schema(schema: object) -> object: # mutable-ok: JSON schema copy
return (
{key: value for key, value in schema.items() if key != "$schema"}
if isinstance(schema, Mapping)
else schema # mutable-ok: JSON schema copy
) # mutable-ok: JSON schema copy
class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
"""
Snowflake Cortex REST API — unified provider.
@ -178,7 +283,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
if "description" in func:
anthropic_tool["description"] = func["description"]
if "parameters" in func:
anthropic_tool["input_schema"] = func["parameters"]
anthropic_tool["input_schema"] = _clean_input_schema(func["parameters"])
else:
anthropic_tool["input_schema"] = {
"type": "object",
@ -186,10 +291,16 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
}
anthropic_tools.append(anthropic_tool)
else:
anthropic_tools.append(tool)
anthropic_tools.append(
{**tool, "input_schema": _clean_input_schema(tool["input_schema"])} # mutable-ok: JSON wire tool
if "input_schema" in tool
else tool
)
return anthropic_tools
def _extract_system_and_messages(self, messages: list[AllMessageValues]) -> tuple[str | None, list[dict]]:
def _extract_system_and_messages( # mutable-ok: JSON wire messages
self, messages: list[AllMessageValues]
) -> tuple[list[dict] | None, list[dict]]:
"""
Split messages into system prompt and conversation turns for Anthropic format.
@ -197,26 +308,39 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
- assistant messages with tool_calls → tool_use content blocks
- tool role messages → user role with tool_result content blocks
"""
system_parts: Final[list[str]] = []
conversation: Final[list[dict]] = []
system_parts: Final[list[dict]] = [] # mutable-ok: JSON wire messages
conversation: Final[list[dict]] = [] # mutable-ok: JSON wire messages
for msg in messages:
if isinstance(msg, dict):
role = msg.get("role", "")
content: Any = msg.get("content", "")
msg_cache_control: object = msg.get("cache_control")
else:
role = getattr(msg, "role", "")
content = getattr(msg, "content", "")
msg_cache_control = getattr(msg, "cache_control", None)
if role == "system":
if isinstance(content, str) and content:
system_parts.append(content)
system_parts.append({"type": "text", "text": content}) # mutable-ok: JSON wire system block
elif isinstance(content, list):
system_parts.append("\n".join(b.get("text", "") for b in content if b.get("type") == "text"))
system_parts.extend(
{ # mutable-ok: JSON wire system block
"type": "text",
"text": block.get("text", ""),
**(
{"cache_control": block["cache_control"]} if "cache_control" in block else {}
), # mutable-ok: JSON wire block
}
for block in content
if isinstance(block, Mapping) and block.get("type") == "text"
)
elif role == "assistant":
tool_calls = msg.get("tool_calls") if isinstance(msg, dict) else getattr(msg, "tool_calls", None)
thinking_blocks = _signed_thinking_blocks(msg)
if tool_calls:
content_blocks: list[dict[str, object]] = []
content_blocks: list[dict[str, object]] = list(thinking_blocks) # mutable-ok: JSON wire blocks
if content:
content_blocks.append({"type": "text", "text": content})
for tc in tool_calls:
@ -239,18 +363,26 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
}
)
conversation.append({"role": "assistant", "content": content_blocks})
elif thinking_blocks:
thinking_content = (
[
*thinking_blocks,
*copy.deepcopy(content),
]
if isinstance(content, list)
else [*thinking_blocks, *([{"type": "text", "text": content}] if content else [])]
) # rebind-ok: loop-local normalized content
conversation.append({"role": "assistant", "content": thinking_content})
else:
conversation.append({"role": "assistant", "content": content})
elif role == "tool":
tool_call_id = (
tool_call_id_value = (
msg.get("tool_call_id", "") if isinstance(msg, dict) else getattr(msg, "tool_call_id", "")
)
tool_content = content if isinstance(content, str) else json.dumps(content)
tool_result_block = {
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": tool_content,
}
tool_call_id = (
tool_call_id_value if isinstance(tool_call_id_value, str) else ""
) # rebind-ok: normalized loop value
tool_result_block = _convert_tool_result_to_anthropic(content, tool_call_id, msg_cache_control)
if (
conversation
and conversation[-1]["role"] == "user"
@ -260,11 +392,18 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
):
conversation[-1]["content"].append(tool_result_block)
else:
conversation.append({"role": "user", "content": [tool_result_block]})
conversation.append(
{"role": "user", "content": [tool_result_block]} # mutable-ok: JSON wire message
) # mutable-ok: JSON wire message
else:
conversation.append({"role": role, "content": content})
conversation.append( # mutable-ok: JSON wire message
{ # mutable-ok: JSON wire message
"role": role,
"content": _convert_image_url_blocks_to_anthropic(content),
} # mutable-ok: JSON wire message
)
system: Final[str | None] = "\n\n".join(system_parts) if system_parts else None
system: Final[list[dict] | None] = system_parts if system_parts else None # mutable-ok: JSON wire messages
return system, conversation
def transform_request(
@ -339,7 +478,9 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
extra_body: dict,
) -> dict:
"""Anthropic Messages format for /messages endpoint."""
system, conversation = self._extract_system_and_messages(messages)
passthrough_system: Final = optional_params.pop("system", None)
extracted_system, conversation = self._extract_system_and_messages(messages)
system: Final = passthrough_system if passthrough_system is not None else extracted_system
if "tools" in optional_params:
optional_params["tools"] = self._transform_tools_to_anthropic(optional_params["tools"])
@ -353,16 +494,19 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
model_name: Final = model.removeprefix("snowflake/")
body: Final[dict[str, object]] = {
"model": model_name,
"messages": conversation,
"stream": stream,
**optional_params,
**extra_body,
}
body: Final[dict[str, object]] = normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire body
{ # mutable-ok: JSON wire body
"model": model_name,
"messages": conversation,
"stream": stream,
**optional_params,
**extra_body, # mutable-ok: JSON wire body
}
)
if system is not None:
body["system"] = system
body["system"] = normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire payload
{"system": system} # mutable-ok: JSON wire payload
)["system"]
if "max_tokens" not in body:
body["max_tokens"] = 4096 # reasonable default; Anthropic API max varies by model
@ -435,23 +579,10 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
additional_args={"complete_input_dict": request_data},
)
text_content = ""
tool_calls: Final = []
for block in response_json.get("content", []):
if block.get("type") == "text":
text_content += block.get("text", "")
elif block.get("type") == "tool_use":
tool_calls.append(
ChatCompletionMessageToolCall(
id=block.get("id", ""),
type="function",
function=Function(
name=block.get("name", ""),
arguments=json.dumps(block.get("input", {})),
),
)
)
anthropic_config: Final = AnthropicConfig()
text_content, _, thinking_blocks, reasoning_content, tool_calls, _, _, _ = (
anthropic_config.extract_response_content(completion_response=dict(response_json))
)
_stop_reason_map: Final = {
"end_turn": "stop",
@ -461,9 +592,13 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
}
finish_reason: Final = _stop_reason_map.get(response_json.get("stop_reason", "end_turn"), "stop")
message: Final = Message(content=text_content or None, role="assistant")
if tool_calls:
message.tool_calls = tool_calls
message: Final = Message(
content=text_content or None,
role="assistant",
tool_calls=tool_calls or None,
thinking_blocks=thinking_blocks,
reasoning_content=reasoning_content,
)
choice: Final = Choices(
finish_reason=finish_reason,
@ -471,11 +606,13 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
message=message,
)
usage_data: Final = response_json.get("usage", {})
usage: Final = Usage(
prompt_tokens=usage_data.get("input_tokens", 0),
completion_tokens=usage_data.get("output_tokens", 0),
total_tokens=usage_data.get("input_tokens", 0) + usage_data.get("output_tokens", 0),
# Cortex reports prompt-cache creation/read counts alongside input_tokens; the
# shared calculator folds them into prompt_tokens_details so cached input is
# visible and billed at its own rate.
usage: Final = anthropic_config.calculate_usage(
usage_object=response_json.get("usage", {}),
reasoning_content=reasoning_content,
completion_response=dict(response_json),
)
model_response.choices = [choice]
@ -516,15 +653,19 @@ class SnowflakeStreamingHandler(BaseModelResponseIterator):
json_mode: bool | None = False,
):
super().__init__(streaming_response=streaming_response, sync_stream=sync_stream)
self._tool_index = 0
self._tool_id = ""
self._tool_name = ""
self._input_tokens = 0
# Cortex streams the Anthropic SSE dialect on /messages, so its events are parsed
# by Anthropic's own parser: thinking deltas, signatures and prompt-cache usage
# all arrive the way they do on every other Anthropic-dialect provider.
self._anthropic_parser: Final = AnthropicStreamParser(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
def chunk_parser(self, chunk: dict) -> GenericStreamingChunk | ModelResponseStream:
if "choices" in chunk:
return self._parse_openai_chunk(chunk)
return self._parse_anthropic_chunk(chunk)
return self._anthropic_parser.chunk_parser(chunk)
def _parse_openai_chunk(self, chunk: dict) -> GenericStreamingChunk:
choices: Final = chunk.get("choices", [])
@ -566,117 +707,3 @@ class SnowflakeStreamingHandler(BaseModelResponseIterator):
index=choice.get("index", 0),
tool_use=tool_use,
)
def _parse_anthropic_chunk(self, chunk: dict) -> GenericStreamingChunk:
event_type: Final = chunk.get("type", "")
if event_type == "message_start":
message: Final = chunk.get("message", {})
usage_data = message.get("usage", {})
self._input_tokens = usage_data.get("input_tokens", 0)
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)
elif event_type == "content_block_delta":
delta = chunk.get("delta", {})
delta_type: Final = delta.get("type", "")
if delta_type == "text_delta":
return GenericStreamingChunk(
text=delta.get("text", ""),
is_finished=False,
finish_reason="",
usage=None,
index=chunk.get("index", 0),
tool_use=None,
)
elif delta_type == "input_json_delta":
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=chunk.get("index", 0),
tool_use=ChatCompletionToolCallChunk(
id=self._tool_id,
type="function",
function={
"name": self._tool_name,
"arguments": delta.get("partial_json", ""),
},
index=self._tool_index,
),
)
elif event_type == "content_block_start":
content_block: Final = chunk.get("content_block", {})
if content_block.get("type") == "tool_use":
self._tool_id = content_block.get("id", "")
self._tool_name = content_block.get("name", "")
self._tool_index = chunk.get("index", 0)
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=chunk.get("index", 0),
tool_use=ChatCompletionToolCallChunk(
id=self._tool_id,
type="function",
function={"name": self._tool_name, "arguments": ""},
index=self._tool_index,
),
)
elif event_type == "message_delta":
delta = chunk.get("delta", {})
stop_reason: Final = delta.get("stop_reason", "")
usage_data = chunk.get("usage", {})
_stop_map: Final = {
"end_turn": "stop",
"max_tokens": "length",
"tool_use": "tool_calls",
"stop_sequence": "stop",
}
usage = None
if usage_data or self._input_tokens:
output_t: Final = usage_data.get("output_tokens", 0)
input_t: Final = self._input_tokens or usage_data.get("input_tokens", 0)
usage = ChatCompletionUsageBlock(
prompt_tokens=input_t,
completion_tokens=output_t,
total_tokens=input_t + output_t,
)
return GenericStreamingChunk(
text="",
is_finished=True,
finish_reason=_stop_map.get(stop_reason, "stop"),
usage=usage,
index=0,
tool_use=None,
)
elif event_type == "message_stop":
return GenericStreamingChunk(
text="",
is_finished=True,
finish_reason="stop",
usage=None,
index=0,
tool_use=None,
)
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)

40
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -3638,30 +3638,48 @@ class MCPServerManager:
user_api_key_auth: UserAPIKeyAuth | None,
raw_headers: Mapping[str, str] | None = None,
) -> None:
"""Run the OBO exchange for a caller-supplied subject at the transport edge.
"""Mint an exchange-backed server's upstream credential at the transport edge.
Single-server routes call this before the MCP session opens, where an HTTP status and
``WWW-Authenticate`` still reach the client. A rejected subject raises the RFC 9728
challenge and any other ``CredError`` maps onto its public HTTP status, so an exchange
failure surfaces as a failure instead of the session continuing into an empty tool list.
A successful exchange is cached by the exchanger, so the session's list/call reuses it.
Each mode pre-flights only where it would resolve the subject the session goes on to use,
which is what keeps the pre-flight from reaching a verdict the session would contradict.
``oauth2_token_exchange`` mints from the caller's inbound bearer, so without one there is
nothing to exchange and the missing-subject case stays the preemptive challenge's job.
``oauth2_id_jag`` is the mirror image: tool listing resolves it from the identity assertion
captured for this user at SSO login and never from the inbound bearer, so the pre-flight is
faithful exactly when no identity bearer was sent (a LiteLLM key in ``Authorization`` is not one),
and a caller that did send one is passed through
untouched rather than judged against a subject the listing will not use. That store-sourced
case is the one whose missing-assertion 412 and store-outage 503 the session cannot report.
Only OBO has a discovery challenge to raise; ID-JAG's failures are plain statuses whose body
already names what the user has to do, so they map through ``raise_public`` as at egress.
"""
if server.auth_type != MCPAuth.oauth2_token_exchange:
return
if not self._extract_bearer_token(oauth2_headers, None):
return
resolved_server: Final = await self.ensure_oauth_metadata_discovered(server)
spec: Final = to_server_spec(resolved_server)
if spec is None or not isinstance(spec.config, TokenExchangeConfig):
return
subject_token: Final = self._extract_subject_token(oauth2_headers, raw_headers, user_api_key_auth)
if subject_token is None:
match server.auth_type:
case MCPAuth.oauth2_token_exchange:
if not self._extract_bearer_token(oauth2_headers, None):
return
case MCPAuth.oauth2_id_jag:
if subject_token is not None:
return
case _:
return
resolved_server: Final = await self.ensure_oauth_metadata_discovered(server)
spec: Final = _to_server_spec_fail_closed(resolved_server)
if spec is None or not isinstance(spec.config, (TokenExchangeConfig, IdJagConfig)):
return
if subject_token is None and isinstance(spec.config, TokenExchangeConfig):
raise_token_exchange_challenge(resolved_server, root_path=get_server_root_path())
match await self._cred_provider.resolve_credentials(to_subject(user_api_key_auth, subject_token), spec):
case Ok(_):
return
case Error(err):
if err.tag == "unauthorized":
if err.tag == "unauthorized" and isinstance(spec.config, TokenExchangeConfig):
raise_token_exchange_challenge(
resolved_server,
root_path=get_server_root_path(),

74
litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py
View file

@ -11,7 +11,8 @@ being registered, so a gateway with no EMA upstream never stores bearer material
The row is one encrypted payload per user, latest login wins. ``expires_at`` mirrors the
id_token ``exp`` claim and is judged by the reader, never enforced by deletion here: an
expired assertion with a refresh token is still renewable, and the DB row is the source of
truth, the same contract as the per-user OAuth credential store.
truth, the same contract as the per-user OAuth credential store. Reads use a per-process cache with
TTL ``MCP_SSO_ASSERTION_CACHE_TTL_SECONDS``; invalidation also guards against stale in-flight reads.
"""
from __future__ import annotations
@ -24,6 +25,8 @@ import jwt
from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE, MCP_SSO_ASSERTION_CACHE_TTL_SECONDS
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
@ -45,6 +48,46 @@ class SSOIdentityAssertion(BaseModel):
expires_at: datetime | None = None
class SSOAssertionCache:
"""Process-local read cache. ``invalidate`` bumps a process-wide epoch so a fetch that started
before a login cannot repopulate the old assertion after it."""
def __init__(self, ttl_seconds: int = MCP_SSO_ASSERTION_CACHE_TTL_SECONDS) -> None:
self._entries = InMemoryCache(
max_size_in_memory=MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE,
default_ttl=ttl_seconds,
)
self._epoch: int = 0
def epoch(self) -> int:
return self._epoch
def get(self, user_id: str) -> SSOIdentityAssertion | None:
cached: Final = self._entries.get_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped
user_id
)
return cached if isinstance(cached, SSOIdentityAssertion) else None
def set_if_unchanged(self, user_id: str, assertion: SSOIdentityAssertion, seen_epoch: int) -> None:
if self._epoch != seen_epoch:
return
self._entries.set_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped
user_id, assertion
)
def invalidate(self, user_id: str) -> None:
self._epoch += 1
self._entries.delete_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped
user_id
)
def flush(self) -> None:
self._entries.flush_cache() # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped
_ASSERTION_CACHE: Final = SSOAssertionCache()
class _IdTokenClaims(BaseModel):
exp: float | None = None
iss: str | None = None
@ -107,7 +150,9 @@ async def ema_assertion_retention_enabled() -> bool:
return row is not None
async def persist_sso_identity_assertion(user_id: str, assertion: SSOIdentityAssertion) -> None:
async def persist_sso_identity_assertion(
user_id: str, assertion: SSOIdentityAssertion, cache: SSOAssertionCache = _ASSERTION_CACHE
) -> None:
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper # noqa: PLC0415 # runtime global
from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global
@ -127,11 +172,10 @@ async def persist_sso_identity_assertion(user_id: str, assertion: SSOIdentityAss
"update": {"assertion_b64": encoded},
},
)
cache.invalidate(user_id)
async def fetch_sso_identity_assertion(user_id: str) -> SSOIdentityAssertion | None:
"""The stored assertion for ``user_id``, or ``None`` when absent, undecryptable (salt-key
rotation), or unparseable. Expiry is not judged here; the reader owns that policy."""
async def _read_assertion_from_db(user_id: str) -> SSOIdentityAssertion | None:
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper # noqa: PLC0415 # runtime global
from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # runtime global
@ -160,6 +204,21 @@ async def fetch_sso_identity_assertion(user_id: str) -> SSOIdentityAssertion | N
)
async def fetch_sso_identity_assertion(
user_id: str, cache: SSOAssertionCache = _ASSERTION_CACHE
) -> SSOIdentityAssertion | None:
"""The stored assertion for ``user_id``, or ``None`` when absent, undecryptable (salt-key
rotation), or unparseable. Expiry is not judged here; the reader owns that policy."""
cached: Final = cache.get(user_id)
if cached is not None:
return cached
seen_epoch: Final = cache.epoch()
assertion: Final = await _read_assertion_from_db(user_id)
if assertion is not None:
cache.set_if_unchanged(user_id, assertion, seen_epoch)
return assertion
class AssertionStoreUnavailable(Exception):
"""Raised by ``fetch`` when the backing store is unreachable (e.g. the DB is down).
@ -189,9 +248,12 @@ class DbSSOAssertionStore:
from credential resolution and from the upstream-401 retry.
"""
def __init__(self, cache: SSOAssertionCache = _ASSERTION_CACHE) -> None:
self._cache = cache
async def fetch(self, user_id: str) -> SSOIdentityAssertion | None:
try:
return await fetch_sso_identity_assertion(user_id)
return await fetch_sso_identity_assertion(user_id, cache=self._cache)
except Exception as exc: # noqa: BLE001 # any driver/storage failure is an outage, not an absence
raise AssertionStoreUnavailable(str(exc)) from exc

14
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -3851,15 +3851,15 @@ if MCP_AVAILABLE:
raise_token_exchange_challenge(server, root_path=get_server_root_path())
# token_exchange (OBO) with a subject present: run the exchange here at the transport
# edge, so a rejected subject raises the RFC 9728 challenge (and a gateway fault its
# public status) instead of the session opening and list_tools masking the failure as
# an empty tool list. Gated to single-server routes; the multi-server aggregate keeps
# absorbing per-server auth failures so one bad server cannot 401 the whole connect.
# Exchange-backed modes (token_exchange's OBO mint, id_jag's stored-assertion mint): run
# the exchange here at the transport edge, so a rejected subject raises the RFC 9728
# challenge and any other failure its public status, instead of the session opening and
# list_tools masking it as an empty tool list. The manager owns which modes pre-flight
# and what each mints from. Gated to single-server routes the key may reach; the
# multi-server aggregate keeps absorbing per-server auth failures so one bad server
# cannot 401 the whole connect.
if (
server
and server.auth_type == MCPAuth.oauth2_token_exchange
and oauth2_headers
and len(mcp_servers or []) == 1
and server.server_id
in frozenset(

11
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -3151,6 +3151,17 @@
}
],
"title": "Team Id"
},
"user_email": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "User Email"
}
},
"title": "KeyMetadata",

12
litellm/proxy/_types.py
View file

@ -737,6 +737,9 @@ class LiteLLMRoutes(enum.Enum):
"/.well-known/litellm-ui-config",
"/public/model_hub",
"/public/v1/model_hub",
"/public/v1/model_hub/providers",
"/public/v1/model_hub/modes",
"/public/v1/model_hub/features",
"/public/model_hub/info",
"/public/agent_hub",
"/public/mcp_hub",
@ -2404,6 +2407,15 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"""
completion_model: str | None = Field(None, description="proxy level default model for all chat completion calls")
max_in_flight_requests_per_worker: int | None = Field(
None, gt=0, description="maximum concurrent requests handled by each worker"
)
max_queued_requests_per_worker: int | None = Field(
None, ge=0, description="maximum requests waiting for a worker slot"
)
admission_queue_timeout_seconds: float = Field(
1.0, gt=0, description="maximum time a request waits for a worker slot"
)
plugins: list[PluginConfig] | None = Field(
None, description="external services registered as embeddable UI plugins"
)

42
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -9,7 +9,7 @@ from fastapi.responses import JSONResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping
from litellm.anthropic_interface.exceptions import AnthropicErrorResponse, AnthropicExceptionMapping
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.llms.anthropic.experimental_pass_through.context_management import (
AnthropicContextManagementError,
@ -30,6 +30,27 @@ from litellm.types.utils import TokenCountResponse
router: Final = APIRouter()
def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
from litellm.proxy.proxy_server import (
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
)
status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500
_close_dangling_otel_server_span(request, status_code, exc=exc)
envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
status_code=status_code,
raw_message=exc.message,
request_id=request.headers.get("x-request-id"),
)
if not exc.provider_specific_fields:
return JSONResponse(status_code=status_code, content=envelope, headers=exc.headers)
content: Final[AnthropicErrorResponse] = {
**envelope,
"error": {**envelope["error"], "provider_specific_fields": exc.provider_specific_fields},
}
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
def _strip_total_tokens_from_anthropic_response(response: Any) -> None:
"""Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM
injects into Anthropic /v1/messages responses.
@ -195,7 +216,7 @@ async def anthropic_response(
verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e)
if isinstance(e, ProxyException):
raise
return _anthropic_error_json_response(e, request)
# Extract model_id from request metadata (same as success path)
litellm_metadata: Final = data.get("litellm_metadata", {}) or {}
@ -216,15 +237,18 @@ async def anthropic_response(
)
if isinstance(e, HTTPException):
raise proxy_exception_from_http_exception(e, headers)
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request)
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", 500),
headers=headers,
return _anthropic_error_json_response(
ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", 500),
headers=headers,
),
request,
)

31
litellm/proxy/auth/model_checks.py
View file

@ -387,33 +387,22 @@ def _get_wildcard_models(
all_wildcard_models: Final = []
for model in unique_models:
if _check_wildcard_routing(model=model):
if return_wildcard_routes: # will add the wildcard route to the list eg: anthropic/*.
if return_wildcard_routes:
all_wildcard_models.append(model)
## get litellm params from model
if llm_router is not None:
model_list = llm_router.get_model_list(model_name=model, team_id=team_id)
if model_list:
for router_model in model_list:
wildcard_models = get_known_models_from_wildcard(
models_to_remove.add(model)
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
if model_list:
for router_model in model_list:
all_wildcard_models.extend(
get_known_models_from_wildcard(
wildcard_model=model,
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
)
all_wildcard_models.extend(wildcard_models)
else:
# Router has no deployment for this wildcard (e.g., BYOK team models)
# Fall back to expanding from known provider models
wildcard_models = get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)
if wildcard_models:
models_to_remove.add(model)
all_wildcard_models.extend(wildcard_models)
)
else:
# get all known provider models
wildcard_models = get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)
if wildcard_models:
models_to_remove.add(model)
all_wildcard_models.extend(wildcard_models)
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
for model in models_to_remove:
unique_models.remove(model)

2
litellm/proxy/client/models.py
View file

@ -32,7 +32,7 @@ class ModelsManagementClient:
headers["Authorization"] = f"Bearer {self._api_key}"
return headers
def list(self, return_request: bool = False) -> list[dict[str, Any]] | requests.Request:
def list(self, return_request: bool = False) -> builtins.list[dict[str, Any]] | requests.Request:
"""
Get the list of models supported by the server.

2
litellm/proxy/client/teams.py
View file

@ -40,7 +40,7 @@ class TeamsManagementClient:
self,
user_id: str | None = None,
organization_id: str | None = None,
) -> list[dict[str, Any]]:
) -> builtins.list[dict[str, Any]]:
"""
List teams that the user belongs to.

19
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -50,6 +50,9 @@ from litellm.proxy.health_check import (
perform_health_check,
run_with_timeout,
)
from litellm.proxy.middleware.admission_control_middleware import (
get_admission_control_stats,
)
from litellm.proxy.middleware.in_flight_requests_middleware import (
get_in_flight_requests,
)
@ -63,6 +66,13 @@ from litellm.secret_managers.main import get_secret_bool
#### Health ENDPOINTS ####
class _HealthBacklogResponse(TypedDict):
in_flight_requests: ReadOnly[int]
admitted_requests: ReadOnly[int]
queued_requests: ReadOnly[int]
rejected_requests: ReadOnly[int]
def _reject_os_environ_references(params: dict) -> None:
"""
Validate that the provided params do not contain any ``os.environ/``
@ -1759,7 +1769,14 @@ async def health_backlog():
for the event loop to get to them, adding latency before LiteLLM even starts
its own timer.
"""
return {"in_flight_requests": get_in_flight_requests()}
stats: Final = get_admission_control_stats()
response: Final[_HealthBacklogResponse] = {
"in_flight_requests": get_in_flight_requests(),
"admitted_requests": stats.admitted,
"queued_requests": stats.queued,
"rejected_requests": stats.rejected_total,
}
return response
@router.get(

12
litellm/proxy/list_api/in_memory.py
View file

@ -141,3 +141,15 @@ class InMemoryListExecutor(Generic[TRow]):
async def find_many(self, plan: QueryPlan) -> Sequence[TRow]:
page: Final = _ordered(self._matching(plan.where), plan.order)[plan.skip : plan.skip + plan.take]
return await self.enrich_page(tuple(row for _, row in page))
async def distinct(self, field: str, where: tuple[Predicate, ...]) -> Sequence[str]:
"""A repeated field contributes each of its elements, so a facet over `providers`
lists providers rather than the tuples rows happen to carry."""
cells: Final = (cells.get(field) for cells, _ in self._matching(where))
values: Final = (
value
for cell in cells
for value in (cell if isinstance(cell, tuple) else (cell,))
if isinstance(value, str) and value
)
return tuple(sorted(frozenset(values)))

82
litellm/proxy/list_api/list_framework.py
View file

@ -28,12 +28,15 @@ from litellm.proxy.list_api.common import (
PROBLEM_TYPE_BASE,
ManagementProblem,
build_list_links,
build_page_links,
escape_like,
unknown_query_param_problem,
)
from litellm.types.proxy.management_endpoints.management_v1 import (
FacetListResponse,
ListMeta,
ListResponse,
PageMeta,
ProblemDetail,
)
@ -186,6 +189,13 @@ class ListExecutor(Protocol[TRow_co]):
async def find_many(self, plan: QueryPlan) -> Sequence[TRow_co]: ...
class FacetExecutor(Protocol):
"""The half of a facet that knows the rows. Separate from `ListExecutor` so a SQL
executor is not forced to implement `distinct` to keep serving entity lists."""
async def distinct(self, field: str, where: tuple[Predicate, ...]) -> Sequence[str]: ...
def order_by_sql(order: tuple[SortKey, ...]) -> str:
"""`ORDER BY` body for a plan, NULLS LAST in both directions.
@ -515,6 +525,78 @@ def build_query_plan(
)
def _facet_allowed_params(spec: ListSpec[TRow, TOut]) -> tuple[str, ...]:
"""A facet's values are always ascending, so `sort` is not one of its parameters."""
return tuple(name for name in _allowed_params(spec) if name != SORT_PARAM)
def _facet_where(
spec: ListSpec[TRow, TOut],
params: Mapping[str, str],
caller: UserAPIKeyAuth,
) -> tuple[Predicate, ...] | ProblemDetail:
scope_predicates: Final = _scope_predicates(spec.scope(caller))
if isinstance(scope_predicates, ProblemDetail):
return scope_predicates
filters: Final = _parse_filters(spec, params)
if isinstance(filters, ProblemDetail):
return filters
search: Final = _search_predicate(spec, params)
return scope_predicates + filters + ((search,) if search is not None else ())
async def handle_facet(
spec: ListSpec[TRow, TOut],
executor: FacetExecutor,
request: Request,
caller: UserAPIKeyAuth,
field: str,
) -> FacetListResponse:
"""The distinct values one column takes over a filtered query on a resource.
Carries the parent's parameters so a filter dropdown offers exactly the values the
table can show, and `has_more` rather than a total, which would cost a COUNT(*) over
the whole match set on every keystroke.
"""
params: Final = request.query_params
unknown: Final = tuple(sorted(name for name in params if name == SORT_PARAM or not _is_known_param(spec, name)))
if unknown:
raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=_facet_allowed_params(spec)))
duplicates: Final = _duplicate_params(request)
if duplicates:
raise ManagementProblem(
_problem(
"duplicate-query-parameter",
"Duplicate query parameter",
400,
f"Repeated query parameter(s): {', '.join(duplicates)}. Each may appear once; "
f"use a comma-separated list for multiple filter values.",
)
)
page: Final = _parse_page(params)
if isinstance(page, ProblemDetail):
raise ManagementProblem(page)
page_size: Final = _parse_page_size(spec, params)
if isinstance(page_size, ProblemDetail):
raise ManagementProblem(page_size)
where: Final = _facet_where(spec, params, caller)
if isinstance(where, ProblemDetail):
raise ManagementProblem(where)
values: Final = await executor.distinct(field, where)
skip: Final = (page - 1) * page_size
window: Final = values[skip : skip + page_size + 1]
has_more: Final = len(window) > page_size
return FacetListResponse(
data=tuple(window[:page_size]),
meta=PageMeta(page=page, page_size=page_size, has_more=has_more),
links=build_page_links(request=request, page=page, has_more=has_more),
)
def _duplicate_params(request: Request) -> tuple[str, ...]:
names: Final = tuple(name for name, _ in request.query_params.multi_items())
return tuple(sorted(frozenset(name for name in names if names.count(name) > 1)))

4
litellm/proxy/logging_endpoints/callback_logs_endpoints.py
View file

@ -88,8 +88,10 @@ class CallbackLogsReplayer:
)
metadata: Final[dict[str, Any]] = payload.get("metadata") or {}
user_api_key_hash: Final = metadata.get("user_api_key_hash")
litellm_metadata: Final[dict[str, Any]] = {
"user_api_key": metadata.get("user_api_key_hash"),
"user_api_key": user_api_key_hash,
"user_api_key_hash": user_api_key_hash,
"user_api_key_alias": metadata.get("user_api_key_alias"),
"user_api_key_user_id": metadata.get("user_api_key_user_id"),
"user_api_key_team_id": metadata.get("user_api_key_team_id"),

84
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -2,15 +2,19 @@ import asyncio
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from types import MappingProxyType, SimpleNamespace
from typing import TYPE_CHECKING, Final, Protocol
from fastapi import HTTPException, status
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.constants import PTU_SENTINEL_API_KEY
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.spend_tracking.key_metadata_recovery import (
attach_user_emails,
recover_double_hashed_key_metadata,
)
from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
@ -111,8 +115,19 @@ class DailySpendRecord(Protocol):
class _KeyMetadataDict(TypedDict, total=False):
key_alias: str | None
team_id: str | None
key_alias: ReadOnly[str | None]
team_id: ReadOnly[str | None]
user_id: ReadOnly[str | None]
user_email: ReadOnly[str | None]
def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
meta: Final = api_key_metadata.get(api_key, {})
return KeyMetadata(
key_alias=meta.get("key_alias"),
team_id=meta.get("team_id"),
user_email=meta.get("user_email"),
)
_WhereValue = str | dict[str, object]
@ -283,10 +298,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.models[model_key].api_key_breakdown:
breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics,
@ -310,10 +322,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown:
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics,
@ -335,10 +344,7 @@ def update_breakdown_metrics(
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = (
KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
)
@ -363,10 +369,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.providers[provider].api_key_breakdown:
breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics,
@ -388,10 +391,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown:
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics,
@ -403,10 +403,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.api_keys:
breakdown.api_keys[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
), # Add any api_key-specific metadata here
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record)
@ -426,10 +423,7 @@ def update_breakdown_metrics(
if record.api_key not in breakdown.entities[entity_value].api_key_breakdown:
breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics,
@ -442,17 +436,23 @@ def update_breakdown_metrics(
async def get_api_key_metadata(
prisma_client: PrismaClient,
api_keys: AbstractSet[str],
) -> dict[str, _KeyMetadataDict]:
) -> Mapping[str, _KeyMetadataDict]:
"""Get api key metadata, falling back to deleted keys table for keys not found in active table.
This ensures that key_alias and team_id are preserved in historical activity logs
even after a key is deleted or regenerated.
even after a key is deleted or regenerated. Also recovers aliases for api_key
values that were double-hashed by the v1.99 spend-log provenance gate.
"""
key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many(
where={"token": {"in": list(api_keys)}}
)
result: Final[dict[str, _KeyMetadataDict]] = {
k.token: {"key_alias": k.key_alias, "team_id": k.team_id} for k in key_records
k.token: {
"key_alias": k.key_alias,
"team_id": k.team_id,
"user_id": getattr(k, "user_id", None),
}
for k in key_records
}
# For any keys not found in the active table, check the deleted keys table
@ -471,6 +471,7 @@ async def get_api_key_metadata(
result[k.token] = {
"key_alias": k.key_alias,
"team_id": k.team_id,
"user_id": getattr(k, "user_id", None),
}
except Exception as e:
verbose_proxy_logger.warning(
@ -479,7 +480,13 @@ async def get_api_key_metadata(
e,
)
return result
still_missing: Final = api_keys - frozenset(result)
combined: Final = (
result
if not still_missing
else MappingProxyType({**result, **(await recover_double_hashed_key_metadata(prisma_client, still_missing))})
)
return await attach_user_emails(prisma_client, combined)
def _adjust_dates_for_timezone(
@ -951,11 +958,6 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
)
def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
meta: Final = api_key_metadata.get(api_key, {})
return KeyMetadata(key_alias=meta.get("key_alias"), team_id=meta.get("team_id"))
def _aggregate_grouping_sets_records_sync(
*,
records: Sequence[_GroupingSetsRow],

15
litellm/proxy/management_endpoints/policy_endpoints/endpoints.py
View file

@ -12,7 +12,7 @@ All /policy management endpoints
import copy
import json
import os
from collections.abc import AsyncIterator
from collections.abc import AsyncGenerator, AsyncIterator
from typing import TYPE_CHECKING, Final, Literal, cast
from fastapi import APIRouter, Depends, HTTPException, Request
@ -20,6 +20,7 @@ from fastapi.responses import Response, StreamingResponse
from pydantic import BaseModel, Field
from typing_extensions import TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
COMPETITOR_LLM_TEMPERATURE,
@ -32,6 +33,10 @@ from litellm.llms.openai.chat.guardrail_translation.handler import (
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
wrap_sse_stream_with_keepalive_pings,
)
from litellm.proxy.guardrails.guardrail_hooks.custom_code import (
RESPONSE_REJECTION_GUARDRAIL_CODE,
CustomCodeGuardrail,
@ -811,7 +816,7 @@ async def _stream_competitor_events(
llm_enrichment: dict,
brand_name: str,
model: str,
) -> AsyncIterator[str]:
) -> AsyncGenerator[str, None]:
"""Stream competitor names as SSE events, then emit a final 'done' event."""
competitors: Final[list[str]] = list(data.competitors or [])
@ -883,7 +888,11 @@ async def enrich_policy_template_stream(
model: Final = data.model or DEFAULT_COMPETITOR_DISCOVERY_MODEL
return StreamingResponse(
_stream_competitor_events(data, template, llm_enrichment, brand_name, model),
wrap_sse_stream_with_keepalive_pings(
_stream_competitor_events(data, template, llm_enrichment, brand_name, model),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)

4
litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py
View file

@ -4,7 +4,7 @@ usage/spend data by querying the aggregated daily activity endpoints.
"""
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
from datetime import date
from typing import Any, Final, Literal, Protocol, cast, overload
@ -543,7 +543,7 @@ async def stream_usage_ai_chat(
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
) -> AsyncIterator[str]:
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages

19
litellm/proxy/management_endpoints/usage_endpoints/endpoints.py
View file

@ -10,8 +10,13 @@ from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
wrap_sse_stream_with_keepalive_pings,
)
router: Final = APIRouter()
@ -56,11 +61,15 @@ async def usage_ai_chat(
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
return StreamingResponse(
stream_usage_ai_chat(
messages=messages,
model=data.model,
user_id=user_id,
is_admin=is_admin,
wrap_sse_stream_with_keepalive_pings(
stream_usage_ai_chat(
messages=messages,
model=data.model,
user_id=user_id,
is_admin=is_admin,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},

315
litellm/proxy/middleware/admission_control_middleware.py Normal file
View file

@ -0,0 +1,315 @@
import asyncio
import os
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import lru_cache
from typing import Annotated, Final, Protocol, TypeAlias, runtime_checkable
from pydantic import Field, TypeAdapter, ValidationError
from starlette.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
from litellm._logging import verbose_proxy_logger
_EXEMPT_PATHS: Final[frozenset[str]] = frozenset(
{
"/health/liveliness",
"/health/liveness",
"/health/readiness",
"/health/readiness/details",
"/health/backlog",
"/health/drain",
"/metrics",
"/metrics/",
}
)
@dataclass(frozen=True, slots=True)
class AdmissionControlSettings:
max_in_flight_requests: int
max_queued_requests: int
queue_timeout_seconds: float
AdmissionControlSettingsGetter: TypeAlias = Callable[[], AdmissionControlSettings | None] # mutable-ok: Callable params
@dataclass(frozen=True, slots=True)
class AdmissionControlStats:
admitted: int
queued: int
rejected_total: int
@runtime_checkable
class _Gauge(Protocol):
def inc(self, amount: float = 1) -> None: ...
def dec(self, amount: float = 1) -> None: ...
@runtime_checkable
class _CounterChild(Protocol):
def inc(self, amount: float = 1) -> None: ...
@runtime_checkable
class _Counter(Protocol):
def labels(self, reason: str) -> _CounterChild: ...
@dataclass(frozen=True, slots=True)
class AdmissionControlMetrics:
admitted_gauge: _Gauge
queued_gauge: _Gauge
rejected_counter: _Counter
AdmissionControlMetricsFactory: TypeAlias = Callable[[], AdmissionControlMetrics | None] # mutable-ok: Callable params
class AdmissionControlState:
"""Per-process admission counters and the in-flight semaphore shared by one worker's requests."""
def __init__(self, metrics_factory: AdmissionControlMetricsFactory) -> None:
self._metrics_factory = metrics_factory
self._metrics: AdmissionControlMetrics | None = None
self._metrics_init_attempted = False
self._admitted = 0
self._queued = 0
self._rejected_total = 0
self._semaphore: asyncio.Semaphore | None = None
self._semaphore_loop: asyncio.AbstractEventLoop | None = None
def get_stats(self) -> AdmissionControlStats:
return AdmissionControlStats(
admitted=self._admitted,
queued=self._queued,
rejected_total=self._rejected_total,
)
def get_semaphore(self, max_in_flight_requests: int) -> asyncio.Semaphore:
loop: Final = asyncio.get_running_loop()
if self._semaphore_loop is not loop:
self._semaphore = asyncio.Semaphore(max_in_flight_requests)
self._semaphore_loop = loop
semaphore: Final = self._semaphore
if semaphore is None:
raise RuntimeError("Admission control semaphore was not initialized")
return semaphore
def record_admission(self) -> None:
self._admitted += 1
metrics: Final = self._get_metrics()
if metrics is not None:
metrics.admitted_gauge.inc()
def record_release(self) -> None:
self._admitted -= 1
metrics: Final = self._get_metrics()
if metrics is not None:
metrics.admitted_gauge.dec()
def record_queue(self) -> None:
self._queued += 1
metrics: Final = self._get_metrics()
if metrics is not None:
metrics.queued_gauge.inc()
def record_dequeue(self) -> None:
self._queued -= 1
metrics: Final = self._get_metrics()
if metrics is not None:
metrics.queued_gauge.dec()
def record_rejection(self, reason: str) -> None:
self._rejected_total += 1
metrics: Final = self._get_metrics()
if metrics is not None:
metrics.rejected_counter.labels(reason=reason).inc()
def _get_metrics(self) -> AdmissionControlMetrics | None:
if not self._metrics_init_attempted:
self._metrics_init_attempted = True
self._metrics = self._metrics_factory()
return self._metrics
class AdmissionControlMiddleware:
def __init__(
self,
app: ASGIApp,
get_settings: AdmissionControlSettingsGetter,
state: AdmissionControlState,
) -> None:
self.app = app
self.get_settings = get_settings
self.state = state
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
settings: Final = self.get_settings()
if settings is None or _get_route_path(scope) in _EXEMPT_PATHS:
await self.app(scope, receive, send)
return
state: Final = self.state
semaphore: Final = state.get_semaphore(settings.max_in_flight_requests)
if not semaphore.locked():
await semaphore.acquire()
state.record_admission()
elif state.get_stats().queued >= settings.max_queued_requests:
state.record_rejection("queue_full")
await _overloaded_response(state)(scope, receive, send)
return
else:
state.record_queue()
try:
await asyncio.wait_for(
semaphore.acquire(),
timeout=settings.queue_timeout_seconds,
)
except asyncio.TimeoutError:
state.record_dequeue()
state.record_rejection("queue_timeout")
await _overloaded_response(state)(scope, receive, send)
return
except asyncio.CancelledError:
state.record_dequeue()
raise
state.record_dequeue()
state.record_admission()
try:
await self.app(scope, receive, send)
finally:
semaphore.release()
state.record_release()
def _get_route_path(scope: Scope) -> str:
"""Strip the ASGI root_path (SERVER_ROOT_PATH) the same way Starlette does before route matching."""
path: Final[str] = scope["path"]
root_path: Final[str] = scope.get("root_path", "")
if not root_path or not path.startswith(root_path):
return path
if path == root_path:
return ""
if path[len(root_path)] == "/":
return path[len(root_path) :]
return path
def _create_gauge(gauge_type: Callable[..., object], name: str, description: str) -> _Gauge:
metric: Final = (
gauge_type(name, description, multiprocess_mode="livesum")
if "PROMETHEUS_MULTIPROC_DIR" in os.environ
else gauge_type(name, description)
)
if not isinstance(metric, _Gauge):
raise TypeError("Admission gauge has an unexpected type")
return metric
def create_prometheus_admission_metrics() -> AdmissionControlMetrics | None:
try:
from prometheus_client import Counter, Gauge
return AdmissionControlMetrics(
admitted_gauge=_create_gauge(
Gauge,
"litellm_admission_admitted_requests",
"Number of requests admitted by this worker",
),
queued_gauge=_create_gauge(
Gauge,
"litellm_admission_queued_requests",
"Number of requests queued by this worker",
),
rejected_counter=Counter( # mutable-ok: Prometheus requires runtime Counter construction
"litellm_admission_rejected_requests_total",
"Number of requests rejected by this worker",
labelnames=("reason",),
),
)
except (ImportError, ValueError):
return None
admission_control_state: Final = AdmissionControlState(create_prometheus_admission_metrics)
def get_admission_control_stats() -> AdmissionControlStats:
return admission_control_state.get_stats()
_PositiveInt: TypeAlias = Annotated[int, Field(gt=0)]
_NonNegativeInt: TypeAlias = Annotated[int, Field(ge=0)]
_PositiveFloat: TypeAlias = Annotated[float, Field(gt=0)]
_AdmissionControlRaw: TypeAlias = int | float | str | None
def _hashable(value: object) -> _AdmissionControlRaw:
return value if value is None or isinstance(value, (int, float, str)) else repr(value)
_POSITIVE_INT_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(_PositiveInt)
_NON_NEGATIVE_INT_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(_NonNegativeInt)
_POSITIVE_FLOAT_ADAPTER: Final[TypeAdapter[float]] = TypeAdapter(_PositiveFloat)
@lru_cache(maxsize=16)
def _parse_admission_control_settings(
max_in_flight_raw: _AdmissionControlRaw,
max_queued_raw: _AdmissionControlRaw,
queue_timeout_raw: _AdmissionControlRaw,
) -> AdmissionControlSettings | None:
try:
max_in_flight: Final = _POSITIVE_INT_ADAPTER.validate_python(max_in_flight_raw)
max_queued: Final = (
max_in_flight if max_queued_raw is None else _NON_NEGATIVE_INT_ADAPTER.validate_python(max_queued_raw)
)
queue_timeout: Final = _POSITIVE_FLOAT_ADAPTER.validate_python(queue_timeout_raw)
except ValidationError as exc:
verbose_proxy_logger.error(
"Ignoring invalid admission control settings, per-worker admission control is disabled: %s",
exc,
)
return None
return AdmissionControlSettings(
max_in_flight_requests=max_in_flight,
max_queued_requests=max_queued,
queue_timeout_seconds=queue_timeout,
)
def get_admission_control_settings(settings: Mapping[str, object]) -> AdmissionControlSettings | None:
max_in_flight_raw: Final = settings.get("max_in_flight_requests_per_worker")
if max_in_flight_raw is None:
return None
return _parse_admission_control_settings(
_hashable(max_in_flight_raw),
_hashable(settings.get("max_queued_requests_per_worker")),
_hashable(settings.get("admission_queue_timeout_seconds", 1.0)),
)
def _overloaded_response(state: AdmissionControlState) -> JSONResponse:
stats: Final = state.get_stats()
return JSONResponse(
status_code=503,
headers={"retry-after": "1"}, # mutable-ok: Starlette expects a plain headers mapping
content={ # mutable-ok: Starlette serializes a plain response mapping
"error": { # mutable-ok: nested response mapping
"message": (
f"Worker at capacity: {stats.admitted} in-flight, {stats.queued} queued requests. Retry later."
),
"type": "overloaded_error",
"code": "503",
}
},
)

179
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -9,10 +9,11 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc.
from __future__ import annotations
import hmac
import inspect
import json
import os
import re
from collections.abc import Callable, Mapping
from collections.abc import AsyncGenerator, Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, cast
@ -32,6 +33,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.passthrough.main import AsyncPassthroughStreamingResponse
from litellm.proxy._types import *
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.route_checks import RouteChecks
@ -40,6 +42,7 @@ from litellm.proxy.auth.user_api_key_auth import (
user_api_key_auth,
user_api_key_auth_websocket,
)
from litellm.proxy.common_request_processing import open_sse_before_first_byte
from litellm.proxy.common_utils.http_parsing_utils import (
_read_request_body,
_safe_get_request_headers,
@ -47,6 +50,9 @@ from litellm.proxy.common_utils.http_parsing_utils import (
get_form_data,
get_request_body,
)
from litellm.proxy.common_utils.sse_keepalive import (
wrap_passthrough_sse_bytes_with_keepalive_pings,
)
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
HttpPassThroughEndpointHelpers,
@ -1478,6 +1484,74 @@ def is_azure_ai_search_service_level_index_create(method: str, endpoint: str) ->
return path == "indexes" or path.endswith("/indexes")
async def _relay_upstream_bytes(upstream: AsyncGenerator[bytes, bytes]) -> AsyncGenerator[bytes, None]:
try:
async for chunk in upstream:
yield chunk
finally:
await upstream.aclose()
async def _relay_azure_router_model(
llm_router: litellm.Router,
model: str,
endpoint: str,
request: Request,
request_body: Mapping[str, object],
is_streaming_request: bool,
user_api_key_dict: UserAPIKeyAuth,
) -> Response:
result: Final = await llm_router.allm_passthrough_route(
model=model,
method=request.method,
endpoint=endpoint,
request_query_params=request.query_params,
request_headers=_safe_get_request_headers(request),
stream=is_streaming_request,
content=None,
data=None,
files=None,
json=(request_body if request.headers.get("content-type") == "application/json" else None),
params=None,
headers=None,
cookies=None,
litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict),
)
if not is_streaming_request:
upstream: Final = cast(httpx.Response, result)
return Response(
content=await upstream.aread(),
status_code=upstream.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(headers=upstream.headers, custom_headers=None),
)
if inspect.isasyncgen(result):
sse_headers: Final = {"content-type": "text/event-stream"}
return StreamingResponse(
content=wrap_passthrough_sse_bytes_with_keepalive_pings(
stream=_relay_upstream_bytes(result),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
upstream_headers=sse_headers,
),
status_code=200,
headers=sse_headers,
)
upstream_stream: Final = cast(AsyncPassthroughStreamingResponse, result)
return StreamingResponse(
content=wrap_passthrough_sse_bytes_with_keepalive_pings(
stream=_relay_upstream_bytes(upstream_stream),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
upstream_headers=upstream_stream.headers,
),
status_code=upstream_stream.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=upstream_stream.headers, custom_headers=None
),
)
@router.api_route(
"/azure_ai/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
@ -1528,55 +1602,18 @@ async def azure_proxy_route(
if is_router_model:
request_body = await get_request_body(request)
is_streaming_request = is_passthrough_request_streaming(request_body)
result = await llm_router.allm_passthrough_route(
model=part,
method=request.method,
endpoint=endpoint,
request_query_params=request.query_params,
request_headers=_safe_get_request_headers(request),
stream=is_streaming_request,
content=None,
data=None,
files=None,
json=(request_body if request.headers.get("content-type") == "application/json" else None),
params=None,
headers=None,
cookies=None,
litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict),
)
if is_streaming_request:
# Check if result is an async generator (from _async_streaming)
import inspect
if inspect.isasyncgen(result):
# Result is already an async generator, use it directly
return StreamingResponse(
content=result,
status_code=200,
headers={"content-type": "text/event-stream"},
)
else:
# Result is an httpx.Response, use aiter_bytes()
result = cast(httpx.Response, result)
return StreamingResponse(
content=result.aiter_bytes(),
status_code=result.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=result.headers,
custom_headers=None,
),
)
# Non-streaming response
result = cast(httpx.Response, result)
content = await result.aread()
return Response(
content=content,
status_code=result.status_code,
headers=HttpPassThroughEndpointHelpers.get_response_headers(
headers=result.headers,
custom_headers=None,
return await open_sse_before_first_byte(
_relay_azure_router_model(
llm_router=llm_router,
model=part,
endpoint=endpoint,
request=request,
request_body=request_body,
is_streaming_request=is_streaming_request,
user_api_key_dict=user_api_key_dict,
),
ping_interval_seconds=(
litellm.sse_keepalive_ping_interval_seconds if is_streaming_request else None
),
)
elif is_vector_store_index:
@ -1659,6 +1696,12 @@ async def azure_proxy_route(
from abc import ABC, abstractmethod
_VERTEX_LOCATION_REQUIRED_DETAIL: Final = (
"No Vertex AI location for this request. Include /projects/<project>/locations/<location>/ in the "
"route, set vertex_location in default_vertex_config (or DEFAULT_VERTEXAI_LOCATION), or add the "
"model to model_list with use_in_pass_through: true."
)
class BaseVertexAIPassThroughHandler(ABC):
@staticmethod
@ -1666,29 +1709,18 @@ class BaseVertexAIPassThroughHandler(ABC):
def get_default_base_target_url(vertex_location: str | None) -> str:
pass
@staticmethod
@abstractmethod
def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: str | None) -> str:
pass
class VertexAIDiscoveryPassThroughHandler(BaseVertexAIPassThroughHandler):
@staticmethod
def get_default_base_target_url(vertex_location: str | None) -> str:
return "https://discoveryengine.googleapis.com/"
@staticmethod
def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: str | None) -> str:
return base_target_url
class VertexAIPassThroughHandler(BaseVertexAIPassThroughHandler):
@staticmethod
def get_default_base_target_url(vertex_location: str | None) -> str:
return get_vertex_base_url(vertex_location)
@staticmethod
def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: str | None) -> str:
if vertex_location is None:
raise HTTPException(status_code=400, detail=_VERTEX_LOCATION_REQUIRED_DETAIL)
return get_vertex_base_url(vertex_location)
@ -1911,10 +1943,8 @@ async def _prepare_vertex_auth_headers(
router_credentials: LiteLLM_ManagedVectorStore | None,
vertex_project: str | None,
vertex_location: str | None,
base_target_url: str | None,
get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler,
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[Mapping[str, str], str | None, bool, str | None, str | None]:
) -> tuple[Mapping[str, str], bool, str | None, str | None]:
"""
Prepare authentication headers for Vertex AI pass-through requests.
@ -1924,15 +1954,12 @@ async def _prepare_vertex_auth_headers(
router_credentials: Optional vector store credentials from registry
vertex_project: Vertex project ID
vertex_location: Vertex location
base_target_url: Base URL for the Vertex AI service
get_vertex_pass_through_handler: Handler for the specific Vertex AI service
user_api_key_dict: The caller's resolved authentication, so only the secret that
authenticated them is stripped on the credential-less branch
Returns:
tuple containing:
- headers: dict - Authentication headers to use
- base_target_url: str | None - Updated base target URL
- headers_passed_through: bool - Whether headers were passed through from request
- vertex_project: str | None - Updated vertex project ID
- vertex_location: str | None - Updated vertex location
@ -1985,14 +2012,8 @@ async def _prepare_vertex_auth_headers(
# Add the Authorization header with vendor credentials
headers["Authorization"] = f"Bearer {auth_header}"
if base_target_url is not None:
base_target_url = get_vertex_pass_through_handler.update_base_target_url_with_credential_location(
base_target_url, vertex_location
)
return (
headers,
base_target_url,
headers_passed_through,
vertex_project,
vertex_location,
@ -2085,12 +2106,9 @@ async def _base_vertex_proxy_route(
location=vertex_location,
)
base_target_url = get_vertex_pass_through_handler.get_default_base_target_url(vertex_location)
# Prepare authentication headers
(
headers,
base_target_url,
headers_passed_through,
vertex_project,
vertex_location,
@ -2100,13 +2118,10 @@ async def _base_vertex_proxy_route(
router_credentials=router_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
base_target_url=base_target_url,
get_vertex_pass_through_handler=get_vertex_pass_through_handler,
user_api_key_dict=user_api_key_dict,
)
if base_target_url is None:
base_target_url = get_vertex_base_url(vertex_location)
base_target_url: Final = get_vertex_pass_through_handler.get_default_base_target_url(vertex_location)
request_route: Final = encoded_endpoint
verbose_proxy_logger.debug("request_route %s", request_route)

Some files were not shown because too many files have changed in this diff Show more