mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_registry_audit_2026_09_02
This commit is contained in:
commit
93abc3a0cd
134 changed files with 2591 additions and 703 deletions
29
.github/workflows/_test-unit-base.yml
vendored
29
.github/workflows/_test-unit-base.yml
vendored
|
|
@ -57,9 +57,15 @@ permissions:
|
|||
|
||||
jobs:
|
||||
run:
|
||||
name: Run tests
|
||||
name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: ${{ inputs.job-timeout-minutes }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
|
@ -82,7 +88,7 @@ jobs:
|
|||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
|
|
@ -96,12 +102,10 @@ jobs:
|
|||
timeout-minutes: 5
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
path: ${{ env.UV_CACHE_DIR }}
|
||||
key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-
|
||||
|
||||
- name: Cache the Rust build
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
|
|
@ -113,6 +117,7 @@ jobs:
|
|||
timeout-minutes: 8
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
|
||||
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
|
|
@ -134,13 +139,7 @@ jobs:
|
|||
WORKERS: ${{ inputs.workers }}
|
||||
RERUNS: ${{ inputs.reruns }}
|
||||
DIST: ${{ inputs.dist }}
|
||||
# coverage.py's sys.monitoring backend (PEP 669), the cheapest core it has.
|
||||
# It is only the default from Python 3.14, and these shards run 3.12, so it
|
||||
# has to be asked for. Coverage refuses it when branch measurement is on
|
||||
# (`branch_right_left` needs > 3.14.0a5) and falls back to the slow core with
|
||||
# a `no-sysmon` warning, so turning on `branch = true` here means giving this
|
||||
# back until the runners move to 3.14.
|
||||
COVERAGE_CORE: sysmon
|
||||
COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }}
|
||||
run: |
|
||||
if [ "${WORKERS}" = "0" ]; then
|
||||
uv run --no-sync pytest ${TEST_PATH:?} \
|
||||
|
|
@ -167,7 +166,7 @@ jobs:
|
|||
fi
|
||||
|
||||
- name: Save coverage report
|
||||
if: always() && steps.changes.outputs.decision != 'skip'
|
||||
if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip'
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"limit": 40
|
||||
},
|
||||
"reportDeprecated": {
|
||||
"limit": 211
|
||||
"limit": 209
|
||||
},
|
||||
"reportDuplicateImport": {
|
||||
"limit": 19
|
||||
|
|
@ -45,7 +45,7 @@
|
|||
"limit": 24
|
||||
},
|
||||
"reportInvalidTypeForm": {
|
||||
"limit": 34
|
||||
"limit": 30
|
||||
},
|
||||
"reportInvalidTypeVarUse": {
|
||||
"limit": 2
|
||||
|
|
@ -105,13 +105,13 @@
|
|||
"limit": 109
|
||||
},
|
||||
"reportUnknownMemberType": {
|
||||
"limit": 38311
|
||||
"limit": 38309
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 19624
|
||||
"limit": 19622
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 29847
|
||||
"limit": 29846
|
||||
},
|
||||
"reportUnnecessaryCast": {
|
||||
"limit": 111
|
||||
|
|
|
|||
|
|
@ -161,6 +161,7 @@ class CheckBatchCost:
|
|||
metadata: dict[str, object] = {
|
||||
"user_api_key_user_id": job.created_by,
|
||||
"user_api_key": api_key,
|
||||
"user_api_key_hash": api_key,
|
||||
"user_api_key_team_id": team_id,
|
||||
**(await self._get_user_info(batch_id, job.created_by)),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ datasource client {
|
|||
|
||||
generator client {
|
||||
provider = "prisma-client-py"
|
||||
recursive_type_depth = -1
|
||||
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,18 @@
|
|||
# AGENTS.md
|
||||
|
||||
litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
|
||||
litellm-rust has five crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
|
||||
|
||||
## Crates
|
||||
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
|
||||
| litellm-config | Config-loading boundary. Returns resolved core deployment data and optionally delegates loading to Python. |
|
||||
| litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
|
||||
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
|
||||
|
||||
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
|
||||
Dependency direction is acyclic: `litellm-config` depends on `litellm-core`, the gateway depends on both, and `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`. The interop foundation depends on no LiteLLM domain crate.
|
||||
|
||||
## Where a route lives
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ the base when behavior is genuinely different, and say so explicitly in the PR.
|
|||
## Crates (see AGENTS.md)
|
||||
|
||||
`litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
|
||||
`litellm-config` is the config-loading boundary and returns resolved core types.
|
||||
`litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
|
||||
`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop`
|
||||
holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate
|
||||
|
|
|
|||
12
litellm-rust/Cargo.lock
generated
12
litellm-rust/Cargo.lock
generated
|
|
@ -1412,8 +1412,8 @@ dependencies = [
|
|||
"base64",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"litellm-config",
|
||||
"litellm-core",
|
||||
"pyo3",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -1425,6 +1425,16 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-config"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-core",
|
||||
"pyo3",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-core"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
[workspace]
|
||||
members = [
|
||||
"crates/core",
|
||||
"crates/config",
|
||||
"crates/ai-gateway",
|
||||
"crates/python-interop",
|
||||
"crates/python-bridge",
|
||||
|
|
@ -17,6 +18,7 @@ repository = "https://github.com/BerriAI/litellm"
|
|||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
|
||||
litellm-core = { path = "crates/core" }
|
||||
litellm-config = { path = "crates/config" }
|
||||
litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
|
||||
litellm-python-interop = { path = "crates/python-interop" }
|
||||
axum = "0.7"
|
||||
|
|
|
|||
|
|
@ -25,11 +25,12 @@ coverage and production evidence.
|
|||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. |
|
||||
| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
|
||||
| litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
|
||||
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
|
||||
|
||||
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
|
||||
Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
|
||||
|
||||
## Layout
|
||||
|
||||
|
|
@ -38,6 +39,7 @@ crates/
|
|||
core/ The SDK: route modules + provider transforms.
|
||||
src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client
|
||||
src/providers/anthropic/messages/transformation.rs
|
||||
config/ Config loading and resolved deployments.
|
||||
ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints.
|
||||
python-interop/ Domain-neutral PyO3 conversion and GIL primitives.
|
||||
python-bridge/ PyO3 API adapter for Python LiteLLM.
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
|
|||
|
||||
22. A Python -> Rust bridge keeps the Python side minimal: the Python interface only marshals inputs and calls the Rust interface, with no transform, handler, or business logic. Aim for well under 100 lines of interface code per route; if the Python grows past that, the logic belongs in Rust.
|
||||
23. Do not bloat `litellm/main.py`. A route's provider dispatch lives in a thin dispatch class under `litellm/llms/<provider>/<route>/` that calls the Rust bridge; `main.py` only instantiates it and calls its sync/async method.
|
||||
24. Do not add new feature flags unless explicitly requested. Reuse the existing litellm rust rollout mechanism (`use_litellm_rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
|
||||
24. Do not add new feature flags unless explicitly requested. Reuse the existing LiteLLM Rust rollout mechanism (`litellm.rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
|
||||
|
||||
## Checks before push
|
||||
|
||||
|
|
|
|||
|
|
@ -9,19 +9,15 @@ such as `litellm_core::messages::messages`. No provider handler lives here.
|
|||
src/
|
||||
main.rs # entrypoint: build AppState (router + master key), bind, serve
|
||||
state.rs # AppState — shared Arc<Router> + master_key
|
||||
gil.rs # GIL-activity tracker (records Python acquisitions)
|
||||
auth/ # authentication as an axum extractor — added to handler args
|
||||
mod.rs # RequireMasterKey: FromRequestParts, single master key (LITELLM_MASTER_KEY)
|
||||
routes/ # one module per route, all matching the same template
|
||||
AGENTS.md # ← the route template (read this before adding a route)
|
||||
mod.rs # app(): merges every module's router()
|
||||
health.rs # simple route (one file): router() + liveness/readiness
|
||||
gil.rs # simple route (one file): router() + GET /health/gil
|
||||
realtime/ # route with logic → axum surface + a no-axum service:
|
||||
mod.rs # router() + handler + WS<->events adapter (the axum surface)
|
||||
service.rs # business logic (select deployment, call provider) — no axum, testable
|
||||
python/ # Python interop (feature: python-config) — load-time only
|
||||
mod.rs, config.rs, AGENTS.md
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
|
@ -53,5 +49,6 @@ proxy in a later phase. Health routes don't add the extractor (unauthenticated).
|
|||
|
||||
## Python interop
|
||||
|
||||
Anything that calls into Python lives in `python/` and is **load-time only** — see
|
||||
`python/AGENTS.md`. The realtime data path never takes the GIL.
|
||||
Python-backed loading lives in `litellm-config` and is **load-time only**. The
|
||||
gateway's `python-config` feature forwards to that crate. The realtime data path
|
||||
never takes the GIL.
|
||||
|
|
|
|||
|
|
@ -9,4 +9,6 @@ flowchart LR
|
|||
C[client] <--> G[Rust ai-gateway<br/>LLM inference]
|
||||
G <--> O[OpenAI realtime]
|
||||
G -. spend tracking callback .-> P[litellm proxy]
|
||||
F[litellm-config<br/>load-time only] --> G
|
||||
F -. Python backend .-> P
|
||||
```
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ required-features = ["server"]
|
|||
[dependencies]
|
||||
tracing.workspace = true
|
||||
litellm-core = { workspace = true, features = ["bedrock-auth"] }
|
||||
litellm-config.workspace = true
|
||||
# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
|
||||
# Python proxy callbacks API.
|
||||
reqwest.workspace = true
|
||||
|
|
@ -31,7 +32,6 @@ subtle = { workspace = true, optional = true }
|
|||
# sha2 hashes the master key into user_api_key_hash (matches the proxy's
|
||||
# SHA-256 hash_token) so the plaintext credential never enters a log payload.
|
||||
sha2 = { workspace = true, optional = true }
|
||||
pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
|
||||
tower = { version = "0.5.3", features = ["util"], optional = true }
|
||||
|
||||
[features]
|
||||
|
|
@ -39,7 +39,7 @@ default = []
|
|||
server = ["dep:axum", "dep:subtle", "dep:sha2"]
|
||||
# Build the gateway's config from the proxy YAML via an embedded Python
|
||||
# interpreter (links libpython; requires `litellm` importable at runtime).
|
||||
python-config = ["dep:pyo3"]
|
||||
python-config = ["litellm-config/python"]
|
||||
trace-parity = ["server", "dep:tower", "litellm-core/observability"]
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -6,25 +6,30 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame.
|
|||
|
||||
## Crates
|
||||
|
||||
`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route:
|
||||
`litellm-rust` has five crates. A crate is a layer or shared foundation, not a route:
|
||||
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. |
|
||||
| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. |
|
||||
| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
|
||||
| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
|
||||
| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. |
|
||||
|
||||
Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
|
||||
Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers and Python interop.
|
||||
|
||||
- **Client endpoint:** `wss://<host>/v1/realtime?model=<model>` (WebSocket)
|
||||
- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset)
|
||||
- **Health:** `GET /health/readiness`, `GET /health/liveness`, `GET /health/gil`
|
||||
- **Health:** `GET /health/readiness`, `GET /health/liveness`
|
||||
- **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging))
|
||||
|
||||
> **Realtime serving is pure Rust.** Python is used at **load time only** — to
|
||||
> read the config once at boot. The realtime hot path never touches Python.
|
||||
|
||||
The former `/health/gil` route and its acquisition counter were removed. They
|
||||
only observed the single startup config load and did not prove that every GIL
|
||||
acquisition was instrumented
|
||||
|
||||
## Configuration (config.yaml)
|
||||
|
||||
The gateway loads its `model_list` from a **config.yaml**, the same as the
|
||||
|
|
@ -43,9 +48,10 @@ model_list:
|
|||
LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway
|
||||
```
|
||||
|
||||
At boot the gateway calls into `litellm.proxy.read_model_list`, which reuses the
|
||||
**real proxy config reader** (`ProxyConfig.get_config`). That means everything
|
||||
the proxy supports in config.yaml works here too:
|
||||
At boot `litellm-config` calls into `litellm.proxy.read_model_list` and returns
|
||||
resolved deployments to the gateway, which constructs the router. The Python
|
||||
backend still reuses the **real proxy config reader** (`ProxyConfig.get_config`),
|
||||
so everything the proxy supports in config.yaml works here too:
|
||||
|
||||
- `include:` to merge in other config files,
|
||||
- `os.environ/VAR` secret references (resolved via the secret manager, never
|
||||
|
|
@ -82,8 +88,8 @@ stand-in built from the environment:
|
|||
|---|---|---|
|
||||
| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). |
|
||||
|
||||
This mode links no libpython and needs no config file, but it only supports one
|
||||
hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
|
||||
The default workspace build links no libpython and needs no config file. This
|
||||
fallback mode only supports one hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the
|
||||
stand-in only for the leanest possible build.
|
||||
|
||||
## Request logging
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# Sample realtime config for the LiteLLM Rust AI Gateway.
|
||||
#
|
||||
# The gateway loads this model_list at boot via the embedded python config
|
||||
# reader (litellm.proxy.read_model_list), which reuses the proxy's own reader —
|
||||
# so include:, os.environ/ secrets, and DB-stored models all work here too.
|
||||
# litellm-config resolves this model_list at boot through the Python config
|
||||
# reader (litellm.proxy.read_model_list), then the gateway builds its router.
|
||||
# Includes, environment secrets, and database-stored models still work.
|
||||
#
|
||||
# Secrets are referenced (never inlined) via os.environ/. A real deploy can
|
||||
# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH).
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
//! GIL-activity tracking.
|
||||
//!
|
||||
//! Every acquisition of the Python GIL is recorded here so the `/health/gil`
|
||||
//! endpoint can report whether Python was touched recently. The design goal is
|
||||
//! that the GIL is acquired **only at load time** (config read) and never on the
|
||||
//! realtime hot path — polling this endpoint during traffic should show the
|
||||
//! count holding steady and `acquired_last_30s` falling to `false`.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// Window (seconds) for the "recently acquired" signal.
|
||||
pub const RECENT_WINDOW_SECS: u64 = 30;
|
||||
|
||||
static GIL_ACQUISITIONS: AtomicU64 = AtomicU64::new(0);
|
||||
/// Unix seconds of the last acquisition; `0` means "never".
|
||||
static LAST_GIL_UNIX_SECS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn now_unix_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Record that the GIL was just acquired. Call immediately before taking the GIL.
|
||||
///
|
||||
/// Only invoked under the `python-config` feature; without it the gateway never
|
||||
/// touches Python, so the recorder is unused (and the endpoint reports zero).
|
||||
#[cfg_attr(not(feature = "python-config"), allow(dead_code))]
|
||||
pub fn record_acquisition() {
|
||||
GIL_ACQUISITIONS.fetch_add(1, Ordering::Relaxed);
|
||||
LAST_GIL_UNIX_SECS.store(now_unix_secs(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Point-in-time view of GIL activity.
|
||||
pub struct GilSnapshot {
|
||||
pub total_acquisitions: u64,
|
||||
pub seconds_since_last: Option<u64>,
|
||||
pub acquired_last_30s: bool,
|
||||
}
|
||||
|
||||
/// Read the current GIL-activity snapshot.
|
||||
pub fn snapshot() -> GilSnapshot {
|
||||
let total = GIL_ACQUISITIONS.load(Ordering::Relaxed);
|
||||
let last = LAST_GIL_UNIX_SECS.load(Ordering::Relaxed);
|
||||
let seconds_since_last = if last == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(now_unix_secs().saturating_sub(last))
|
||||
};
|
||||
let acquired_last_30s = seconds_since_last.is_some_and(|secs| secs <= RECENT_WINDOW_SECS);
|
||||
GilSnapshot {
|
||||
total_acquisitions: total,
|
||||
seconds_since_last,
|
||||
acquired_last_30s,
|
||||
}
|
||||
}
|
||||
|
|
@ -10,18 +10,13 @@
|
|||
//! - [`io`]: compatibility exports and realtime WebSocket splice helpers.
|
||||
//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling
|
||||
//! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway`
|
||||
//! binary turns on. The `python-config` feature additionally pulls in [`python`]
|
||||
//! for the load-time config reader.
|
||||
//! binary turns on.
|
||||
|
||||
pub mod audio_transcription;
|
||||
mod client;
|
||||
pub mod io;
|
||||
pub mod ocr;
|
||||
|
||||
/// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
|
||||
/// the `python-config` reader, so it is available without either feature.
|
||||
pub mod gil;
|
||||
|
||||
#[cfg(feature = "server")]
|
||||
pub mod auth;
|
||||
#[cfg(feature = "server")]
|
||||
|
|
@ -35,6 +30,3 @@ mod constants;
|
|||
pub mod integrations;
|
||||
#[cfg(feature = "server")]
|
||||
mod realtime;
|
||||
|
||||
#[cfg(feature = "python-config")]
|
||||
pub mod python;
|
||||
|
|
|
|||
|
|
@ -14,12 +14,12 @@ use std::sync::Arc;
|
|||
use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key};
|
||||
use litellm_ai_gateway::routes;
|
||||
use litellm_ai_gateway::state::AppState;
|
||||
#[cfg(feature = "python-config")]
|
||||
use litellm_config::load_model_list;
|
||||
use litellm_core::router::{Deployment, LiteLLMParams, Router};
|
||||
|
||||
use litellm_ai_gateway::integrations::custom_logger::CustomLogger;
|
||||
use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger;
|
||||
#[cfg(feature = "python-config")]
|
||||
use litellm_ai_gateway::python;
|
||||
|
||||
/// Bind to localhost by default so the gateway is not a public, unauthenticated
|
||||
/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`).
|
||||
|
|
@ -124,10 +124,10 @@ fn resolve_port() -> u16 {
|
|||
fn build_router() -> Router {
|
||||
#[cfg(feature = "python-config")]
|
||||
if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") {
|
||||
match python::config::load_router_from_config(&config_path) {
|
||||
Ok(router) => {
|
||||
match load_model_list(std::path::Path::new(&config_path)) {
|
||||
Ok(deployments) => {
|
||||
eprintln!("loaded model_list from {config_path} via python config reader");
|
||||
return router;
|
||||
return Router::new(deployments);
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("config load failed ({err}); falling back to env deployment");
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
# ai-gateway/src/python — Python interop (load-time only)
|
||||
|
||||
Functions here embed the Python interpreter (pyo3) and take the GIL to call into
|
||||
`litellm` (e.g. read the proxy `model_list`). Compiled only under the
|
||||
`python-config` feature.
|
||||
|
||||
## Hard rule: non-hot-path functions only
|
||||
|
||||
Everything in this folder MUST run **at most once per process lifetime — at
|
||||
startup / load time** (config read, warm-up). NEVER call into Python on the
|
||||
request path:
|
||||
|
||||
- No GIL acquisition per request, per connection, or per realtime event.
|
||||
- No Python call inside a route handler, the router's hot path, or any loop that
|
||||
scales with traffic.
|
||||
|
||||
**Why:** the GIL serializes execution and would cap throughput; the realtime data
|
||||
path must stay pure Rust. Every acquisition is recorded by `crate::gil` — poll
|
||||
`GET /health/gil`, and `total_acquisitions` MUST stay flat under load.
|
||||
|
||||
## How to add one
|
||||
|
||||
Resolve whatever Python-derived data you need **once at boot** and hand the rest
|
||||
of the gateway an owned, plain-Rust value (e.g. build a `Router` from the
|
||||
resolved `model_list`). Record the acquisition via `crate::gil::record_acquisition()`
|
||||
immediately before taking the GIL. If a function would need to run per request,
|
||||
it does not belong here — move the work to Rust, or pre-resolve it at startup.
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
//! Build the router by calling the Python proxy config reader (load time only).
|
||||
//!
|
||||
//! Embeds the interpreter via pyo3 and calls
|
||||
//! `litellm.proxy.read_model_list.read_model_list`, which reuses the proxy's
|
||||
//! `os.environ/` + secret-manager resolution. The GIL is taken **once at boot**
|
||||
//! (and recorded in [`crate::gil`]); the realtime hot path never touches Python.
|
||||
//!
|
||||
//! Compiled only under the `python-config` feature.
|
||||
use litellm_core::error::Error;
|
||||
use litellm_core::router::{Deployment, Router};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::gil;
|
||||
|
||||
/// Load the router's `model_list` from `config_path` via the Python reader.
|
||||
pub fn load_router_from_config(config_path: &str) -> Result<Router, Error> {
|
||||
gil::record_acquisition();
|
||||
Python::attach(|py| {
|
||||
let model_list = py
|
||||
.import("litellm.proxy.read_model_list")
|
||||
.and_then(|module| module.getattr("read_model_list"))
|
||||
.and_then(|reader| reader.call1((config_path,)))
|
||||
.map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?;
|
||||
|
||||
let model_list_json: String = py
|
||||
.import("json")
|
||||
.and_then(|json| json.getattr("dumps"))
|
||||
.and_then(|dumps| dumps.call1((model_list,)))
|
||||
.and_then(|encoded| encoded.extract())
|
||||
.map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?;
|
||||
|
||||
let deployments: Vec<Deployment> = serde_json::from_str(&model_list_json)
|
||||
.map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?;
|
||||
|
||||
Ok(Router::new(deployments))
|
||||
})
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
//! Python interop for the gateway. See `AGENTS.md`: **load-time / non-hot-path
|
||||
//! only.** Compiled only under the `python-config` feature.
|
||||
|
||||
pub mod config;
|
||||
|
|
@ -13,7 +13,7 @@ private). This is the norm — don't split until it hurts.
|
|||
pub fn router() -> Router<AppState> { Router::new().route(PATH, get(handle)) }
|
||||
async fn handle(...) -> impl IntoResponse { ... }
|
||||
```
|
||||
`health.rs` and `gil.rs` are examples.
|
||||
`health.rs` is the example.
|
||||
|
||||
## Split out `service` when there's real logic
|
||||
When a route has business logic worth testing without axum, put it in a sibling
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
//! `GET /health/gil` — poll to confirm Python is only touched at load time.
|
||||
//! Simple-route template: a `router()` plus its handler, in one file.
|
||||
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::gil;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// This route's contribution to the app router.
|
||||
pub fn router() -> Router<AppState> {
|
||||
Router::new().route("/health/gil", get(status))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct GilStatusResponse {
|
||||
gil_acquired_last_30s: bool,
|
||||
total_acquisitions: u64,
|
||||
seconds_since_last: Option<u64>,
|
||||
}
|
||||
|
||||
async fn status() -> Json<GilStatusResponse> {
|
||||
let snapshot = gil::snapshot();
|
||||
Json(GilStatusResponse {
|
||||
gil_acquired_last_30s: snapshot.acquired_last_30s,
|
||||
total_acquisitions: snapshot.total_acquisitions,
|
||||
seconds_since_last: snapshot.seconds_since_last,
|
||||
})
|
||||
}
|
||||
|
|
@ -2,10 +2,9 @@
|
|||
//!
|
||||
//! **Template:** every route module exposes `pub fn router() -> Router<AppState>`
|
||||
//! that mounts its own paths; [`app`] merges them. A trivial route is a single
|
||||
//! file (`health.rs`, `gil.rs`); a non-trivial one is a folder (`realtime/`) with
|
||||
//! file (`health.rs`); a non-trivial one is a folder (`realtime/`) with
|
||||
//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md.
|
||||
|
||||
pub mod gil;
|
||||
pub mod health;
|
||||
pub mod messages;
|
||||
pub mod realtime;
|
||||
|
|
@ -19,7 +18,6 @@ use crate::state::AppState;
|
|||
pub fn app(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.merge(health::router())
|
||||
.merge(gil::router())
|
||||
.merge(messages::router())
|
||||
.merge(realtime::router())
|
||||
.merge(responses::router())
|
||||
|
|
|
|||
16
litellm-rust/crates/config/Cargo.toml
Normal file
16
litellm-rust/crates/config/Cargo.toml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
[package]
|
||||
name = "litellm-config"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-core.workspace = true
|
||||
pyo3 = { workspace = true, features = ["auto-initialize"], optional = true }
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
python = ["dep:pyo3"]
|
||||
11
litellm-rust/crates/config/src/error.rs
Normal file
11
litellm-rust/crates/config/src/error.rs
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
use thiserror::Error as ThisError;
|
||||
|
||||
#[derive(Debug, ThisError)]
|
||||
pub enum Error {
|
||||
#[error("read_model_list failed: {0}")]
|
||||
PythonLoading(String),
|
||||
#[error("serializing model_list failed: {0}")]
|
||||
Serialization(String),
|
||||
#[error("parsing model_list failed: {0}")]
|
||||
ModelListParsing(#[source] serde_json::Error),
|
||||
}
|
||||
7
litellm-rust/crates/config/src/lib.rs
Normal file
7
litellm-rust/crates/config/src/lib.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
mod error;
|
||||
#[cfg(feature = "python")]
|
||||
mod python;
|
||||
|
||||
pub use error::Error;
|
||||
#[cfg(feature = "python")]
|
||||
pub use python::load_model_list;
|
||||
76
litellm-rust/crates/config/src/python.rs
Normal file
76
litellm-rust/crates/config/src/python.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
use std::path::Path;
|
||||
|
||||
use litellm_core::router::Deployment;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
pub fn load_model_list(config_path: &Path) -> Result<Vec<Deployment>, Error> {
|
||||
Python::attach(|python| {
|
||||
let model_list = python
|
||||
.import("litellm.proxy.read_model_list")
|
||||
.and_then(|module| module.getattr("read_model_list"))
|
||||
.and_then(|reader| reader.call1((config_path.to_string_lossy().as_ref(),)))
|
||||
.map_err(|error| Error::PythonLoading(error.to_string()))?;
|
||||
|
||||
let model_list_json = python
|
||||
.import("json")
|
||||
.and_then(|json| json.getattr("dumps"))
|
||||
.and_then(|dumps| dumps.call1((model_list,)))
|
||||
.and_then(|encoded| encoded.extract::<String>())
|
||||
.map_err(|error| Error::Serialization(error.to_string()))?;
|
||||
|
||||
parse_model_list(&model_list_json)
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_model_list(model_list_json: &str) -> Result<Vec<Deployment>, Error> {
|
||||
serde_json::from_str(model_list_json).map_err(Error::ModelListParsing)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_resolved_model_list() {
|
||||
let deployments = parse_model_list(
|
||||
r#"[
|
||||
{
|
||||
"model_name": "realtime",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-realtime",
|
||||
"api_key": "resolved-secret",
|
||||
"api_base": "https://api.example.test/v1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model_name": "without-optional-values",
|
||||
"litellm_params": {"model": "openai/gpt-4.1"}
|
||||
}
|
||||
]"#,
|
||||
)
|
||||
.expect("resolved model list should parse");
|
||||
|
||||
assert_eq!(deployments.len(), 2);
|
||||
assert_eq!(deployments[0].model_name, "realtime");
|
||||
assert_eq!(
|
||||
deployments[0].litellm_params.api_key.as_deref(),
|
||||
Some("resolved-secret")
|
||||
);
|
||||
assert_eq!(
|
||||
deployments[0].litellm_params.api_base.as_deref(),
|
||||
Some("https://api.example.test/v1")
|
||||
);
|
||||
assert_eq!(deployments[1].litellm_params.api_key, None);
|
||||
assert_eq!(deployments[1].litellm_params.api_base, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_model_list_returns_parsing_error() {
|
||||
let error = parse_model_list(r#"[{"model_name":"missing-params"}]"#)
|
||||
.expect_err("missing litellm_params should fail");
|
||||
|
||||
assert!(matches!(error, Error::ModelListParsing(_)));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
//! Enforcement: the litellm-rust workspace has exactly four crates.
|
||||
//! Enforcement: the litellm-rust workspace has exactly five crates.
|
||||
//!
|
||||
//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host),
|
||||
//! `core` (the Rust SDK), `config` (the config-loading boundary),
|
||||
//! `ai-gateway` (the HTTP/WebSocket host),
|
||||
//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the
|
||||
//! PyO3 cdylib). Adding or removing a crate must be a
|
||||
//! deliberate act: this test fails until the allowlist here is updated, forcing
|
||||
|
|
@ -19,13 +20,20 @@ use std::path::{Path, PathBuf};
|
|||
/// workspace legitimately gains or loses a crate.
|
||||
const EXPECTED_MEMBERS: &[&str] = &[
|
||||
"crates/core",
|
||||
"crates/config",
|
||||
"crates/ai-gateway",
|
||||
"crates/python-interop",
|
||||
"crates/python-bridge",
|
||||
];
|
||||
|
||||
/// The crate subdirectory names that must exist under `crates/`.
|
||||
const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"];
|
||||
const EXPECTED_CRATE_DIRS: &[&str] = &[
|
||||
"core",
|
||||
"config",
|
||||
"ai-gateway",
|
||||
"python-interop",
|
||||
"python-bridge",
|
||||
];
|
||||
|
||||
const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact).";
|
||||
|
||||
|
|
|
|||
|
|
@ -1421,7 +1421,7 @@ from .skills.main import (
|
|||
)
|
||||
from .containers.main import *
|
||||
from .ocr.main import *
|
||||
from .rust_bridge import use_litellm_rust
|
||||
from .rust_bridge import rust
|
||||
from .rag.main import *
|
||||
from .sandbox.main import *
|
||||
from .search.main import *
|
||||
|
|
|
|||
|
|
@ -94,8 +94,18 @@ class LiteLLMDatabase:
|
|||
|
||||
try:
|
||||
db_response: Final = await client.db.query_raw(query, *params)
|
||||
# Convert the response to polars DataFrame with full schema inference
|
||||
# This prevents schema mismatch errors when data types vary across rows
|
||||
return pl.DataFrame(db_response, infer_schema_length=None)
|
||||
from litellm.proxy.spend_tracking.key_metadata_recovery import (
|
||||
fill_missing_api_key_aliases,
|
||||
)
|
||||
|
||||
usage_rows: Final = (
|
||||
db_response.to_dicts()
|
||||
if isinstance(db_response, pl.DataFrame)
|
||||
else db_response
|
||||
if isinstance(db_response, list)
|
||||
else []
|
||||
)
|
||||
recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
|
||||
return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
|
||||
except Exception as e:
|
||||
raise Exception(f"Error retrieving usage data: {e}")
|
||||
|
|
|
|||
|
|
@ -96,7 +96,19 @@ class FocusLiteLLMDatabase:
|
|||
|
||||
try:
|
||||
db_response: Final = await client.db.query_raw(query, *query_params)
|
||||
return pl.DataFrame(db_response, infer_schema_length=None)
|
||||
from litellm.proxy.spend_tracking.key_metadata_recovery import (
|
||||
fill_missing_api_key_aliases,
|
||||
)
|
||||
|
||||
usage_rows: Final = (
|
||||
db_response.to_dicts()
|
||||
if isinstance(db_response, pl.DataFrame)
|
||||
else db_response
|
||||
if isinstance(db_response, list)
|
||||
else []
|
||||
)
|
||||
recovered_rows: Final = await fill_missing_api_key_aliases(client, usage_rows)
|
||||
return pl.DataFrame([dict(row) for row in recovered_rows], infer_schema_length=None)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Error retrieving usage data: {exc}") from exc
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Base OCR transformation configuration.
|
||||
"""
|
||||
|
||||
import builtins
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
|
|
@ -93,8 +94,8 @@ class OCRResponse(LiteLLMPydanticObjectBase):
|
|||
document_annotation: Any | None = None
|
||||
usage_info: OCRUsageInfo | None = None
|
||||
content: str | None = None
|
||||
tables: list[dict[str, object]] | None = None
|
||||
keyValuePairs: list[dict[str, object]] | None = None
|
||||
tables: list[dict[str, builtins.object]] | None = None
|
||||
keyValuePairs: list[dict[str, builtins.object]] | None = None
|
||||
object: str = "ocr"
|
||||
|
||||
model_config = {"extra": "allow"}
|
||||
|
|
@ -102,11 +103,11 @@ class OCRResponse(LiteLLMPydanticObjectBase):
|
|||
# Define private attributes using PrivateAttr
|
||||
_hidden_params: dict = PrivateAttr(default_factory=dict)
|
||||
|
||||
def set_provider_native_response(self, native_response: Mapping[str, object]) -> None:
|
||||
def set_provider_native_response(self, native_response: Mapping[str, builtins.object]) -> None:
|
||||
"""Keep the provider's own response payload alongside the normalized one."""
|
||||
self._hidden_params[PROVIDER_NATIVE_RESPONSE_KEY] = native_response
|
||||
|
||||
def get_provider_native_response(self) -> Mapping[str, object] | None:
|
||||
def get_provider_native_response(self) -> Mapping[str, builtins.object] | None:
|
||||
"""The provider's own response payload, when `req_format=native` was requested."""
|
||||
native_response: Final = self._hidden_params.get(PROVIDER_NATIVE_RESPONSE_KEY)
|
||||
return native_response if isinstance(native_response, dict) else None
|
||||
|
|
|
|||
|
|
@ -3151,6 +3151,17 @@
|
|||
}
|
||||
],
|
||||
"title": "Team Id"
|
||||
},
|
||||
"user_email": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "User Email"
|
||||
}
|
||||
},
|
||||
"title": "KeyMetadata",
|
||||
|
|
|
|||
|
|
@ -670,6 +670,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/team/permissions_bulk_update",
|
||||
"/team/daily/activity",
|
||||
"/team/daily/activity/aggregated",
|
||||
"/team/spend/by_user",
|
||||
# gateway request counts (SGR); deployment-wide, admin-only
|
||||
"/gateway/daily/activity",
|
||||
# model
|
||||
|
|
@ -832,6 +833,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/team/permissions_update",
|
||||
"/team/daily/activity",
|
||||
"/team/daily/activity/aggregated",
|
||||
"/team/spend/by_user",
|
||||
"/team/{team_id}/members/me",
|
||||
"/model/new",
|
||||
"/model/update",
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class ModelsManagementClient:
|
|||
headers["Authorization"] = f"Bearer {self._api_key}"
|
||||
return headers
|
||||
|
||||
def list(self, return_request: bool = False) -> list[dict[str, Any]] | requests.Request:
|
||||
def list(self, return_request: bool = False) -> builtins.list[dict[str, Any]] | requests.Request:
|
||||
"""
|
||||
Get the list of models supported by the server.
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ class TeamsManagementClient:
|
|||
self,
|
||||
user_id: str | None = None,
|
||||
organization_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
) -> builtins.list[dict[str, Any]]:
|
||||
"""
|
||||
List teams that the user belongs to.
|
||||
|
||||
|
|
|
|||
|
|
@ -88,8 +88,10 @@ class CallbackLogsReplayer:
|
|||
)
|
||||
|
||||
metadata: Final[dict[str, Any]] = payload.get("metadata") or {}
|
||||
user_api_key_hash: Final = metadata.get("user_api_key_hash")
|
||||
litellm_metadata: Final[dict[str, Any]] = {
|
||||
"user_api_key": metadata.get("user_api_key_hash"),
|
||||
"user_api_key": user_api_key_hash,
|
||||
"user_api_key_hash": user_api_key_hash,
|
||||
"user_api_key_alias": metadata.get("user_api_key_alias"),
|
||||
"user_api_key_user_id": metadata.get("user_api_key_user_id"),
|
||||
"user_api_key_team_id": metadata.get("user_api_key_team_id"),
|
||||
|
|
|
|||
|
|
@ -2,15 +2,19 @@ import asyncio
|
|||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from typing_extensions import TypedDict
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import PTU_SENTINEL_API_KEY
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.proxy.spend_tracking.key_metadata_recovery import (
|
||||
attach_user_emails,
|
||||
recover_double_hashed_key_metadata,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
|
||||
|
|
@ -111,8 +115,19 @@ class DailySpendRecord(Protocol):
|
|||
|
||||
|
||||
class _KeyMetadataDict(TypedDict, total=False):
|
||||
key_alias: str | None
|
||||
team_id: str | None
|
||||
key_alias: ReadOnly[str | None]
|
||||
team_id: ReadOnly[str | None]
|
||||
user_id: ReadOnly[str | None]
|
||||
user_email: ReadOnly[str | None]
|
||||
|
||||
|
||||
def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
|
||||
meta: Final = api_key_metadata.get(api_key, {})
|
||||
return KeyMetadata(
|
||||
key_alias=meta.get("key_alias"),
|
||||
team_id=meta.get("team_id"),
|
||||
user_email=meta.get("user_email"),
|
||||
)
|
||||
|
||||
|
||||
_WhereValue = str | dict[str, object]
|
||||
|
|
@ -283,10 +298,7 @@ def update_breakdown_metrics(
|
|||
if record.api_key not in breakdown.models[model_key].api_key_breakdown:
|
||||
breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
),
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics(
|
||||
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics,
|
||||
|
|
@ -310,10 +322,7 @@ def update_breakdown_metrics(
|
|||
if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown:
|
||||
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
),
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics(
|
||||
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics,
|
||||
|
|
@ -335,10 +344,7 @@ def update_breakdown_metrics(
|
|||
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = (
|
||||
KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
),
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -363,10 +369,7 @@ def update_breakdown_metrics(
|
|||
if record.api_key not in breakdown.providers[provider].api_key_breakdown:
|
||||
breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
),
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics(
|
||||
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics,
|
||||
|
|
@ -388,10 +391,7 @@ def update_breakdown_metrics(
|
|||
if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown:
|
||||
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
),
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics(
|
||||
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics,
|
||||
|
|
@ -403,10 +403,7 @@ def update_breakdown_metrics(
|
|||
if record.api_key not in breakdown.api_keys:
|
||||
breakdown.api_keys[record.api_key] = KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
), # Add any api_key-specific metadata here
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record)
|
||||
|
||||
|
|
@ -426,10 +423,7 @@ def update_breakdown_metrics(
|
|||
if record.api_key not in breakdown.entities[entity_value].api_key_breakdown:
|
||||
breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
|
||||
metrics=SpendMetrics(),
|
||||
metadata=KeyMetadata(
|
||||
key_alias=api_key_metadata.get(record.api_key, {}).get("key_alias", None),
|
||||
team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None),
|
||||
),
|
||||
metadata=_key_metadata(api_key_metadata, record.api_key),
|
||||
)
|
||||
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics(
|
||||
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics,
|
||||
|
|
@ -442,17 +436,23 @@ def update_breakdown_metrics(
|
|||
async def get_api_key_metadata(
|
||||
prisma_client: PrismaClient,
|
||||
api_keys: AbstractSet[str],
|
||||
) -> dict[str, _KeyMetadataDict]:
|
||||
) -> Mapping[str, _KeyMetadataDict]:
|
||||
"""Get api key metadata, falling back to deleted keys table for keys not found in active table.
|
||||
|
||||
This ensures that key_alias and team_id are preserved in historical activity logs
|
||||
even after a key is deleted or regenerated.
|
||||
even after a key is deleted or regenerated. Also recovers aliases for api_key
|
||||
values that were double-hashed by the v1.99 spend-log provenance gate.
|
||||
"""
|
||||
key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many(
|
||||
where={"token": {"in": list(api_keys)}}
|
||||
)
|
||||
result: Final[dict[str, _KeyMetadataDict]] = {
|
||||
k.token: {"key_alias": k.key_alias, "team_id": k.team_id} for k in key_records
|
||||
k.token: {
|
||||
"key_alias": k.key_alias,
|
||||
"team_id": k.team_id,
|
||||
"user_id": getattr(k, "user_id", None),
|
||||
}
|
||||
for k in key_records
|
||||
}
|
||||
|
||||
# For any keys not found in the active table, check the deleted keys table
|
||||
|
|
@ -471,6 +471,7 @@ async def get_api_key_metadata(
|
|||
result[k.token] = {
|
||||
"key_alias": k.key_alias,
|
||||
"team_id": k.team_id,
|
||||
"user_id": getattr(k, "user_id", None),
|
||||
}
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
@ -479,7 +480,13 @@ async def get_api_key_metadata(
|
|||
e,
|
||||
)
|
||||
|
||||
return result
|
||||
still_missing: Final = api_keys - frozenset(result)
|
||||
combined: Final = (
|
||||
result
|
||||
if not still_missing
|
||||
else MappingProxyType({**result, **(await recover_double_hashed_key_metadata(prisma_client, still_missing))})
|
||||
)
|
||||
return await attach_user_emails(prisma_client, combined)
|
||||
|
||||
|
||||
def _adjust_dates_for_timezone(
|
||||
|
|
@ -951,11 +958,6 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
|
|||
)
|
||||
|
||||
|
||||
def _key_metadata(api_key_metadata: Mapping[str, _KeyMetadataDict], api_key: str) -> KeyMetadata:
|
||||
meta: Final = api_key_metadata.get(api_key, {})
|
||||
return KeyMetadata(key_alias=meta.get("key_alias"), team_id=meta.get("team_id"))
|
||||
|
||||
|
||||
def _aggregate_grouping_sets_records_sync(
|
||||
*,
|
||||
records: Sequence[_GroupingSetsRow],
|
||||
|
|
|
|||
|
|
@ -170,6 +170,8 @@ from litellm.types.proxy.management_endpoints.team_endpoints import (
|
|||
TeamMemberAddResult,
|
||||
TeamMemberInfoResponse,
|
||||
TeamMetadataSchemaResponse,
|
||||
TeamUserSpendResponse,
|
||||
TeamUserSpendRow,
|
||||
UpdateTeamMemberPermissionsRequest,
|
||||
)
|
||||
|
||||
|
|
@ -6231,3 +6233,124 @@ async def get_team_daily_activity_aggregated(
|
|||
timezone_offset_minutes=timezone,
|
||||
include_entity_breakdown=True,
|
||||
)
|
||||
|
||||
|
||||
def _team_user_spend_sql(*, team_count: int, restrict_to_user: bool) -> str:
|
||||
team_placeholders: Final = ", ".join(f"${i}" for i in range(3, 3 + team_count))
|
||||
user_clause: Final = f' AND sl."user" = ${3 + team_count}' if restrict_to_user else ""
|
||||
return f"""
|
||||
SELECT
|
||||
sl.team_id,
|
||||
sl."user" AS user_id,
|
||||
u.user_email,
|
||||
u.user_alias,
|
||||
SUM(sl.spend)::float AS spend,
|
||||
SUM(sl.prompt_tokens)::bigint AS prompt_tokens,
|
||||
SUM(sl.completion_tokens)::bigint AS completion_tokens,
|
||||
SUM(sl.total_tokens)::bigint AS total_tokens,
|
||||
COUNT(*)::bigint AS api_requests,
|
||||
COUNT(*) FILTER (WHERE sl.status IS DISTINCT FROM 'failure')::bigint AS successful_requests,
|
||||
COUNT(*) FILTER (WHERE sl.status = 'failure')::bigint AS failed_requests
|
||||
FROM "LiteLLM_SpendLogs" sl
|
||||
LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = sl."user"
|
||||
WHERE sl."startTime" >= $1::timestamp
|
||||
AND sl."startTime" < $2::timestamp + INTERVAL '1 day'
|
||||
AND sl.team_id IN ({team_placeholders}){user_clause}
|
||||
GROUP BY sl.team_id, sl."user", u.user_email, u.user_alias
|
||||
ORDER BY spend DESC, sl.team_id, sl."user"
|
||||
"""
|
||||
|
||||
|
||||
class _TeamUserSpendDbRow(TypedDict):
|
||||
team_id: ReadOnly[str]
|
||||
user_id: ReadOnly[str | None]
|
||||
user_email: ReadOnly[str | None]
|
||||
user_alias: ReadOnly[str | None]
|
||||
spend: ReadOnly[float]
|
||||
prompt_tokens: ReadOnly[int]
|
||||
completion_tokens: ReadOnly[int]
|
||||
total_tokens: ReadOnly[int]
|
||||
api_requests: ReadOnly[int]
|
||||
successful_requests: ReadOnly[int]
|
||||
failed_requests: ReadOnly[int]
|
||||
|
||||
|
||||
@router.get(
|
||||
"/team/spend/by_user",
|
||||
response_model=TeamUserSpendResponse,
|
||||
tags=["team management"], # mutable-ok: fastapi route tags must be a list
|
||||
)
|
||||
async def get_team_spend_by_user(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
team_ids: str | None = None,
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
) -> TeamUserSpendResponse:
|
||||
"""
|
||||
Spend per user within the given teams, attributed per request from spend logs.
|
||||
|
||||
Proxy admins may query any team. Team admins and members holding the
|
||||
`/team/daily/activity` permission see every user of the requested teams;
|
||||
other members only see their own row.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
if prisma_client is None:
|
||||
raise _daily_activity_error(status_code=500, message=CommonProxyErrors.db_not_connected_error.value)
|
||||
|
||||
range_error: Final = _aggregated_date_range_error(start_date, end_date)
|
||||
if range_error is not None or start_date is None or end_date is None:
|
||||
raise _daily_activity_error(status_code=400, message=range_error or "Please provide start_date and end_date")
|
||||
|
||||
if not team_ids:
|
||||
raise _daily_activity_error(status_code=400, message="Please provide team_ids")
|
||||
|
||||
scope: Final = await _resolve_team_daily_activity_scope(
|
||||
team_ids=team_ids,
|
||||
exclude_team_ids=None,
|
||||
api_key=None,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
scoped_team_ids: Final = tuple(scope.team_ids or ())
|
||||
if not scoped_team_ids:
|
||||
return TeamUserSpendResponse(start_date=start_date, end_date=end_date, results=())
|
||||
|
||||
own_user_only: Final = scope.api_key_filter is not None
|
||||
user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else ()
|
||||
rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw(
|
||||
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
|
||||
start_date,
|
||||
end_date,
|
||||
*scoped_team_ids,
|
||||
*user_param,
|
||||
)
|
||||
results: Final = tuple(
|
||||
TeamUserSpendRow(
|
||||
team_id=row["team_id"],
|
||||
team_alias=_team_alias_or_none(scope.team_alias_metadata.get(row["team_id"])),
|
||||
user_id=row["user_id"] or "",
|
||||
user_email=row["user_email"],
|
||||
user_alias=row["user_alias"],
|
||||
spend=row["spend"],
|
||||
prompt_tokens=row["prompt_tokens"],
|
||||
completion_tokens=row["completion_tokens"],
|
||||
total_tokens=row["total_tokens"],
|
||||
api_requests=row["api_requests"],
|
||||
successful_requests=row["successful_requests"],
|
||||
failed_requests=row["failed_requests"],
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
return TeamUserSpendResponse(start_date=start_date, end_date=end_date, results=results)
|
||||
|
||||
|
||||
def _team_alias_or_none(metadata: Mapping[str, object] | None) -> str | None:
|
||||
alias: Final = metadata.get("team_alias") if metadata is not None else None
|
||||
return alias if isinstance(alias, str) else None
|
||||
|
|
|
|||
|
|
@ -15212,6 +15212,7 @@ async def async_queue_request(
|
|||
# extra_body); see above for the same guard upstream.
|
||||
data["metadata"] = {}
|
||||
data["metadata"]["user_api_key"] = user_api_key_dict.api_key
|
||||
data["metadata"]["user_api_key_hash"] = user_api_key_dict.api_key
|
||||
data["metadata"]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata)
|
||||
_headers: Final = _safe_get_request_headers(request).copy()
|
||||
_headers.pop("authorization", None) # do not store the original `sk-..` api key in the db
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ datasource client {
|
|||
|
||||
generator client {
|
||||
provider = "prisma-client-py"
|
||||
recursive_type_depth = -1
|
||||
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
|
||||
}
|
||||
|
||||
|
|
|
|||
236
litellm/proxy/spend_tracking/key_metadata_recovery.py
Normal file
236
litellm/proxy/spend_tracking/key_metadata_recovery.py
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
_ACTIVE_TOKEN_DIGEST_SQL: Final = """
|
||||
SELECT encode(sha256(convert_to(token, 'UTF8')), 'hex') AS digest, key_alias, team_id, user_id
|
||||
FROM "LiteLLM_VerificationToken"
|
||||
WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[])
|
||||
"""
|
||||
|
||||
_DELETED_TOKEN_DIGEST_SQL: Final = """
|
||||
SELECT DISTINCT ON (token)
|
||||
encode(sha256(convert_to(token, 'UTF8')), 'hex') AS digest, key_alias, team_id, user_id
|
||||
FROM "LiteLLM_DeletedVerificationToken"
|
||||
WHERE encode(sha256(convert_to(token, 'UTF8')), 'hex') = ANY($1::text[])
|
||||
ORDER BY token, deleted_at DESC
|
||||
"""
|
||||
|
||||
|
||||
class KeyMetadataDict(TypedDict, total=False):
|
||||
key_alias: ReadOnly[str | None]
|
||||
team_id: ReadOnly[str | None]
|
||||
user_id: ReadOnly[str | None]
|
||||
user_email: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _TokenDigestRow(BaseModel):
|
||||
digest: str
|
||||
key_alias: str | None = None
|
||||
team_id: str | None = None
|
||||
user_id: str | None = None
|
||||
|
||||
|
||||
_TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
|
||||
_EMPTY_KEY_METADATA: Final[Mapping[str, KeyMetadataDict]] = MappingProxyType({})
|
||||
_EMPTY_EMAILS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
async def _db_or_empty(
|
||||
load: Callable[[], Awaitable[_T]],
|
||||
warning: str,
|
||||
count: int,
|
||||
) -> _T | None:
|
||||
from prisma.errors import PrismaError
|
||||
|
||||
try:
|
||||
return await load()
|
||||
except PrismaError as e:
|
||||
verbose_proxy_logger.warning(warning, count, e)
|
||||
return None
|
||||
|
||||
|
||||
async def _reverse_hash_key_metadata(
|
||||
prisma_client: PrismaClient,
|
||||
sql: str,
|
||||
wanted: AbstractSet[str],
|
||||
*,
|
||||
warning: str,
|
||||
) -> Mapping[str, KeyMetadataDict]:
|
||||
rows: Final = await _db_or_empty(
|
||||
lambda: prisma_client.db.query_raw(sql, sorted(wanted)),
|
||||
warning,
|
||||
len(wanted),
|
||||
)
|
||||
if rows is None:
|
||||
return _EMPTY_KEY_METADATA
|
||||
return MappingProxyType(
|
||||
{
|
||||
row.digest: KeyMetadataDict(key_alias=row.key_alias, team_id=row.team_id, user_id=row.user_id)
|
||||
for row in _TOKEN_DIGEST_ROWS.validate_python(rows)
|
||||
if row.digest in wanted
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _emails_for_user_ids(
|
||||
prisma_client: PrismaClient,
|
||||
user_ids: AbstractSet[str],
|
||||
) -> Mapping[str, str]:
|
||||
if not user_ids:
|
||||
return _EMPTY_EMAILS
|
||||
users: Final = await _db_or_empty(
|
||||
lambda: UserRepository(prisma_client).table.find_many(
|
||||
where={"user_id": {"in": list(user_ids)}}, # mutable-ok: Prisma find_many where= is a dict
|
||||
),
|
||||
"Failed user_email recovery for %d user ids: %s",
|
||||
len(user_ids),
|
||||
)
|
||||
if users is None:
|
||||
return _EMPTY_EMAILS
|
||||
return MappingProxyType(
|
||||
{
|
||||
user.user_id: user.user_email
|
||||
for user in users
|
||||
if getattr(user, "user_id", None) and getattr(user, "user_email", None)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _meta_with_email(meta: KeyMetadataDict, emails: Mapping[str, str]) -> KeyMetadataDict:
|
||||
if meta.get("user_email"):
|
||||
return meta
|
||||
user_id: Final = meta.get("user_id")
|
||||
if not isinstance(user_id, str) or user_id not in emails:
|
||||
return meta
|
||||
updated: Final[KeyMetadataDict] = {**meta, "user_email": emails[user_id]}
|
||||
return updated
|
||||
|
||||
|
||||
async def attach_user_emails(
|
||||
prisma_client: PrismaClient,
|
||||
recovered: Mapping[str, KeyMetadataDict],
|
||||
) -> Mapping[str, KeyMetadataDict]:
|
||||
needing_email: Final = frozenset(
|
||||
user_id
|
||||
for meta in recovered.values()
|
||||
for user_id in (meta.get("user_id"),)
|
||||
if isinstance(user_id, str) and user_id and not meta.get("user_email")
|
||||
)
|
||||
emails: Final = await _emails_for_user_ids(prisma_client, needing_email)
|
||||
if not emails:
|
||||
return recovered
|
||||
return MappingProxyType({api_key: _meta_with_email(meta, emails) for api_key, meta in recovered.items()})
|
||||
|
||||
|
||||
async def recover_double_hashed_key_metadata(
|
||||
prisma_client: PrismaClient,
|
||||
missing_keys: AbstractSet[str],
|
||||
) -> Mapping[str, KeyMetadataDict]:
|
||||
"""
|
||||
Recover key_alias/team_id/user_id for DailyUserSpend.api_key values that
|
||||
were double-hashed by the v1.99 spend-log provenance gate.
|
||||
|
||||
Those rows store hash(VerificationToken.token) instead of the token, so the
|
||||
exact join misses. Postgres hashes the token column itself, one pass over
|
||||
active keys and one over deleted keys, so no key row crosses the wire.
|
||||
"""
|
||||
sha_missing: Final = frozenset(key for key in missing_keys if is_valid_sha256_hash(key))
|
||||
if not sha_missing:
|
||||
return _EMPTY_KEY_METADATA
|
||||
|
||||
from_active: Final = await _reverse_hash_key_metadata(
|
||||
prisma_client,
|
||||
_ACTIVE_TOKEN_DIGEST_SQL,
|
||||
sha_missing,
|
||||
warning="Failed reverse-hash recovery against active keys for %d missing keys: %s",
|
||||
)
|
||||
still_missing: Final = sha_missing - frozenset(from_active)
|
||||
if not still_missing:
|
||||
return from_active
|
||||
from_deleted: Final = await _reverse_hash_key_metadata(
|
||||
prisma_client,
|
||||
_DELETED_TOKEN_DIGEST_SQL,
|
||||
still_missing,
|
||||
warning="Failed reverse-hash recovery against deleted keys for %d missing keys: %s",
|
||||
)
|
||||
return MappingProxyType({**from_active, **from_deleted})
|
||||
|
||||
|
||||
def _row_with_recovered_fields(
|
||||
row: Mapping[str, object],
|
||||
recovered: Mapping[str, KeyMetadataDict],
|
||||
*,
|
||||
api_key_field: str,
|
||||
alias_field: str,
|
||||
team_id_field: str,
|
||||
user_email_field: str,
|
||||
) -> Mapping[str, object]:
|
||||
api_key: Final = row.get(api_key_field)
|
||||
if not isinstance(api_key, str) or api_key not in recovered:
|
||||
return row
|
||||
meta: Final = recovered[api_key]
|
||||
return MappingProxyType(
|
||||
{
|
||||
**row,
|
||||
alias_field: meta.get("key_alias") or row.get(alias_field),
|
||||
team_id_field: meta.get("team_id") or row.get(team_id_field),
|
||||
user_email_field: row.get(user_email_field) or meta.get("user_email"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def fill_missing_api_key_aliases(
|
||||
prisma_client: PrismaClient,
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
*,
|
||||
api_key_field: str = "api_key",
|
||||
alias_field: str = "api_key_alias",
|
||||
team_id_field: str = "team_id",
|
||||
user_email_field: str = "user_email",
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""
|
||||
Fill null api_key_alias / team_id / user_email on export rows whose api_key
|
||||
was double-hashed.
|
||||
|
||||
Used by CloudZero and Focus, which join DailyUserSpend.api_key to
|
||||
VerificationToken.token and otherwise export null aliases for those rows.
|
||||
"""
|
||||
missing_keys: Final = frozenset(
|
||||
key
|
||||
for row in rows
|
||||
for key in (row.get(api_key_field),)
|
||||
if isinstance(key, str) and key and row.get(alias_field) in (None, "")
|
||||
)
|
||||
if not missing_keys:
|
||||
return tuple(rows)
|
||||
|
||||
recovered: Final = await attach_user_emails(
|
||||
prisma_client,
|
||||
await recover_double_hashed_key_metadata(prisma_client, missing_keys),
|
||||
)
|
||||
if not recovered:
|
||||
return tuple(rows)
|
||||
|
||||
return tuple(
|
||||
_row_with_recovered_fields(
|
||||
row,
|
||||
recovered,
|
||||
api_key_field=api_key_field,
|
||||
alias_field=alias_field,
|
||||
team_id_field=team_id_field,
|
||||
user_email_field=user_email_field,
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
"""LiteLLM Rust bridge package."""
|
||||
|
||||
from litellm.rust_bridge.configuration import use_litellm_rust
|
||||
from litellm.rust_bridge.configuration import rust
|
||||
from litellm.rust_bridge.loader import (
|
||||
get_native_bridge,
|
||||
native_bridge_available,
|
||||
reset_native_bridge_cache,
|
||||
)
|
||||
|
||||
__all__ = ["get_native_bridge", "native_bridge_available", "reset_native_bridge_cache", "use_litellm_rust"]
|
||||
__all__ = ["get_native_bridge", "native_bridge_available", "reset_native_bridge_cache", "rust"]
|
||||
|
|
|
|||
|
|
@ -2,13 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import os
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.rust_bridge.messages import RustAmessages, RustMessages
|
||||
from litellm.rust_bridge.ocr import RustAocr, RustOcr
|
||||
from litellm.rust_bridge.responses_websocket import RustResponsesWebSocketConnection
|
||||
from litellm.rust_bridge.transcription import RustAtranscription, RustTranscription
|
||||
from typing import Final
|
||||
|
||||
DEFAULT_RUST_ENABLED: Final = False
|
||||
_TRUE_ENV_VALUES: Final = frozenset({"1", "true", "yes", "on"})
|
||||
|
|
@ -16,13 +10,6 @@ _GLOBAL_ENV_NAME: Final = "LITELLM_RUST"
|
|||
_LEGACY_OCR_ENV_NAME: Final = "LITELLM_USE_RUST_OCR"
|
||||
|
||||
|
||||
class _Unset:
|
||||
pass
|
||||
|
||||
|
||||
_UNSET: Final = _Unset()
|
||||
|
||||
|
||||
class _RustConfiguration:
|
||||
def __init__(self) -> None:
|
||||
self.override: bool | None = None
|
||||
|
|
@ -42,7 +29,7 @@ def resolve_rust_enabled(
|
|||
request_override: bool | None,
|
||||
process_override: bool | None,
|
||||
environment_override: bool | None,
|
||||
legacy_ocr_override: bool | None = None,
|
||||
legacy_environment_override: bool | None = None,
|
||||
release_default: bool = DEFAULT_RUST_ENABLED,
|
||||
) -> bool:
|
||||
if request_override is not None:
|
||||
|
|
@ -51,25 +38,12 @@ def resolve_rust_enabled(
|
|||
return process_override
|
||||
if environment_override is not None:
|
||||
return environment_override
|
||||
if legacy_ocr_override is not None:
|
||||
return legacy_ocr_override
|
||||
if legacy_environment_override is not None:
|
||||
return legacy_environment_override
|
||||
return release_default
|
||||
|
||||
|
||||
def rust_enabled(*, request_override: bool | None = None) -> bool:
|
||||
if request_override is not None:
|
||||
return request_override
|
||||
process_override: Final = _CONFIGURATION.override
|
||||
if process_override is not None:
|
||||
return process_override
|
||||
return resolve_rust_enabled(
|
||||
request_override=None,
|
||||
process_override=None,
|
||||
environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)),
|
||||
)
|
||||
|
||||
|
||||
def rust_ocr_enabled(*, request_override: bool | None = None) -> bool:
|
||||
if request_override is not None:
|
||||
return request_override
|
||||
process_override: Final = _CONFIGURATION.override
|
||||
|
|
@ -87,62 +61,21 @@ def rust_ocr_enabled(*, request_override: bool | None = None) -> bool:
|
|||
request_override=None,
|
||||
process_override=None,
|
||||
environment_override=global_override,
|
||||
legacy_ocr_override=legacy_override,
|
||||
legacy_environment_override=legacy_override,
|
||||
)
|
||||
|
||||
|
||||
def rust_ocr_enabled(*, request_override: bool | None = None) -> bool:
|
||||
return rust_enabled(request_override=request_override)
|
||||
|
||||
|
||||
def reset_rust_configuration() -> None:
|
||||
_CONFIGURATION.override = None
|
||||
|
||||
|
||||
def use_litellm_rust(
|
||||
enabled: bool = True,
|
||||
*,
|
||||
ocr: RustOcr | None | _Unset = _UNSET,
|
||||
aocr: RustAocr | None | _Unset = _UNSET,
|
||||
messages: RustMessages | None | _Unset = _UNSET,
|
||||
amessages: RustAmessages | None | _Unset = _UNSET,
|
||||
responses_websocket: type[RustResponsesWebSocketConnection] | None | _Unset = _UNSET,
|
||||
transcription: RustTranscription | None | _Unset = _UNSET,
|
||||
atranscription: RustAtranscription | None | _Unset = _UNSET,
|
||||
) -> None:
|
||||
def rust(enabled: bool) -> None:
|
||||
"""Set the process override for optional Rust paths.
|
||||
|
||||
Rust-only paths, including Bedrock transcription, are not controlled by this switch.
|
||||
"""
|
||||
_CONFIGURATION.override = enabled
|
||||
bindings: Final = (ocr, aocr, messages, amessages, responses_websocket, transcription, atranscription)
|
||||
if all(isinstance(binding, _Unset) for binding in bindings):
|
||||
return
|
||||
warnings.warn(
|
||||
"Injecting Rust bridge implementations through use_litellm_rust() is deprecated; "
|
||||
"use the internal bridge setters in tests",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if not isinstance(ocr, _Unset) or not isinstance(aocr, _Unset):
|
||||
from litellm.rust_bridge.ocr import set_rust_ocr
|
||||
|
||||
if not isinstance(ocr, _Unset):
|
||||
set_rust_ocr(ocr=ocr)
|
||||
if not isinstance(aocr, _Unset):
|
||||
set_rust_ocr(aocr=aocr)
|
||||
if not isinstance(messages, _Unset) or not isinstance(amessages, _Unset):
|
||||
from litellm.rust_bridge.messages import set_rust_messages
|
||||
|
||||
if not isinstance(messages, _Unset):
|
||||
set_rust_messages(messages=messages)
|
||||
if not isinstance(amessages, _Unset):
|
||||
set_rust_messages(amessages=amessages)
|
||||
if not isinstance(responses_websocket, _Unset):
|
||||
from litellm.rust_bridge.responses_websocket import set_rust_responses_websocket
|
||||
|
||||
set_rust_responses_websocket(connection=responses_websocket)
|
||||
if not isinstance(transcription, _Unset) or not isinstance(atranscription, _Unset):
|
||||
from litellm.rust_bridge.transcription import configure_rust_transcription
|
||||
|
||||
if not isinstance(transcription, _Unset):
|
||||
configure_rust_transcription(transcription=transcription)
|
||||
if not isinstance(atranscription, _Unset):
|
||||
configure_rust_transcription(atranscription=atranscription)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from litellm.rust_bridge import configuration as _configuration
|
|||
from litellm.rust_bridge.timeouts import timeout_to_seconds as _timeout_to_seconds
|
||||
|
||||
rust_ocr_enabled = _configuration.rust_ocr_enabled
|
||||
use_litellm_rust = _configuration.use_litellm_rust
|
||||
rust = _configuration.rust
|
||||
|
||||
|
||||
class RustOcr(Protocol):
|
||||
|
|
|
|||
|
|
@ -632,7 +632,7 @@ class ChatCompletionReasoningItem(TypedDict, total=False):
|
|||
type: Required[Literal["reasoning"]]
|
||||
id: str
|
||||
encrypted_content: str | None
|
||||
summary: list["ChatCompletionReasoningSummaryTextBlock"]
|
||||
summary: ReadOnly[list[ChatCompletionReasoningSummaryTextBlock]]
|
||||
|
||||
|
||||
class WebSearchOptionsUserLocationApproximate(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ class KeyMetadata(BaseModel):
|
|||
|
||||
key_alias: str | None = None
|
||||
team_id: str | None = None
|
||||
user_email: str | None = None
|
||||
|
||||
|
||||
class KeyMetricWithMetadata(MetricBase):
|
||||
|
|
|
|||
|
|
@ -143,3 +143,24 @@ class TeamMetadataSchemaResponse(BaseModel):
|
|||
"""Response for GET /team/metadata_schema; ``fields`` is empty when no schema is configured."""
|
||||
|
||||
fields: tuple[TeamMetadataFieldSchema, ...]
|
||||
|
||||
|
||||
class TeamUserSpendRow(BaseModel):
|
||||
team_id: str
|
||||
team_alias: str | None = None
|
||||
user_id: str
|
||||
user_email: str | None = None
|
||||
user_alias: str | None = None
|
||||
spend: float = 0.0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
api_requests: int = 0
|
||||
successful_requests: int = 0
|
||||
failed_requests: int = 0
|
||||
|
||||
|
||||
class TeamUserSpendResponse(BaseModel):
|
||||
start_date: str
|
||||
end_date: str
|
||||
results: tuple[TeamUserSpendRow, ...]
|
||||
|
|
|
|||
|
|
@ -5121,14 +5121,8 @@ def get_response_string(response_obj: ModelResponse | ModelResponseStream) -> st
|
|||
return "".join(response_parts)
|
||||
|
||||
|
||||
def get_utc_datetime():
|
||||
import datetime as dt
|
||||
from datetime import datetime
|
||||
|
||||
if hasattr(dt, "UTC"):
|
||||
return datetime.now(dt.UTC)
|
||||
else:
|
||||
return datetime.utcnow()
|
||||
def get_utc_datetime() -> datetime.datetime:
|
||||
return datetime.datetime.now(datetime.timezone.utc)
|
||||
|
||||
|
||||
def get_max_tokens(model: str) -> int | None:
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ def _direct_vector_store_embedding_executor(
|
|||
|
||||
|
||||
def mock_vector_store_search_response(
|
||||
mock_results: list[VectorStoreSearchResult] | None = None,
|
||||
mock_results: builtins.list[VectorStoreSearchResult] | None = None,
|
||||
):
|
||||
"""Mock response for vector store search"""
|
||||
if mock_results is None:
|
||||
|
|
@ -108,7 +108,7 @@ def mock_vector_store_create_response(
|
|||
@client
|
||||
async def acreate(
|
||||
name: str | None = None,
|
||||
file_ids: list[str] | None = None,
|
||||
file_ids: builtins.list[str] | None = None,
|
||||
expires_after: dict | None = None,
|
||||
chunking_strategy: dict | None = None,
|
||||
metadata: dict[str, str] | None = None,
|
||||
|
|
@ -172,7 +172,7 @@ async def acreate(
|
|||
@client
|
||||
def create(
|
||||
name: str | None = None,
|
||||
file_ids: list[str] | None = None,
|
||||
file_ids: builtins.list[str] | None = None,
|
||||
expires_after: dict | None = None,
|
||||
chunking_strategy: dict | None = None,
|
||||
metadata: dict[str, str] | None = None,
|
||||
|
|
@ -285,7 +285,7 @@ def create(
|
|||
@client
|
||||
async def asearch(
|
||||
vector_store_id: str,
|
||||
query: str | list[str],
|
||||
query: str | builtins.list[str],
|
||||
filters: dict | None = None,
|
||||
max_num_results: int | None = None,
|
||||
ranking_options: dict | None = None,
|
||||
|
|
@ -360,7 +360,7 @@ async def asearch(
|
|||
@client
|
||||
def search(
|
||||
vector_store_id: str,
|
||||
query: str | list[str],
|
||||
query: str | builtins.list[str],
|
||||
filters: dict | None = None,
|
||||
max_num_results: int | None = None,
|
||||
ranking_options: dict | None = None,
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ proxy = [
|
|||
"gunicorn>=23.0.0,<24.0",
|
||||
"uvicorn>=0.33.0,<1.0",
|
||||
"granian>=2.7.4,<3.0",
|
||||
"uvloop>=0.21.0,<1.0; sys_platform != 'win32'",
|
||||
"uvloop>=0.22.1,<1.0; sys_platform != 'win32'",
|
||||
"fastapi>=0.136.3,<1.0",
|
||||
"starlette>=1.0.1,<2.0",
|
||||
"backoff>=2.2.1,<3.0",
|
||||
|
|
@ -179,6 +179,7 @@ dev = [
|
|||
"basedpyright==1.39.7",
|
||||
"keyring==25.7.0",
|
||||
"pytest==9.0.3",
|
||||
"tomli==2.4.1; python_version < '3.11'",
|
||||
"pytest-mock==3.15.1",
|
||||
"pytest-asyncio==1.3.0",
|
||||
"pytest-postgresql==7.0.2",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
"limit": 809
|
||||
},
|
||||
"ANN201": {
|
||||
"limit": 2000
|
||||
"limit": 1999
|
||||
},
|
||||
"ANN202": {
|
||||
"limit": 835
|
||||
|
|
@ -87,7 +87,7 @@
|
|||
"limit": 2
|
||||
},
|
||||
"DTZ003": {
|
||||
"limit": 26
|
||||
"limit": 24
|
||||
},
|
||||
"DTZ005": {
|
||||
"limit": 233
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ datasource client {
|
|||
|
||||
generator client {
|
||||
provider = "prisma-client-py"
|
||||
recursive_type_depth = -1
|
||||
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ seen the red and accepted it.
|
|||
Usage:
|
||||
python scripts/budget_ratchet_check.py [--base REF] [budget.json ...]
|
||||
|
||||
Stdlib only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -33,11 +32,15 @@ import argparse
|
|||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import NamedTuple
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
import tomli as tomllib
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_BASE = "origin/litellm_internal_staging"
|
||||
DEFAULT_BUDGETS: tuple[str, ...] = (
|
||||
|
|
|
|||
|
|
@ -18,13 +18,17 @@ import json
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
from collections import defaultdict
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Final, NamedTuple
|
||||
from textwrap import dedent
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
import tomli as tomllib
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"]
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ GET /tag/user-agent/per-user-analytics
|
|||
GET /tag/wau
|
||||
GET /team/daily/activity
|
||||
GET /team/daily/activity/aggregated
|
||||
GET /team/spend/by_user
|
||||
GET /team/spend/report
|
||||
GET /user/daily/activity
|
||||
GET /user/daily/activity/aggregated
|
||||
|
|
|
|||
|
|
@ -21,6 +21,6 @@
|
|||
"limit": 117
|
||||
},
|
||||
"TQ008": {
|
||||
"limit": 11135
|
||||
"limit": 11003
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,12 +6,16 @@ from pathlib import Path
|
|||
import re
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from typing import Callable, Dict, Final, List, Optional, Protocol, Set, Tuple
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
import requests
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
import tomli as tomllib
|
||||
|
||||
DEFAULT_TRANSITIVE_PIN_PACKAGES = (
|
||||
"aiofiles",
|
||||
"anyio",
|
||||
|
|
|
|||
58
tests/proxy_behavior/management/test_team_spend_by_user.py
Normal file
58
tests/proxy_behavior/management/test_team_spend_by_user.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import pytest
|
||||
|
||||
from .actors import Actor
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
|
||||
# GET /team/spend/by_user shares the team-scope resolver with
|
||||
# /team/daily/activity, so the membership matrix must hold here too. team_ids
|
||||
# is mandatory on this route (a per-user rollup with no team is meaningless),
|
||||
# so the bare query is 400 for everyone instead of defaulting to own teams.
|
||||
_MEMBERS = {
|
||||
"alpha": {
|
||||
Actor.TEAM_ADMIN,
|
||||
Actor.INTERNAL_USER,
|
||||
Actor.OWNER,
|
||||
Actor.UNRELATED_SAME_ORG,
|
||||
Actor.SERVICE_ACCOUNT,
|
||||
},
|
||||
"beta": {Actor.CROSS_ORG_USER},
|
||||
}
|
||||
|
||||
|
||||
def _expected(actor: Actor, team: str) -> int:
|
||||
if team == "none":
|
||||
return 400
|
||||
if actor == Actor.PROXY_ADMIN:
|
||||
return 200
|
||||
return 200 if actor in _MEMBERS.get(team, set()) else 404
|
||||
|
||||
|
||||
_CASES = [
|
||||
(f"{team}/{actor.value}", actor, team, _expected(actor, team))
|
||||
for team in ("none", "alpha", "beta")
|
||||
for actor in Actor
|
||||
]
|
||||
|
||||
_DATES = "start_date=2024-01-01&end_date=2024-12-31"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"actor,team,expected_status",
|
||||
[(a, t, s) for (_id, a, t, s) in _CASES],
|
||||
ids=[c[0] for c in _CASES],
|
||||
)
|
||||
async def test_team_spend_by_user_matrix(actor: Actor, team: str, expected_status: int, proxy_client, world):
|
||||
team_id = {"alpha": world.team_alpha_id, "beta": world.team_beta_id}.get(team)
|
||||
query = _DATES if team_id is None else f"{_DATES}&team_ids={team_id}"
|
||||
|
||||
resp = await proxy_client.get(
|
||||
f"/team/spend/by_user?{query}",
|
||||
headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"},
|
||||
)
|
||||
assert resp.status_code == expected_status, f"{actor.value} -> {team}: {resp.status_code} {resp.text}"
|
||||
if expected_status == 200:
|
||||
body = resp.json()
|
||||
assert (body["start_date"], body["end_date"]) == ("2024-01-01", "2024-12-31")
|
||||
assert all(row["team_id"] == team_id for row in body["results"])
|
||||
|
|
@ -2445,6 +2445,7 @@ class TestBatchCostAttribution:
|
|||
metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1")
|
||||
|
||||
assert metadata["user_api_key"] == "hash-alice"
|
||||
assert metadata["user_api_key_hash"] == "hash-alice"
|
||||
assert metadata["user_api_key_user_id"] == "alice"
|
||||
assert metadata["user_api_key_team_id"] == "team-alpha"
|
||||
assert metadata["user_api_key_alias"] == "prod-key"
|
||||
|
|
@ -2553,6 +2554,48 @@ class TestBatchCostAttribution:
|
|||
|
||||
assert metadata["user_api_key_alias"] == "prod-key"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_provenance_keeps_spend_log_api_key_joinable(self):
|
||||
"""
|
||||
CheckBatchCost stores the VerificationToken hash on the managed object. The
|
||||
spend-log writer must receive matching user_api_key_hash provenance so it
|
||||
does not re-hash that value; otherwise DailyUserSpend.api_key no longer joins
|
||||
VerificationToken and Usage shows key-hash-... with a null alias/email.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
token_hash = hash_token("sk-batch-creator-key")
|
||||
instance = self._instance(
|
||||
key_row=SimpleNamespace(key_alias="prod-key"),
|
||||
user_row=SimpleNamespace(user_email="alice@example.com", user_alias=None),
|
||||
)
|
||||
metadata = await instance._build_creator_attribution_metadata(
|
||||
self._job(api_key=token_hash), "batch-1"
|
||||
)
|
||||
|
||||
assert metadata["user_api_key"] == token_hash
|
||||
assert metadata["user_api_key_hash"] == token_hash
|
||||
|
||||
payload = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "gpt-4o",
|
||||
"call_type": "aretrieve_batch",
|
||||
"litellm_params": {"metadata": metadata},
|
||||
},
|
||||
response_obj={
|
||||
"id": "batch_123",
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
start_time=datetime.now(timezone.utc),
|
||||
end_time=datetime.now(timezone.utc),
|
||||
)
|
||||
assert payload["api_key"] == token_hash
|
||||
assert payload["api_key"] != hash_token(token_hash)
|
||||
|
||||
|
||||
class TestPollPageStarvation:
|
||||
"""LIT-5462 regression: a row that can never be costed used to keep its slot in the
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ def test_proxy_reducto_ocr_json_passthrough_data_uri(client_no_auth):
|
|||
pages=[OCRPage(index=0, markdown="Proxy OCR")],
|
||||
model="parse-v3",
|
||||
usage_info=OCRUsageInfo(pages_processed=1, credits=1),
|
||||
tables=[{"cells": [["Total", 42]], "page": 1}],
|
||||
keyValuePairs=[{"key": "approved", "value": True, "confidence": 0.9}],
|
||||
)
|
||||
|
||||
data_uri = "data:application/pdf;base64,JVBERi0xLjQK"
|
||||
|
|
@ -135,3 +137,5 @@ def test_proxy_reducto_ocr_json_passthrough_data_uri(client_no_auth):
|
|||
assert response_body["object"] == "ocr"
|
||||
assert response_body["usage_info"]["credits"] == 1
|
||||
assert response_body["pages"][0]["markdown"] == "Proxy OCR"
|
||||
assert response_body["tables"] == [{"cells": [["Total", 42]], "page": 1}]
|
||||
assert response_body["keyValuePairs"] == [{"key": "approved", "value": True, "confidence": 0.9}]
|
||||
|
|
|
|||
|
|
@ -237,10 +237,12 @@ class TestRouterIndexManagement:
|
|||
- model_name_to_deployment_indices for O(1) + O(k) model_name lookups
|
||||
"""
|
||||
# Methods that are allowed to iterate through self.model_list
|
||||
ALLOWED_METHODS = [
|
||||
"_get_deployment_by_litellm_model", # Edge case: lookup by litellm_params.model (not indexed)
|
||||
"_finalize_adaptive_router_if_configured", # Init-time prefix scan for "auto_router/adaptive_router" (no index for prefix match)
|
||||
]
|
||||
ALLOWED_METHODS = {
|
||||
"_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed",
|
||||
"_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match',
|
||||
"config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)",
|
||||
"heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)",
|
||||
}
|
||||
|
||||
# Get path to router.py
|
||||
router_file = os.path.join(
|
||||
|
|
|
|||
|
|
@ -73,6 +73,15 @@ def _rebuild(repo_root: Path) -> tuple[bool, str]:
|
|||
return completed.returncode == 0, "\n".join(lines[-_FAILURE_OUTPUT_LINES:])
|
||||
|
||||
|
||||
def trace_bridge_error() -> str | None:
|
||||
bridge: Final = get_native_bridge()
|
||||
if bridge is None:
|
||||
return "native Rust bridge is not importable"
|
||||
if getattr(bridge, "_trace", None) is None:
|
||||
return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature"
|
||||
return None
|
||||
|
||||
|
||||
def ensure_trace_bridge(repo_root: Path) -> str | None:
|
||||
native_path: Final = _native_module_path()
|
||||
native_mtime: Final = native_path.stat().st_mtime if native_path is not None and native_path.exists() else None
|
||||
|
|
@ -84,9 +93,4 @@ def ensure_trace_bridge(repo_root: Path) -> str | None:
|
|||
if not succeeded:
|
||||
return f"native Rust bridge rebuild failed:\n{output}"
|
||||
_drop_imported_bridge()
|
||||
bridge: Final = get_native_bridge()
|
||||
if bridge is None:
|
||||
return "native Rust bridge is not importable"
|
||||
if getattr(bridge, "_trace", None) is None:
|
||||
return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature"
|
||||
return None
|
||||
return trace_bridge_error()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ from dataclasses import dataclass, field
|
|||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, assert_never
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .strategy import CaseSpec, StrategyDefinition
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, assert_never
|
||||
from typing import Final, Protocol
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from .models import CaseDisposition, CaseResult
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@ from __future__ import annotations
|
|||
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import CodeType, FrameType
|
||||
from types import CodeType, FrameType, FunctionType, MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ class PythonProfiler:
|
|||
def __call__(self, frame: FrameType, event: str, _arg: object) -> None:
|
||||
if event != "call" or frame in self._seen_frames:
|
||||
return
|
||||
function_name: Final = self.function_name(frame.f_code)
|
||||
function_name: Final = self.function_name(frame)
|
||||
if function_name is None:
|
||||
return
|
||||
event_id: Final = len(self.events)
|
||||
|
|
@ -48,11 +49,12 @@ class PythonProfiler:
|
|||
self._event_ids[frame] = event_id
|
||||
self.events.append(FunctionTraceEvent(id=event_id, parent_id=parent_id, function=function_name))
|
||||
|
||||
def function_name(self, code: CodeType) -> str | None:
|
||||
def function_name(self, frame: FrameType) -> str | None:
|
||||
code: Final = frame.f_code
|
||||
if not code.co_filename.startswith(self._source_root):
|
||||
return None
|
||||
relative: Final = code.co_filename.removeprefix(self._source_root)
|
||||
return f"{relative}:{code.co_firstlineno} {getattr(code, 'co_qualname', code.co_name)}"
|
||||
return f"{relative}:{code.co_firstlineno} {_qualified_name(frame)}"
|
||||
|
||||
|
||||
class PythonFunctionUsageProfiler:
|
||||
|
|
@ -68,11 +70,65 @@ class PythonFunctionUsageProfiler:
|
|||
if not code.co_filename.startswith(self._source_root):
|
||||
return
|
||||
relative: Final = code.co_filename.removeprefix(self._source_root)
|
||||
function: Final = f"{relative}:{code.co_firstlineno} {getattr(code, 'co_qualname', code.co_name)}"
|
||||
function: Final = f"{relative}:{code.co_firstlineno} {_qualified_name(frame)}"
|
||||
if function in self._functions:
|
||||
self.called.add(function)
|
||||
|
||||
|
||||
def _qualified_name(frame: FrameType) -> str:
|
||||
code: Final = frame.f_code
|
||||
native: Final = getattr(code, "co_qualname", None)
|
||||
if isinstance(native, str):
|
||||
return native
|
||||
enclosing: Final = next(
|
||||
(
|
||||
name
|
||||
for ancestor in _frame_ancestors(frame)
|
||||
for declared_code, name in _declared_functions(ancestor.f_locals, frozenset())
|
||||
if declared_code is code
|
||||
),
|
||||
None,
|
||||
)
|
||||
if enclosing is not None:
|
||||
return enclosing
|
||||
module_name: Final = frame.f_globals.get("__name__")
|
||||
if not isinstance(module_name, str):
|
||||
return code.co_name
|
||||
return _module_qualnames(module_name).get(code, code.co_name)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _module_qualnames(module_name: str) -> Mapping[CodeType, str]:
|
||||
module: Final = sys.modules.get(module_name)
|
||||
if module is None:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType(dict(_declared_functions(vars(module), frozenset())))
|
||||
|
||||
|
||||
def _declared_functions(namespace: Mapping[str, object], visited: frozenset[int]) -> Iterator[tuple[CodeType, str]]:
|
||||
for attribute in tuple(namespace.values()):
|
||||
for value in _accessors(attribute):
|
||||
if isinstance(value, FunctionType):
|
||||
yield from ((wrapped.__code__, wrapped.__qualname__) for wrapped in _unwrapped(value))
|
||||
elif isinstance(value, type) and id(value) not in visited:
|
||||
yield from _declared_functions(dict(vars(value)), visited | {id(value)})
|
||||
|
||||
|
||||
def _unwrapped(function: FunctionType) -> Iterator[FunctionType]:
|
||||
yield function
|
||||
inner: Final = getattr(function, "__wrapped__", None)
|
||||
if isinstance(inner, FunctionType):
|
||||
yield from _unwrapped(inner)
|
||||
|
||||
|
||||
def _accessors(value: object) -> tuple[object, ...]:
|
||||
if isinstance(value, (staticmethod, classmethod)):
|
||||
return (value.__func__,)
|
||||
if isinstance(value, property):
|
||||
return tuple(accessor for accessor in (value.fget, value.fset, value.fdel) if accessor is not None)
|
||||
return (value,)
|
||||
|
||||
|
||||
def _frame_ancestors(frame: FrameType) -> Generator[FrameType]:
|
||||
ancestor: Final = frame.f_back
|
||||
if ancestor is not None:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import tempfile
|
|||
import warnings
|
||||
from collections.abc import Generator, Sequence
|
||||
from pathlib import Path
|
||||
from types import CodeType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pluggy import HookimplMarker
|
||||
|
|
@ -62,9 +63,12 @@ class PythonFunctionReference(BaseModel):
|
|||
value: object = importlib.import_module(self.module)
|
||||
for component in self.qualname.split("."):
|
||||
value = getattr(value, component)
|
||||
if not callable(value):
|
||||
raise ValueError(f"Python function is not callable: {self.module}:{self.qualname}")
|
||||
function: Final = inspect.unwrap(value)
|
||||
code: Final = getattr(function, "__code__", None)
|
||||
if code is None:
|
||||
qualname: Final = getattr(function, "__qualname__", None)
|
||||
if not isinstance(code, CodeType) or not isinstance(qualname, str):
|
||||
raise ValueError(f"Python function has no code object: {self.module}:{self.qualname}")
|
||||
source: Final = Path(code.co_filename).resolve()
|
||||
try:
|
||||
|
|
@ -74,7 +78,7 @@ class PythonFunctionReference(BaseModel):
|
|||
return PythonFunctionIdentity(
|
||||
file=relative.as_posix(),
|
||||
line=code.co_firstlineno,
|
||||
qualname=code.co_qualname,
|
||||
qualname=qualname,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,38 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from types import FunctionType
|
||||
from typing import Final, ParamSpec, TypeVar, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from .profiler import FunctionTraceEvent, PythonProfiler, profile_python, profile_python_function_usage
|
||||
from .profiler import (
|
||||
FunctionTraceEvent,
|
||||
PythonProfiler,
|
||||
_module_qualnames,
|
||||
profile_python,
|
||||
profile_python_function_usage,
|
||||
)
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def _passthrough(function: Callable[_P, _T]) -> Callable[_P, _T]:
|
||||
@wraps(function)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:
|
||||
return function(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class Decorated:
|
||||
@_passthrough
|
||||
def call(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _events_named(profiler: PythonProfiler, name: str) -> tuple[FunctionTraceEvent, ...]:
|
||||
|
|
@ -26,6 +52,14 @@ def test_profiler_keeps_repeated_calls() -> None:
|
|||
assert len(_events_named(profiler, "called")) == 2
|
||||
|
||||
|
||||
def test_profiler_qualifies_decorated_methods_by_class() -> None:
|
||||
with profile_python(Path(__file__).parent) as profiler:
|
||||
Decorated().call()
|
||||
|
||||
assert any(event.function.endswith(" Decorated.call") for event in profiler.events)
|
||||
assert _module_qualnames(__name__)[cast(FunctionType, Decorated.call.__wrapped__).__code__] == "Decorated.call"
|
||||
|
||||
|
||||
def test_profiler_records_real_frame_ancestry() -> None:
|
||||
def called() -> None:
|
||||
return None
|
||||
|
|
@ -98,8 +132,7 @@ def test_function_usage_profiler_records_only_selected_functions() -> None:
|
|||
return None
|
||||
|
||||
source_root: Final = Path(__file__).parent
|
||||
function: Final = PythonProfiler(source_root).function_name(selected.__code__)
|
||||
assert function is not None
|
||||
function: Final = f"{Path(__file__).name}:{selected.__code__.co_firstlineno} {selected.__qualname__}"
|
||||
|
||||
with profile_python_function_usage(source_root, frozenset((function,))) as profiler:
|
||||
selected()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from pathlib import Path
|
|||
from typing import Final, cast
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.ocr import use_litellm_rust
|
||||
from litellm.rust_bridge.ocr import rust, set_rust_ocr
|
||||
from ......shared.parity.fixtures.recording import (
|
||||
RecordedInteraction,
|
||||
UpstreamEndpoint,
|
||||
|
|
@ -53,7 +53,8 @@ def main() -> None:
|
|||
parser.add_argument("--fixture-dir", type=Path, default=configured_fixture_directory())
|
||||
args: Final = parser.parse_args()
|
||||
directory: Final = cast(Path, args.fixture_dir)
|
||||
use_litellm_rust(False, ocr=None, aocr=None)
|
||||
rust(False)
|
||||
set_rust_ocr(ocr=None, aocr=None)
|
||||
paths: Final = tuple(sorted(directory.rglob("*.json")))
|
||||
for path in paths:
|
||||
print(f"Migrated {path.name} to {migrate_fixture(path).name}")
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from typing import Final, cast
|
|||
from dotenv import load_dotenv
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.ocr import use_litellm_rust
|
||||
from litellm.rust_bridge.ocr import rust, set_rust_ocr
|
||||
from ......shared.parity.fixtures.cli import parse_recording_args
|
||||
from ......shared.parity.fixtures.media import structured_image_data_uri
|
||||
from ......shared.parity.fixtures.pipeline import record_fixtures
|
||||
|
|
@ -67,7 +67,8 @@ def main() -> int:
|
|||
os.environ.get(FIXTURE_DIR_ENV),
|
||||
DEFAULT_FIXTURE_DIRECTORY,
|
||||
)
|
||||
use_litellm_rust(False, ocr=None, aocr=None)
|
||||
rust(False)
|
||||
set_rust_ocr(ocr=None, aocr=None)
|
||||
summary: Final = record_fixtures(targets, root, args.examples, args.concurrency, OcrParityCase)
|
||||
return summary.exit_code
|
||||
|
||||
|
|
|
|||
|
|
@ -165,9 +165,10 @@ def run_trace_cases(
|
|||
) -> tuple[int, HarnessRun]:
|
||||
selected_scenarios: Final = frozenset(runner_args)
|
||||
run: Final = HarnessRun.from_cases(cases)
|
||||
bridge_error: Final = ensure_trace_bridge(repo_root)
|
||||
runnable_cases: Final = tuple(case for case in cases if isinstance(case.spec, ModuleCaseSpec))
|
||||
bridge_error: Final = ensure_trace_bridge(repo_root) if runnable_cases else None
|
||||
if bridge_error is not None:
|
||||
for harness_case in cases:
|
||||
for harness_case in runnable_cases:
|
||||
_record_setup_failure(run, harness_case, bridge_error, "bridge")
|
||||
run.finished_at = monotonic()
|
||||
on_update(run)
|
||||
|
|
|
|||
|
|
@ -406,7 +406,7 @@ OCR_CONTRACT: Final = UnitTestContract(
|
|||
),
|
||||
exclusions=(
|
||||
UnitParityExclusionSpec(
|
||||
nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_use_litellm_rust_toggles_flag",
|
||||
nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_rust_toggles_flag",
|
||||
reason="This test asserts the process-level backend flag selected by the parity runner.",
|
||||
),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -106,15 +106,22 @@ def test_missing_model_parameter_curl(curl_command):
|
|||
# Run the curl command and capture the output
|
||||
key = generate_key_sync()
|
||||
curl_command = curl_command.replace("sk-1234", key)
|
||||
result = subprocess.run(curl_command, shell=True, capture_output=True, text=True)
|
||||
result = subprocess.run(
|
||||
f'{curl_command} -s -w "\\n%{{http_code}}"',
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
body, _, status_code = result.stdout.rpartition("\n")
|
||||
# Parse the JSON response
|
||||
response = json.loads(result.stdout)
|
||||
response = json.loads(body)
|
||||
|
||||
# Check that we got an error response
|
||||
assert "error" in response
|
||||
print("error in response", json.dumps(response, indent=4))
|
||||
|
||||
assert "litellm.BadRequestError" in response["error"]["message"]
|
||||
assert status_code == "400", f"expected HTTP 400, got {status_code}: {response}"
|
||||
assert isinstance(response["error"]["message"], str) and response["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -121,23 +121,25 @@ def _reset_rust_flag():
|
|||
|
||||
def test_load_rust_messages_returns_injected_impl():
|
||||
bridge = RecordingMessages()
|
||||
litellm.use_litellm_rust(True, messages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(messages=bridge)
|
||||
assert rust_messages.load_rust_messages() is bridge
|
||||
|
||||
|
||||
def test_bare_use_litellm_rust_still_toggles_ocr():
|
||||
def test_bare_rust_still_toggles_ocr():
|
||||
from litellm.rust_bridge.ocr import rust_ocr_enabled
|
||||
|
||||
litellm.use_litellm_rust(True)
|
||||
litellm.rust(True)
|
||||
assert rust_ocr_enabled() is True
|
||||
|
||||
litellm.use_litellm_rust(False)
|
||||
litellm.rust(False)
|
||||
assert rust_ocr_enabled() is False
|
||||
|
||||
|
||||
def test_load_rust_amessages_returns_injected_impl():
|
||||
bridge = RecordingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
assert rust_messages.load_rust_amessages() is bridge
|
||||
|
||||
|
||||
|
|
@ -147,7 +149,7 @@ def test_messages_wrapper_returns_none_when_bridge_absent(monkeypatch):
|
|||
"get_native_bridge",
|
||||
lambda: None,
|
||||
)
|
||||
litellm.use_litellm_rust(True)
|
||||
litellm.rust(True)
|
||||
assert rust_messages.load_rust_messages() is None
|
||||
result = rust_messages.messages(
|
||||
model="claude",
|
||||
|
|
@ -163,7 +165,8 @@ def test_messages_wrapper_returns_none_when_bridge_absent(monkeypatch):
|
|||
|
||||
def test_messages_wrapper_forwards_args_and_converts_timeout():
|
||||
bridge = RecordingMessages()
|
||||
litellm.use_litellm_rust(True, messages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(messages=bridge)
|
||||
|
||||
response = rust_messages.messages(
|
||||
model="claude-sonnet-4-5",
|
||||
|
|
@ -190,7 +193,8 @@ def test_messages_wrapper_forwards_args_and_converts_timeout():
|
|||
@pytest.mark.asyncio
|
||||
async def test_amessages_wrapper_forwards_args():
|
||||
bridge = RecordingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await rust_messages.amessages(
|
||||
model="claude-sonnet-4-5",
|
||||
|
|
@ -226,7 +230,8 @@ def _gate(**overrides):
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_invokes_rust_and_marks_response_header():
|
||||
bridge = RecordingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await _gate()
|
||||
|
||||
|
|
@ -245,7 +250,8 @@ async def test_gate_invokes_rust_and_marks_response_header():
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_falls_back_to_python_when_bridge_raises():
|
||||
bridge = RaisingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await _gate()
|
||||
|
||||
|
|
@ -268,7 +274,7 @@ async def test_gate_skips_rust_when_flag_absent():
|
|||
async def test_gate_uses_process_enable_without_request_override():
|
||||
bridge = RecordingAsyncMessages()
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
litellm.use_litellm_rust(True)
|
||||
litellm.rust(True)
|
||||
|
||||
response = await _gate(litellm_params=GenericLiteLLMParams(api_key="sk-azure"))
|
||||
|
||||
|
|
@ -279,7 +285,8 @@ async def test_gate_uses_process_enable_without_request_override():
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_skips_rust_when_flag_false():
|
||||
bridge = ExplodingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await _gate(litellm_params=GenericLiteLLMParams(api_key="sk-azure", rust=False))
|
||||
|
||||
|
|
@ -290,7 +297,8 @@ async def test_gate_skips_rust_when_flag_false():
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_invokes_rust_for_native_anthropic_provider():
|
||||
bridge = RecordingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await _gate(
|
||||
custom_llm_provider="anthropic",
|
||||
|
|
@ -339,7 +347,8 @@ async def test_gate_env_var_falsey_does_not_enable(monkeypatch):
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_skips_rust_for_unsupported_provider():
|
||||
bridge = ExplodingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await _gate(custom_llm_provider="openai")
|
||||
|
||||
|
|
@ -350,7 +359,8 @@ async def test_gate_skips_rust_for_unsupported_provider():
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_skips_rust_for_agentic_hook():
|
||||
bridge = ExplodingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
response = await _gate(has_agentic_hook=True)
|
||||
|
||||
|
|
@ -361,7 +371,8 @@ async def test_gate_skips_rust_for_agentic_hook():
|
|||
@pytest.mark.asyncio
|
||||
async def test_gate_streams_through_rust_when_eligible_and_strips_stream_flag():
|
||||
bridge = RecordingAsyncMessages()
|
||||
litellm.use_litellm_rust(True, amessages=bridge)
|
||||
litellm.rust(True)
|
||||
rust_messages.set_rust_messages(amessages=bridge)
|
||||
|
||||
streaming_body = {**REQUEST_BODY, "stream": True}
|
||||
response = await _gate(
|
||||
|
|
@ -398,7 +409,7 @@ async def test_gate_falls_back_when_bridge_unavailable(monkeypatch):
|
|||
"get_native_bridge",
|
||||
lambda: None,
|
||||
)
|
||||
litellm.use_litellm_rust(True)
|
||||
litellm.rust(True)
|
||||
|
||||
response = await _gate()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from importlib import import_module
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,15 +14,12 @@ def mock_gcs_dependencies():
|
|||
mock_async_client = AsyncMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.caching.gcs_cache._get_httpx_client", return_value=mock_sync_client
|
||||
patch.object(import_module("litellm.caching.gcs_cache"), "_get_httpx_client", return_value=mock_sync_client
|
||||
),
|
||||
patch(
|
||||
"litellm.caching.gcs_cache.get_async_httpx_client",
|
||||
patch.object(import_module("litellm.caching.gcs_cache"), "get_async_httpx_client",
|
||||
return_value=mock_async_client,
|
||||
),
|
||||
patch(
|
||||
"litellm.caching.gcs_cache.GCSBucketBase.sync_construct_request_headers",
|
||||
patch.object(import_module("litellm.caching.gcs_cache").GCSBucketBase, "sync_construct_request_headers",
|
||||
return_value={},
|
||||
),
|
||||
):
|
||||
|
|
|
|||
|
|
@ -495,7 +495,7 @@ def _closed_port() -> int:
|
|||
pytest.param(lambda c: c.async_get_ttl("lit4930"), id="async_get_ttl"),
|
||||
],
|
||||
)
|
||||
async def test_circuit_breaker_opens_when_method_swallows_redis_failure(redis_no_ping, call_method):
|
||||
async def test_circuit_breaker_opens_when_method_swallows_redis_failure(call_method):
|
||||
"""A guarded method that swallows its own Redis error must still count as a failure.
|
||||
|
||||
These methods catch connection errors and return a default so callers degrade instead
|
||||
|
|
@ -506,7 +506,7 @@ async def test_circuit_breaker_opens_when_method_swallows_redis_failure(redis_no
|
|||
"""
|
||||
from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD
|
||||
|
||||
cache = RedisCache(host="127.0.0.1", port=_closed_port(), socket_timeout=0.5)
|
||||
cache = await asyncio.to_thread(RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5)
|
||||
|
||||
for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD):
|
||||
await call_method(cache)
|
||||
|
|
@ -683,7 +683,7 @@ def test_call_stack_info_skips_guard_frames_when_deployed_without_sources(monkey
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_circuit_breaker_success_still_resets_the_failure_streak(redis_no_ping):
|
||||
async def test_circuit_breaker_success_still_resets_the_failure_streak():
|
||||
"""A reachable Redis must keep the breaker closed, however many earlier calls failed.
|
||||
|
||||
The guard now records success only when nothing failed while the method ran, so this
|
||||
|
|
@ -692,7 +692,7 @@ async def test_circuit_breaker_success_still_resets_the_failure_streak(redis_no_
|
|||
"""
|
||||
from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD
|
||||
|
||||
cache = RedisCache(host="127.0.0.1", port=_closed_port(), socket_timeout=0.5)
|
||||
cache = await asyncio.to_thread(RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5)
|
||||
|
||||
for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD - 1):
|
||||
await cache.async_get_cache("lit4930")
|
||||
|
|
@ -710,7 +710,7 @@ async def test_circuit_breaker_success_still_resets_the_failure_streak(redis_no_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_circuit_breaker_covers_lua_script_execution(redis_no_ping):
|
||||
async def test_circuit_breaker_covers_lua_script_execution():
|
||||
"""Lua script execution must feed the breaker like every other Redis call.
|
||||
|
||||
The v3 rate limiter issues all of its Redis traffic through async_register_script, so
|
||||
|
|
@ -722,7 +722,7 @@ async def test_circuit_breaker_covers_lua_script_execution(redis_no_ping):
|
|||
|
||||
from litellm.constants import REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD
|
||||
|
||||
cache = RedisCache(host="127.0.0.1", port=_closed_port(), socket_timeout=0.5)
|
||||
cache = await asyncio.to_thread(RedisCache, host="127.0.0.1", port=_closed_port(), socket_timeout=0.5)
|
||||
run_script = cache.async_register_script("return 1")
|
||||
|
||||
for _ in range(REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from importlib import import_module
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -64,7 +65,7 @@ async def test_redis_cluster_async_batch_get(mock_init_redis_cluster):
|
|||
|
||||
@patch("litellm._redis.get_redis_connection_pool")
|
||||
@patch("litellm._redis.get_redis_client")
|
||||
@patch("litellm.caching.redis_cache.RedisCache._setup_health_pings")
|
||||
@patch.object(import_module("litellm.caching.redis_cache").RedisCache, "_setup_health_pings")
|
||||
def test_cache_init_creates_cluster_cache_from_env_var(
|
||||
mock_health, mock_get_client, mock_get_pool, monkeypatch
|
||||
):
|
||||
|
|
@ -91,7 +92,7 @@ def test_cache_init_creates_cluster_cache_from_env_var(
|
|||
|
||||
@patch("litellm._redis.get_redis_connection_pool")
|
||||
@patch("litellm._redis.get_redis_client")
|
||||
@patch("litellm.caching.redis_cache.RedisCache._setup_health_pings")
|
||||
@patch.object(import_module("litellm.caching.redis_cache").RedisCache, "_setup_health_pings")
|
||||
def test_cache_init_creates_redis_cache_without_cluster_config(
|
||||
mock_health, mock_get_client, mock_get_pool, monkeypatch
|
||||
):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from importlib import import_module
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -92,7 +93,7 @@ def _make_redis_cache():
|
|||
patches = [
|
||||
patch("litellm._redis.get_redis_client", return_value=mock_sync_client),
|
||||
patch("litellm._redis.get_redis_connection_pool", return_value=mock_async_pool),
|
||||
patch("litellm.caching.redis_cache.RedisCache._setup_health_pings"),
|
||||
patch.object(import_module("litellm.caching.redis_cache").RedisCache, "_setup_health_pings"),
|
||||
]
|
||||
for p in patches:
|
||||
p.start()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from importlib import import_module
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -1453,7 +1454,7 @@ def test_cache_forwards_semantic_cache_embedding_timeout():
|
|||
from litellm.caching.caching import Cache
|
||||
from litellm.types.caching import LiteLLMCacheType
|
||||
|
||||
with patch("litellm.caching.caching.RedisSemanticCache") as backend:
|
||||
with patch.object(import_module("litellm.caching.caching"), "RedisSemanticCache") as backend:
|
||||
Cache(
|
||||
type=LiteLLMCacheType.REDIS_SEMANTIC,
|
||||
similarity_threshold=0.8,
|
||||
|
|
|
|||
|
|
@ -258,11 +258,9 @@ async def test_s3_cache_async_set_cache_pipeline(mock_s3_dependencies):
|
|||
|
||||
# Verify each call
|
||||
calls = cache.s3_client.put_object.call_args_list
|
||||
for i, (key, value) in enumerate(cache_list):
|
||||
call_args = calls[i][1]
|
||||
assert call_args["Bucket"] == "test-bucket"
|
||||
assert call_args["Key"] == key
|
||||
assert call_args["Body"] == json.dumps(value)
|
||||
assert {(call.kwargs["Bucket"], call.kwargs["Key"], call.kwargs["Body"]) for call in calls} == {
|
||||
("test-bucket", key, json.dumps(value)) for key, value in cache_list
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -285,10 +283,12 @@ async def test_s3_cache_concurrent_async_operations(mock_s3_dependencies):
|
|||
|
||||
# Verify each call had correct parameters
|
||||
calls = cache.s3_client.put_object.call_args_list
|
||||
for i, call in enumerate(calls):
|
||||
call_args = call[1]
|
||||
assert call_args["Bucket"] == "test-bucket"
|
||||
assert f"concurrent_key_{i}" == call_args["Key"]
|
||||
assert {call.kwargs["Key"] for call in calls} == {f"concurrent_key_{i}" for i in range(5)}
|
||||
for call in calls:
|
||||
assert call.kwargs["Bucket"] == "test-bucket"
|
||||
payload = json.loads(call.kwargs["Body"])
|
||||
assert call.kwargs["Key"] == f"concurrent_key_{payload['id']}"
|
||||
assert payload["data"] == f"test_data_{payload['id']}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import hashlib
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -74,6 +75,8 @@ class TestCloudZeroHourlyExport:
|
|||
fake_db = MagicMock()
|
||||
|
||||
async def query_raw_mock(query: str, *params):
|
||||
if "sha256(" in query:
|
||||
return []
|
||||
start_time_utc = params[0] if len(params) > 0 else None
|
||||
end_time_utc = params[1] if len(params) > 1 else None
|
||||
limit = params[2] if len(params) > 2 else None
|
||||
|
|
@ -146,6 +149,9 @@ class TestCloudZeroHourlyExport:
|
|||
return joined
|
||||
|
||||
fake_db.query_raw = AsyncMock(side_effect=query_raw_mock)
|
||||
fake_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
|
||||
fake_db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[])
|
||||
fake_db.litellm_usertable.find_many = AsyncMock(return_value=[])
|
||||
fake_client.db = fake_db
|
||||
mock_prisma_client_getter.return_value = fake_client
|
||||
|
||||
|
|
@ -160,3 +166,26 @@ class TestCloudZeroHourlyExport:
|
|||
logger = CloudZeroLogger(api_key="test", connection_id="test")
|
||||
|
||||
await logger._hourly_usage_data_export()
|
||||
|
||||
|
||||
class TestLiteLLMDatabaseUsageData:
|
||||
@pytest.mark.asyncio
|
||||
async def test_builds_frame_from_rows_recovered_for_double_hashed_keys(self, monkeypatch: pytest.MonkeyPatch):
|
||||
double_hashed = hashlib.sha256(b"sk-hashed-token").hexdigest()
|
||||
joined_row = {"api_key": "sk-joined", "api_key_alias": "joined", "team_id": "team-0", "user_email": None, "spend": 0.1}
|
||||
dirty_row = {"api_key": double_hashed, "api_key_alias": None, "team_id": None, "user_email": None, "spend": 0.5}
|
||||
|
||||
async def query_raw(query: str, *params):
|
||||
if "sha256(" in query:
|
||||
return [{"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": None}]
|
||||
return [joined_row, dirty_row]
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.db.query_raw = AsyncMock(side_effect=query_raw)
|
||||
db = LiteLLMDatabase()
|
||||
monkeypatch.setattr(db, "_ensure_prisma_client", lambda: fake_client)
|
||||
|
||||
result = await db.get_usage_data()
|
||||
|
||||
assert result["api_key_alias"].to_list() == ["joined", "batch-worker"]
|
||||
assert result["team_id"].to_list() == ["team-0", "team-1"]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Tests for FocusLiteLLMDatabase query construction."""
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
|
@ -87,3 +88,24 @@ async def test_should_join_organization_table(monkeypatch: pytest.MonkeyPatch):
|
|||
)
|
||||
assert "ot.organization_alias as organization_alias" in query_text
|
||||
assert 'LEFT JOIN "LiteLLM_OrganizationTable" ot' in query_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_build_frame_from_rows_recovered_for_double_hashed_keys(monkeypatch: pytest.MonkeyPatch):
|
||||
double_hashed = hashlib.sha256(b"sk-hashed-token").hexdigest()
|
||||
joined_row = {"api_key": "sk-joined", "api_key_alias": "joined", "team_id": "team-0", "user_email": None, "spend": 0.1}
|
||||
dirty_row = {"api_key": double_hashed, "api_key_alias": None, "team_id": None, "user_email": None, "spend": 0.5}
|
||||
|
||||
async def query_raw(query: str, *params):
|
||||
if "sha256(" in query:
|
||||
return [{"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": None}]
|
||||
return [joined_row, dirty_row]
|
||||
|
||||
mock_client = SimpleNamespace(db=SimpleNamespace(query_raw=AsyncMock(side_effect=query_raw)))
|
||||
db = FocusLiteLLMDatabase()
|
||||
monkeypatch.setattr(db, "_ensure_prisma_client", lambda: mock_client)
|
||||
|
||||
result = await db.get_usage_data()
|
||||
|
||||
assert result["api_key_alias"].to_list() == ["joined", "batch-worker"]
|
||||
assert result["team_id"].to_list() == ["team-0", "team-1"]
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ Covers:
|
|||
"""
|
||||
|
||||
import time
|
||||
from importlib import import_module
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -107,16 +108,16 @@ class TestResponsesStreamingIteratorMaxDuration:
|
|||
|
||||
def test_should_not_raise_when_duration_is_none(self):
|
||||
it = self._make_base_iterator()
|
||||
with patch(
|
||||
"litellm.responses.streaming_iterator.LITELLM_MAX_STREAMING_DURATION_SECONDS",
|
||||
with patch.object(
|
||||
import_module("litellm.responses.streaming_iterator"), "LITELLM_MAX_STREAMING_DURATION_SECONDS",
|
||||
None,
|
||||
):
|
||||
it._check_max_streaming_duration()
|
||||
|
||||
def test_should_not_raise_when_under_limit(self):
|
||||
it = self._make_base_iterator()
|
||||
with patch(
|
||||
"litellm.responses.streaming_iterator.LITELLM_MAX_STREAMING_DURATION_SECONDS",
|
||||
with patch.object(
|
||||
import_module("litellm.responses.streaming_iterator"), "LITELLM_MAX_STREAMING_DURATION_SECONDS",
|
||||
60.0,
|
||||
):
|
||||
it._check_max_streaming_duration()
|
||||
|
|
@ -124,8 +125,8 @@ class TestResponsesStreamingIteratorMaxDuration:
|
|||
def test_should_raise_timeout_when_exceeded(self):
|
||||
it = self._make_base_iterator()
|
||||
it._stream_created_time = time.time() - 20
|
||||
with patch(
|
||||
"litellm.responses.streaming_iterator.LITELLM_MAX_STREAMING_DURATION_SECONDS",
|
||||
with patch.object(
|
||||
import_module("litellm.responses.streaming_iterator"), "LITELLM_MAX_STREAMING_DURATION_SECONDS",
|
||||
10.0,
|
||||
):
|
||||
with pytest.raises(litellm.Timeout, match="max streaming duration"):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from importlib import import_module
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -163,8 +164,8 @@ async def test_anthropic_messages_with_mcp_forwards_the_callers_mcp_credentials(
|
|||
).LiteLLM_Proxy_MCP_Handler,
|
||||
"_process_mcp_tools_without_openai_transform",
|
||||
new=process,
|
||||
), patch(
|
||||
"litellm.responses.mcp.litellm_proxy_mcp_handler.LiteLLM_Proxy_MCP_Handler._execute_tool_calls",
|
||||
), patch.object(
|
||||
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls",
|
||||
new=execute,
|
||||
), patch(
|
||||
"litellm.anthropic_messages", new=AsyncMock(side_effect=responses)
|
||||
|
|
@ -218,11 +219,11 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped
|
|||
|
||||
with patch.object(
|
||||
MCPRequestContext, "resolve", return_value=MCPRequestContext(user_api_key_auth="auth")
|
||||
), patch(
|
||||
"litellm.responses.mcp.litellm_proxy_mcp_handler.LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform",
|
||||
), patch.object(
|
||||
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, "_process_mcp_tools_without_openai_transform",
|
||||
new=AsyncMock(return_value=([], {})),
|
||||
), patch(
|
||||
"litellm.responses.mcp.litellm_proxy_mcp_handler.LiteLLM_Proxy_MCP_Handler._execute_tool_calls",
|
||||
), patch.object(
|
||||
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls",
|
||||
new=AsyncMock(return_value=[]),
|
||||
), patch(
|
||||
"litellm.anthropic_messages", new=anthropic_messages_mock
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ Coverage:
|
|||
|
||||
import base64
|
||||
from typing import Any, Dict, List, Optional
|
||||
from importlib import import_module
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -223,28 +224,28 @@ class TestFileSearchGuardInResponsesMain:
|
|||
expected = {"ok": True}
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.responses.main.litellm.get_llm_provider",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").litellm, "get_llm_provider",
|
||||
return_value=("claude-sonnet-4-5", "anthropic", None, None),
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.update_responses_input_with_model_file_ids",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main"), "update_responses_input_with_model_file_ids",
|
||||
return_value="hello",
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.update_responses_tools_with_model_file_ids",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main"), "update_responses_tools_with_model_file_ids",
|
||||
return_value=tools,
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.ResponsesAPIRequestUtils.get_requested_response_api_optional_param",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").ResponsesAPIRequestUtils, "get_requested_response_api_optional_param",
|
||||
return_value={},
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.run_async_function", return_value=expected
|
||||
patch.object(
|
||||
import_module("litellm.responses.main"), "run_async_function", return_value=expected
|
||||
) as run_async_mock,
|
||||
):
|
||||
result = responses(
|
||||
|
|
@ -274,28 +275,28 @@ class TestFileSearchGuardInResponsesMain:
|
|||
mock_config.supports_native_file_search.return_value = False
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.responses.main.litellm.get_llm_provider",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").litellm, "get_llm_provider",
|
||||
return_value=("claude-sonnet-4-5", "anthropic", None, None),
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.update_responses_input_with_model_file_ids",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main"), "update_responses_input_with_model_file_ids",
|
||||
return_value="hello",
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.update_responses_tools_with_model_file_ids",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main"), "update_responses_tools_with_model_file_ids",
|
||||
return_value=tools,
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config",
|
||||
return_value=mock_config,
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.ResponsesAPIRequestUtils.get_requested_response_api_optional_param",
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").ResponsesAPIRequestUtils, "get_requested_response_api_optional_param",
|
||||
return_value={},
|
||||
),
|
||||
patch(
|
||||
"litellm.responses.main.run_async_function", return_value=expected
|
||||
patch.object(
|
||||
import_module("litellm.responses.main"), "run_async_function", return_value=expected
|
||||
) as run_async_mock,
|
||||
):
|
||||
result = responses(
|
||||
|
|
@ -758,8 +759,8 @@ class TestEmulatedFileSearchHandler:
|
|||
mock_search_response.data = [search_result]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.responses.file_search.emulated_handler._call_aresponses",
|
||||
patch.object(
|
||||
import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses",
|
||||
new=AsyncMock(side_effect=[first_resp, final_resp]),
|
||||
),
|
||||
patch(
|
||||
|
|
@ -821,8 +822,8 @@ class TestEmulatedFileSearchHandler:
|
|||
mock_search_response.data = [search_result]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.responses.file_search.emulated_handler._call_aresponses",
|
||||
patch.object(
|
||||
import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses",
|
||||
new=AsyncMock(side_effect=[first_resp_plural, final_resp]),
|
||||
),
|
||||
patch(
|
||||
|
|
@ -855,8 +856,8 @@ class TestEmulatedFileSearchHandler:
|
|||
text="I already know the answer."
|
||||
)
|
||||
|
||||
with patch(
|
||||
"litellm.responses.file_search.emulated_handler._call_aresponses",
|
||||
with patch.object(
|
||||
import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses",
|
||||
new=AsyncMock(return_value=direct_resp),
|
||||
):
|
||||
result = await aresponses_with_emulated_file_search(
|
||||
|
|
@ -905,8 +906,8 @@ class TestEmulatedFileSearchHandler:
|
|||
mock_search_response.data = [search_result]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.responses.file_search.emulated_handler._call_aresponses",
|
||||
patch.object(
|
||||
import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses",
|
||||
new=AsyncMock(side_effect=[first_resp, final_resp]),
|
||||
) as mock_call,
|
||||
patch(
|
||||
|
|
|
|||
|
|
@ -228,7 +228,8 @@ def _reset_rust_flag():
|
|||
def fake_bridge():
|
||||
"""Enable the Rust path with an injected recording bridge (no native wheel)."""
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
return bridge
|
||||
|
||||
|
||||
|
|
@ -236,15 +237,16 @@ def fake_bridge():
|
|||
def fake_async_bridge():
|
||||
"""Enable the async Rust path with an injected recording bridge."""
|
||||
bridge = RecordingAsyncBridge()
|
||||
litellm.use_litellm_rust(True, aocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(aocr=bridge)
|
||||
return bridge
|
||||
|
||||
|
||||
def test_use_litellm_rust_toggles_flag():
|
||||
def test_rust_toggles_flag():
|
||||
assert rust_bridge.rust_ocr_enabled() is False
|
||||
litellm.use_litellm_rust()
|
||||
litellm.rust(True)
|
||||
assert rust_bridge.rust_ocr_enabled() is True
|
||||
litellm.use_litellm_rust(False)
|
||||
litellm.rust(False)
|
||||
assert rust_bridge.rust_ocr_enabled() is False
|
||||
|
||||
|
||||
|
|
@ -255,14 +257,15 @@ def test_env_var_enables_rust_ocr(monkeypatch):
|
|||
|
||||
|
||||
def test_explicit_false_overrides_process_enable():
|
||||
litellm.use_litellm_rust(True)
|
||||
litellm.rust(True)
|
||||
|
||||
assert ocr_main._rust_ocr_enabled(build_prepared_request(litellm_params={"rust": False})) is False
|
||||
|
||||
|
||||
def test_load_rust_ocr_returns_injected_impl():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
assert rust_bridge.load_rust_ocr() is bridge
|
||||
|
||||
|
||||
|
|
@ -325,25 +328,22 @@ def test_native_bridge_available_reflects_loader(monkeypatch):
|
|||
|
||||
def test_load_rust_aocr_returns_injected_impl():
|
||||
bridge = RecordingAsyncBridge()
|
||||
litellm.use_litellm_rust(True, aocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(aocr=bridge)
|
||||
assert rust_bridge.load_rust_aocr() is bridge
|
||||
|
||||
|
||||
def test_toggle_without_ocr_arg_preserves_injected_impl():
|
||||
"""Regression: routine enable/disable calls must not clobber a prior injection.
|
||||
|
||||
Earlier, ``use_litellm_rust()`` unconditionally assigned the keyword default
|
||||
of ``None`` to ``_rust_ocr_impl``, silently dropping a custom bridge whenever
|
||||
a caller toggled the flag without re-passing ``ocr=``.
|
||||
"""
|
||||
"""The public flag must not clobber an internal test binding."""
|
||||
bridge = RecordingBridge()
|
||||
async_bridge = RecordingAsyncBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge, aocr=async_bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge, aocr=async_bridge)
|
||||
|
||||
litellm.use_litellm_rust(False)
|
||||
litellm.rust(False)
|
||||
assert rust_bridge.load_rust_ocr() is bridge
|
||||
assert rust_bridge.load_rust_aocr() is async_bridge
|
||||
litellm.use_litellm_rust(True)
|
||||
litellm.rust(True)
|
||||
assert rust_bridge.load_rust_ocr() is bridge
|
||||
assert rust_bridge.load_rust_aocr() is async_bridge
|
||||
|
||||
|
|
@ -356,9 +356,10 @@ def test_explicit_ocr_none_clears_injected_impl(monkeypatch):
|
|||
)
|
||||
bridge = RecordingBridge()
|
||||
async_bridge = RecordingAsyncBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge, aocr=async_bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge, aocr=async_bridge)
|
||||
|
||||
litellm.use_litellm_rust(True, ocr=None, aocr=None)
|
||||
rust_bridge.set_rust_ocr(ocr=None, aocr=None)
|
||||
assert rust_bridge.load_rust_ocr() is None
|
||||
assert rust_bridge.load_rust_aocr() is None
|
||||
|
||||
|
|
@ -371,7 +372,7 @@ def test_load_rust_ocr_none_when_extension_absent(monkeypatch):
|
|||
"get_native_bridge",
|
||||
lambda: None,
|
||||
)
|
||||
litellm.use_litellm_rust(True) # no impl injected; extension isn't built in CI
|
||||
litellm.rust(True) # no impl injected; extension isn't built in CI
|
||||
assert rust_bridge.load_rust_ocr() is None
|
||||
assert rust_bridge.load_rust_aocr() is None
|
||||
|
||||
|
|
@ -389,7 +390,7 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch):
|
|||
lambda: fake_module,
|
||||
)
|
||||
|
||||
litellm.use_litellm_rust(True) # enabled, no impl injected -> import the extension
|
||||
litellm.rust(True) # enabled, no impl injected -> import the extension
|
||||
assert rust_bridge.load_rust_ocr() is fake_module.ocr
|
||||
assert rust_bridge.load_rust_aocr() is fake_module.aocr
|
||||
|
||||
|
|
@ -403,7 +404,9 @@ def test_timeout_to_seconds_handles_float_timeout_and_none():
|
|||
def test_bridge_wrapper_forwards_prepared_args_and_wraps_response():
|
||||
bridge = RecordingBridge()
|
||||
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
response = rust_bridge.ocr(
|
||||
model="mistral-ocr-latest",
|
||||
document=DOCUMENT,
|
||||
|
|
@ -436,7 +439,9 @@ def test_bridge_wrapper_forwards_prepared_args_and_wraps_response():
|
|||
async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response():
|
||||
bridge = RecordingAsyncBridge()
|
||||
|
||||
litellm.use_litellm_rust(True, aocr=bridge)
|
||||
litellm.rust(True)
|
||||
|
||||
rust_bridge.set_rust_ocr(aocr=bridge)
|
||||
response = await rust_bridge.aocr(
|
||||
model="mistral-ocr-maas",
|
||||
document=DOCUMENT,
|
||||
|
|
@ -464,7 +469,8 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response():
|
|||
def test_run_rust_ocr_prepares_request_and_wraps_response():
|
||||
bridge = RecordingBridge()
|
||||
logging_obj = RecordingLogging()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
response = ocr_main._run_rust_ocr(
|
||||
prepared_request=build_prepared_request(
|
||||
|
|
@ -496,7 +502,8 @@ def test_run_rust_ocr_prepares_request_and_wraps_response():
|
|||
|
||||
def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
ocr_main._run_rust_ocr(
|
||||
prepared_request=build_prepared_request(api_key=None, timeout=None),
|
||||
|
|
@ -508,7 +515,8 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing():
|
|||
|
||||
def test_run_rust_ocr_prefers_explicit_key_over_resolver():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
def _resolver(name: str) -> str | None:
|
||||
raise AssertionError(f"resolver should not be called for {name}")
|
||||
|
|
@ -527,7 +535,8 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver():
|
|||
def test_run_rust_ocr_uses_provider_api_key_env_var():
|
||||
bridge = RecordingBridge()
|
||||
resolver_calls = []
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
def _resolver(name):
|
||||
resolver_calls.append(name)
|
||||
|
|
@ -549,7 +558,8 @@ def test_run_rust_ocr_uses_provider_api_key_env_var():
|
|||
|
||||
def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
ocr_main._run_rust_ocr(
|
||||
prepared_request=build_prepared_request(
|
||||
|
|
@ -575,7 +585,8 @@ def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata():
|
|||
|
||||
def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
def _resolver(name: str) -> str | None:
|
||||
return {
|
||||
|
|
@ -598,7 +609,8 @@ def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_mana
|
|||
|
||||
def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
ocr_main._run_rust_ocr(
|
||||
prepared_request=build_prepared_request(
|
||||
|
|
@ -615,7 +627,8 @@ def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager():
|
|||
|
||||
def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint():
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
ocr_main._run_rust_ocr(
|
||||
prepared_request=build_prepared_request(
|
||||
|
|
@ -635,7 +648,8 @@ def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint():
|
|||
def test_run_rust_ocr_runs_pre_call_logging():
|
||||
logging_obj = RecordingLogging()
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(True, ocr=bridge)
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
ocr_main._run_rust_ocr(
|
||||
prepared_request=build_prepared_request(
|
||||
|
|
@ -722,7 +736,8 @@ def test_ocr_exception_type_uses_resolved_provider_context(
|
|||
return CapturedException("wrapped")
|
||||
|
||||
monkeypatch.setattr(ocr_main.litellm, "exception_type", fake_exception_type)
|
||||
litellm.use_litellm_rust(True, ocr=RaisingBridge())
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(ocr=RaisingBridge())
|
||||
|
||||
with pytest.raises(CapturedException):
|
||||
litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test")
|
||||
|
|
@ -767,7 +782,8 @@ async def test_aocr_exception_type_uses_resolved_provider_context(
|
|||
return CapturedException("wrapped")
|
||||
|
||||
monkeypatch.setattr(ocr_main.litellm, "exception_type", fake_exception_type)
|
||||
litellm.use_litellm_rust(True, aocr=RaisingAsyncBridge())
|
||||
litellm.rust(True)
|
||||
rust_bridge.set_rust_ocr(aocr=RaisingAsyncBridge())
|
||||
|
||||
with pytest.raises(CapturedException):
|
||||
await litellm.aocr(model=MODEL, document=DOCUMENT, api_key="sk-test")
|
||||
|
|
@ -795,7 +811,8 @@ def test_ocr_passes_default_request_timeout_to_rust(fake_bridge):
|
|||
def test_ocr_does_not_route_to_rust_when_disabled():
|
||||
"""With the flag off, the bridge must not be consulted even if an impl exists."""
|
||||
bridge = RecordingBridge()
|
||||
litellm.use_litellm_rust(False, ocr=bridge)
|
||||
litellm.rust(False)
|
||||
rust_bridge.set_rust_ocr(ocr=bridge)
|
||||
|
||||
assert rust_bridge.rust_ocr_enabled() is False
|
||||
# The impl stays available for injection, but the disabled flag gates usage,
|
||||
|
|
@ -807,7 +824,7 @@ def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch):
|
|||
"""Rust enabled but no bridge available (no injected impl, no compiled wheel):
|
||||
ocr() must degrade to the Python HTTP handler instead of raising."""
|
||||
monkeypatch.setattr(rust_bridge, "load_rust_ocr", lambda: None)
|
||||
litellm.use_litellm_rust(True) # enabled, but load_rust_ocr() returns None in CI
|
||||
litellm.rust(True) # enabled, but load_rust_ocr() returns None in CI
|
||||
|
||||
captured = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
"""Unit tests for MCP OAuth passthrough tool-fetch behavior."""
|
||||
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
if sys.version_info < (3, 11):
|
||||
from exceptiongroup import ExceptionGroup
|
||||
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
MCPServerManager,
|
||||
|
|
@ -37,7 +42,7 @@ def test_extract_upstream_auth_failure_walks_exception_group():
|
|||
inner = httpx.HTTPStatusError("401", request=response.request, response=response)
|
||||
|
||||
try:
|
||||
raise ExceptionGroup("wrapped", [inner]) # noqa: F821 (PEP 654, py3.11+)
|
||||
raise ExceptionGroup("wrapped", [inner])
|
||||
except Exception as group:
|
||||
result = _extract_upstream_auth_failure(group)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11
|
|||
from mcp.types import Tool as MCPTool
|
||||
|
||||
|
||||
requires_semantic_router = pytest.mark.skipif(
|
||||
sys.version_info >= (3, 14), reason="The semantic-router extra excludes Python 3.14"
|
||||
)
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_basic_filtering():
|
||||
"""
|
||||
|
|
@ -145,6 +151,7 @@ async def test_semantic_filter_basic_filtering():
|
|||
print(f" Filter respects top_k parameter correctly")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_top_k_limiting():
|
||||
"""
|
||||
|
|
@ -328,6 +335,7 @@ async def test_semantic_filter_extract_user_query():
|
|||
assert query3 == ""
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_triggers_on_completion():
|
||||
"""
|
||||
|
|
@ -453,6 +461,7 @@ async def test_semantic_filter_hook_skips_no_tools():
|
|||
print("✅ Hook correctly skips requests without tools")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_preserves_native_tools():
|
||||
"""
|
||||
|
|
@ -584,6 +593,7 @@ async def test_semantic_filter_hook_preserves_native_tools():
|
|||
)
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_all_native_tools():
|
||||
"""
|
||||
|
|
@ -684,6 +694,7 @@ async def test_semantic_filter_hook_all_native_tools():
|
|||
)
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_responses_api_name_collision():
|
||||
"""
|
||||
|
|
@ -774,6 +785,7 @@ async def test_semantic_filter_hook_responses_api_name_collision():
|
|||
print("✅ Responses API tool with MCP-matching name correctly classified as native")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_filters_expanded_litellm_proxy_tools():
|
||||
"""
|
||||
|
|
@ -889,6 +901,7 @@ async def test_semantic_filter_hook_filters_expanded_litellm_proxy_tools():
|
|||
print(f"✅ Expanded litellm_proxy tools filtered: {len(expanded_tools)} -> {len(allowed_tools)}, stats={stats}")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_narrows_mcp_reference_for_chat_completions():
|
||||
"""
|
||||
|
|
@ -1008,6 +1021,7 @@ async def test_semantic_filter_hook_narrows_mcp_reference_for_chat_completions()
|
|||
print(f"✅ chat completions: MCP reference preserved, narrowed to {allowed_tools}")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_zero_matches_exposes_all_tools_on_both_paths():
|
||||
"""
|
||||
|
|
@ -1126,6 +1140,7 @@ async def test_semantic_filter_hook_zero_matches_exposes_all_tools_on_both_paths
|
|||
print("✅ zero matches: both the MCP reference path and the plain tool path expose every tool")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_filters_expanded_tools_with_string_input():
|
||||
"""
|
||||
|
|
@ -1266,6 +1281,7 @@ async def test_semantic_filter_hook_expansion_skips_filter_when_disabled():
|
|||
print("✅ Disabled filter: MCP reference untouched, no spurious stats")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_preserves_tool_order():
|
||||
"""
|
||||
|
|
@ -1651,6 +1667,7 @@ def _make_context_window_filter(state, top_k: int = 3):
|
|||
)
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_fails_closed_on_query_time_context_window_error():
|
||||
"""
|
||||
|
|
@ -1682,6 +1699,7 @@ async def test_semantic_filter_fails_closed_on_query_time_context_window_error()
|
|||
print("✅ Query-time context window overflow fails closed")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_records_build_time_context_window_error():
|
||||
"""
|
||||
|
|
@ -1715,6 +1733,7 @@ async def test_semantic_filter_records_build_time_context_window_error():
|
|||
print("✅ Build-time context window overflow is recorded and fails closed")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_fails_closed_on_context_window_error():
|
||||
"""
|
||||
|
|
@ -1762,6 +1781,7 @@ async def test_semantic_filter_hook_fails_closed_on_context_window_error():
|
|||
print("✅ Hook fails closed with actionable 400 on context window overflow")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_fails_closed_on_expanded_tools_context_window_error():
|
||||
"""
|
||||
|
|
@ -1828,6 +1848,7 @@ async def test_semantic_filter_hook_fails_closed_on_expanded_tools_context_windo
|
|||
print("✅ Expansion path fails closed with actionable 400 on context window overflow")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_ignores_build_error_for_native_only_tools():
|
||||
"""
|
||||
|
|
@ -2018,6 +2039,7 @@ def _weather_tool():
|
|||
)
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_filter_indexes_request_tools_when_startup_index_is_empty():
|
||||
"""
|
||||
|
|
@ -2042,6 +2064,7 @@ async def test_filter_indexes_request_tools_when_startup_index_is_empty():
|
|||
print("✅ Empty startup index is built from authed request-time tools")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_filter_indexes_tools_missing_from_partial_index():
|
||||
"""
|
||||
|
|
@ -2070,6 +2093,7 @@ async def test_filter_indexes_tools_missing_from_partial_index():
|
|||
print("✅ Partial startup index is completed from request-time tools, embedding each tool once")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_filter_fails_open_when_matches_are_not_in_available_tools():
|
||||
"""
|
||||
|
|
@ -2093,6 +2117,7 @@ async def test_filter_fails_open_when_matches_are_not_in_available_tools():
|
|||
print("✅ Matches outside available_tools fail open instead of dropping every tool")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_time_context_window_error_is_request_scoped():
|
||||
"""
|
||||
|
|
@ -2129,6 +2154,7 @@ async def test_request_time_context_window_error_is_request_scoped():
|
|||
print("✅ Request-time context window overflow is scoped to the request, not the worker")
|
||||
|
||||
|
||||
@requires_semantic_router
|
||||
@pytest.mark.asyncio
|
||||
async def test_foreign_index_routes_cannot_displace_available_tools():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3312,6 +3312,41 @@ def test_user_daily_activity_routes_reachable_by_non_admin(route, user_role):
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"user_role",
|
||||
[
|
||||
LitellmUserRoles.INTERNAL_USER.value,
|
||||
LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
|
||||
],
|
||||
)
|
||||
def test_team_spend_by_user_reachable_by_non_admin(user_role):
|
||||
user_obj = LiteLLM_UserTable(
|
||||
user_id="test_user",
|
||||
user_email="test@example.com",
|
||||
user_role=user_role,
|
||||
)
|
||||
valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role)
|
||||
request = MagicMock(spec=Request)
|
||||
request.query_params = {}
|
||||
|
||||
def outcome(route: str) -> str:
|
||||
try:
|
||||
RouteChecks.non_proxy_admin_allowed_routes_check(
|
||||
user_obj=user_obj,
|
||||
_user_role=user_role,
|
||||
route=route,
|
||||
request=request,
|
||||
valid_token=valid_token,
|
||||
request_data={},
|
||||
)
|
||||
except Exception as exc:
|
||||
return f"denied: {exc}"
|
||||
return "allowed"
|
||||
|
||||
assert outcome("/team/spend/by_user") == "allowed"
|
||||
assert outcome("/team/spend/by_key").startswith("denied: Only proxy admin")
|
||||
|
||||
|
||||
def test_user_daily_activity_aggregated_not_covered_by_prefix_match():
|
||||
"""check_route_access is exact-match plus explicit wildcards, so listing the
|
||||
parent /user/daily/activity does not implicitly cover the /aggregated
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ def test_supported_hooks_limited_to_pre_and_post():
|
|||
|
||||
|
||||
def test_during_call_mode_rejected_at_init():
|
||||
with pytest.raises(ValueError, match='Event hook GuardrailEventHooks\\.during_call is not in the'):
|
||||
with pytest.raises(ValueError, match="during_call is not in the supported event hooks"):
|
||||
StraikerGuardrail(api_key="k", event_hook="during_call")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ def test_build_logging_obj_seeds_model_call_details():
|
|||
# Metadata is mapped to the keys the cost-tracking callback reads.
|
||||
md = details["litellm_params"]["metadata"]
|
||||
assert md["user_api_key"] == "rust-gateway-test-key"
|
||||
assert md["user_api_key_hash"] == "rust-gateway-test-key"
|
||||
assert md["user_api_key_user_id"] == "user-cb-logs-test"
|
||||
assert md["user_api_key_team_id"] == "team-cb-logs-test"
|
||||
|
||||
|
|
|
|||
|
|
@ -454,6 +454,151 @@ async def test_get_api_key_metadata_regenerated_key_uses_most_recent_deleted_rec
|
|||
assert result["old-key-hash"]["team_id"] == "latest-team"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_api_key_metadata_recovers_double_hashed_key_via_reverse_hash():
|
||||
"""
|
||||
v1.99 spend logging re-hashed already-hashed api_key values when provenance was
|
||||
missing. Usage joins DailyUserSpend.api_key to VerificationToken.token, so those
|
||||
rows looked like key-hash-... with a null alias. Recovery asks Postgres for the
|
||||
key whose hashed token matches the dirty value and maps it back to its alias.
|
||||
"""
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
double_hashed = hash_token("a" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
|
||||
mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[])
|
||||
mock_prisma.db.litellm_usertable.find_many = AsyncMock(
|
||||
return_value=[SimpleNamespace(user_id="alice", user_email="alice@example.com")]
|
||||
)
|
||||
mock_prisma.db.query_raw = AsyncMock(
|
||||
return_value=[
|
||||
{"digest": double_hashed, "key_alias": "batch-worker", "team_id": "team-1", "user_id": "alice"}
|
||||
]
|
||||
)
|
||||
|
||||
result = await get_api_key_metadata(
|
||||
prisma_client=mock_prisma,
|
||||
api_keys={double_hashed},
|
||||
)
|
||||
|
||||
assert result[double_hashed]["key_alias"] == "batch-worker"
|
||||
assert result[double_hashed]["team_id"] == "team-1"
|
||||
assert result[double_hashed]["user_email"] == "alice@example.com"
|
||||
((digest_sql, digests),) = [call.args for call in mock_prisma.db.query_raw.call_args_list]
|
||||
assert '"LiteLLM_VerificationToken"' in digest_sql
|
||||
assert digests == [double_hashed]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_api_key_metadata_permanent_miss_never_pages_tokens_or_reads_spend_logs():
|
||||
"""A dirty key no table can explain costs two digest lookups, never a token page walk or a SpendLogs scan."""
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
double_hashed = hash_token("b" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
|
||||
mock_prisma.db.litellm_deletedverificationtoken.find_many = AsyncMock(return_value=[])
|
||||
mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[])
|
||||
mock_prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
|
||||
result = await get_api_key_metadata(
|
||||
prisma_client=mock_prisma,
|
||||
api_keys={double_hashed},
|
||||
)
|
||||
|
||||
assert double_hashed not in result
|
||||
issued_sql = [call.args[0] for call in mock_prisma.db.query_raw.call_args_list]
|
||||
assert len(issued_sql) == 2
|
||||
assert not any("LiteLLM_SpendLogs" in sql for sql in issued_sql)
|
||||
token_lookups = (
|
||||
mock_prisma.db.litellm_verificationtoken.find_many.call_args_list
|
||||
+ mock_prisma.db.litellm_deletedverificationtoken.find_many.call_args_list
|
||||
)
|
||||
assert all("take" not in call.kwargs and "skip" not in call.kwargs for call in token_lookups)
|
||||
|
||||
|
||||
def test_key_metadata_includes_recovered_user_email():
|
||||
from litellm.proxy.management_endpoints.common_daily_activity import _key_metadata
|
||||
|
||||
meta = _key_metadata(
|
||||
{
|
||||
"dirty-key": {
|
||||
"key_alias": "batch-worker",
|
||||
"team_id": "team-1",
|
||||
"user_email": "alice@example.com",
|
||||
}
|
||||
},
|
||||
"dirty-key",
|
||||
)
|
||||
|
||||
assert meta.key_alias == "batch-worker"
|
||||
assert meta.user_email == "alice@example.com"
|
||||
|
||||
|
||||
def test_update_breakdown_metrics_includes_user_email():
|
||||
from litellm.proxy.management_endpoints.common_daily_activity import update_breakdown_metrics
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import BreakdownMetrics
|
||||
|
||||
breakdown = BreakdownMetrics()
|
||||
record = SimpleNamespace(
|
||||
api_key="dirty-key",
|
||||
model="gpt-4o-mini",
|
||||
model_group="grp",
|
||||
mcp_namespaced_tool_name="srv/tool",
|
||||
custom_llm_provider="openai",
|
||||
endpoint="/v1/chat/completions",
|
||||
spend=1.23,
|
||||
prompt_tokens=1,
|
||||
completion_tokens=1,
|
||||
cache_read_input_tokens=0,
|
||||
cache_creation_input_tokens=0,
|
||||
compression_saved_tokens=0,
|
||||
compression_savings_spend=0,
|
||||
prompt_caching_savings_spend=0,
|
||||
gateway_injected_caching_savings_spend=0,
|
||||
autorouter_savings_spend=0,
|
||||
total_tokens=2,
|
||||
api_requests=1,
|
||||
successful_requests=1,
|
||||
failed_requests=0,
|
||||
ptu_flat_cost=0.0,
|
||||
user_id="alice",
|
||||
)
|
||||
api_key_metadata = {
|
||||
"dirty-key": {
|
||||
"key_alias": "batch-worker",
|
||||
"team_id": "team-1",
|
||||
"user_email": "alice@example.com",
|
||||
}
|
||||
}
|
||||
|
||||
update_breakdown_metrics(
|
||||
breakdown,
|
||||
record,
|
||||
{},
|
||||
{},
|
||||
api_key_metadata,
|
||||
entity_id_field="user_id",
|
||||
)
|
||||
|
||||
expected = ("batch-worker", "alice@example.com")
|
||||
top = breakdown.api_keys["dirty-key"].metadata
|
||||
assert (top.key_alias, top.user_email) == expected
|
||||
assert (
|
||||
breakdown.models["gpt-4o-mini"].api_key_breakdown["dirty-key"].metadata.key_alias,
|
||||
breakdown.models["gpt-4o-mini"].api_key_breakdown["dirty-key"].metadata.user_email,
|
||||
) == expected
|
||||
assert (
|
||||
breakdown.providers["openai"].api_key_breakdown["dirty-key"].metadata.key_alias,
|
||||
breakdown.providers["openai"].api_key_breakdown["dirty-key"].metadata.user_email,
|
||||
) == expected
|
||||
assert (
|
||||
breakdown.entities["alice"].api_key_breakdown["dirty-key"].metadata.key_alias,
|
||||
breakdown.entities["alice"].api_key_breakdown["dirty-key"].metadata.user_email,
|
||||
) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_daily_activity_metadata_totals_not_zero():
|
||||
"""Test that tag daily activity returns correct metadata totals.
|
||||
|
|
|
|||
|
|
@ -13598,3 +13598,175 @@ async def test_team_member_update_skips_invalidation_when_no_budget_fields_sent(
|
|||
|
||||
assert await real_cache.async_get_cache(key="team-1_member-1") == "still-fresh-membership"
|
||||
assert real_spend_counter_cache.in_memory_cache.get_cache(key="spend:team_member:member-1:team-1") == 1.5
|
||||
|
||||
|
||||
def _team_spend_by_user_team(team_id: str, team_alias: str, member: Member, permissions: list[str]) -> MagicMock:
|
||||
team = MagicMock(spec=LiteLLM_TeamTable)
|
||||
team.team_id = team_id
|
||||
team.team_alias = team_alias
|
||||
team.members_with_roles = [member]
|
||||
team.team_member_permissions = permissions
|
||||
team.model_dump.return_value = {
|
||||
"team_id": team_id,
|
||||
"team_alias": team_alias,
|
||||
"members_with_roles": [{"user_id": member.user_id, "role": member.role}],
|
||||
"team_member_permissions": permissions,
|
||||
}
|
||||
return team
|
||||
|
||||
|
||||
def _team_spend_by_user_caller(user_id: str, teams: list[str]) -> LiteLLM_UserTable:
|
||||
return LiteLLM_UserTable(
|
||||
user_id=user_id, user_email=f"{user_id}@example.com", teams=teams, user_role="internal_user"
|
||||
)
|
||||
|
||||
|
||||
def _team_spend_by_user_db_row(team_id: str, user_id: str, spend: float, requests: int) -> dict:
|
||||
return {
|
||||
"team_id": team_id,
|
||||
"user_id": user_id,
|
||||
"user_email": f"{user_id}@example.com",
|
||||
"user_alias": None,
|
||||
"spend": spend,
|
||||
"prompt_tokens": 10 * requests,
|
||||
"completion_tokens": 5 * requests,
|
||||
"total_tokens": 15 * requests,
|
||||
"api_requests": requests,
|
||||
"successful_requests": requests - 1,
|
||||
"failed_requests": 1,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(mock_db_client):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
|
||||
beta = _team_spend_by_user_team("team-beta", "Team Beta", Member(user_id="alice", role="user"), [])
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha, beta])
|
||||
mock_db_client.db.query_raw = AsyncMock(
|
||||
return_value=[
|
||||
_team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3),
|
||||
_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2),
|
||||
_team_spend_by_user_db_row("team-beta", "alice", 0.1, 1),
|
||||
]
|
||||
)
|
||||
|
||||
response = await get_team_spend_by_user(
|
||||
user_api_key_dict=admin,
|
||||
team_ids="team-alpha,team-beta",
|
||||
start_date="2026-09-01",
|
||||
end_date="2026-09-04",
|
||||
)
|
||||
|
||||
sql, *params = mock_db_client.db.query_raw.call_args.args
|
||||
assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta"]
|
||||
assert 'FROM "LiteLLM_SpendLogs" sl' in sql
|
||||
assert 'sl."startTime" >= $1::timestamp' in sql
|
||||
assert "sl.\"startTime\" < $2::timestamp + INTERVAL '1 day'" in sql
|
||||
assert "sl.team_id IN ($3, $4)" in sql
|
||||
assert 'GROUP BY sl.team_id, sl."user"' in sql
|
||||
assert 'sl."user" = $' not in sql
|
||||
|
||||
assert response.start_date == "2026-09-01"
|
||||
assert response.end_date == "2026-09-04"
|
||||
assert [(r.team_id, r.team_alias, r.user_id, r.user_email, r.spend, r.api_requests) for r in response.results] == [
|
||||
("team-alpha", "Team Alpha", "alice", "alice@example.com", 0.5, 3),
|
||||
("team-alpha", "Team Alpha", "bob", "bob@example.com", 0.25, 2),
|
||||
("team-beta", "Team Beta", "alice", "alice@example.com", 0.1, 1),
|
||||
]
|
||||
assert (response.results[0].successful_requests, response.results[0].failed_requests) == (2, 1)
|
||||
assert (response.results[0].prompt_tokens, response.results[0].completion_tokens) == (30, 15)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_client):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
caller = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER)
|
||||
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
|
||||
mock_db_client.db.query_raw = AsyncMock(return_value=[])
|
||||
mock_db_client.db.litellm_usertable.find_unique = AsyncMock(
|
||||
return_value=_team_spend_by_user_caller("alice", ["team-alpha"])
|
||||
)
|
||||
|
||||
await get_team_spend_by_user(
|
||||
user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04"
|
||||
)
|
||||
|
||||
sql, *params = mock_db_client.db.query_raw.call_args.args
|
||||
assert params == ["2026-09-01", "2026-09-04", "team-alpha"]
|
||||
assert 'sl."user" = $' not in sql
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_client):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
caller = UserAPIKeyAuth(user_id="bob", user_role=LitellmUserRoles.INTERNAL_USER)
|
||||
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="bob", role="user"), ["/key/info"])
|
||||
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
|
||||
mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
|
||||
mock_db_client.db.query_raw = AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)])
|
||||
mock_db_client.db.litellm_usertable.find_unique = AsyncMock(
|
||||
return_value=_team_spend_by_user_caller("bob", ["team-alpha"])
|
||||
)
|
||||
|
||||
response = await get_team_spend_by_user(
|
||||
user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04"
|
||||
)
|
||||
|
||||
sql, *params = mock_db_client.db.query_raw.call_args.args
|
||||
assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob"]
|
||||
assert "sl.team_id IN ($3)" in sql
|
||||
assert 'AND sl."user" = $4' in sql
|
||||
assert [(r.user_id, r.spend) for r in response.results] == [("bob", 0.25)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_client):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
caller = UserAPIKeyAuth(user_id="bob", user_role=LitellmUserRoles.INTERNAL_USER)
|
||||
mock_db_client.db.query_raw = AsyncMock(return_value=[])
|
||||
mock_db_client.db.litellm_usertable.find_unique = AsyncMock(
|
||||
return_value=_team_spend_by_user_caller("bob", ["team-alpha"])
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_team_spend_by_user(
|
||||
user_api_key_dict=caller, team_ids="team-beta", start_date="2026-09-01", end_date="2026-09-04"
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
mock_db_client.db.query_raw.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"team_ids,start_date,end_date,expected_error",
|
||||
[
|
||||
(None, "2026-09-01", "2026-09-04", "team_ids"),
|
||||
("", "2026-09-01", "2026-09-04", "team_ids"),
|
||||
("team-alpha", None, "2026-09-04", "start_date and end_date"),
|
||||
("team-alpha", "2026-09-04", "2026-09-01", "on or after"),
|
||||
("team-alpha", "2020-01-01", "2026-12-31", "at most 400 days"),
|
||||
("team-alpha", "nope", "2026-09-04", "valid YYYY-MM-DD"),
|
||||
],
|
||||
)
|
||||
async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids, start_date, end_date, expected_error):
|
||||
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
|
||||
|
||||
mock_db_client.db.query_raw = AsyncMock(return_value=[])
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_team_spend_by_user(
|
||||
user_api_key_dict=admin, team_ids=team_ids, start_date=start_date, end_date=end_date
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert expected_error in str(exc_info.value.detail)
|
||||
mock_db_client.db.query_raw.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
from collections.abc import Sequence
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from prisma.errors import PrismaError
|
||||
|
||||
from litellm.proxy.spend_tracking.key_metadata_recovery import (
|
||||
fill_missing_api_key_aliases,
|
||||
recover_double_hashed_key_metadata,
|
||||
)
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
|
||||
def _digest_row(digest: str, key_alias: str, team_id: str | None, user_id: str | None) -> dict[str, str | None]:
|
||||
return {"digest": digest, "key_alias": key_alias, "team_id": team_id, "user_id": user_id}
|
||||
|
||||
|
||||
def _query_raw_by_table(
|
||||
active_rows: Sequence[dict[str, str | None]],
|
||||
deleted_rows: Sequence[dict[str, str | None]],
|
||||
) -> AsyncMock:
|
||||
async def query_raw(sql: str, *params: object) -> list[dict[str, str | None]]:
|
||||
if '"LiteLLM_VerificationToken"' in sql:
|
||||
return list(active_rows)
|
||||
if '"LiteLLM_DeletedVerificationToken"' in sql:
|
||||
return list(deleted_rows)
|
||||
raise AssertionError(f"unexpected query: {sql}")
|
||||
|
||||
return AsyncMock(side_effect=query_raw)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_double_hashed_key_metadata_via_active_token_digest():
|
||||
double_hashed = hash_token("a" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = _query_raw_by_table(
|
||||
active_rows=[_digest_row(double_hashed, "batch-worker", "team-1", "alice")],
|
||||
deleted_rows=[],
|
||||
)
|
||||
|
||||
result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed})
|
||||
|
||||
assert result[double_hashed]["key_alias"] == "batch-worker"
|
||||
assert result[double_hashed]["team_id"] == "team-1"
|
||||
assert result[double_hashed]["user_id"] == "alice"
|
||||
((_, digests),) = [call.args for call in mock_prisma.db.query_raw.call_args_list]
|
||||
assert digests == [double_hashed]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_double_hashed_key_metadata_falls_back_to_deleted_tokens():
|
||||
double_hashed = hash_token("y" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = _query_raw_by_table(
|
||||
active_rows=[],
|
||||
deleted_rows=[_digest_row(double_hashed, "deleted-key", "team-del", "erin")],
|
||||
)
|
||||
|
||||
result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed})
|
||||
|
||||
assert result[double_hashed]["key_alias"] == "deleted-key"
|
||||
assert result[double_hashed]["team_id"] == "team-del"
|
||||
assert result[double_hashed]["user_id"] == "erin"
|
||||
assert [call.args[1] for call in mock_prisma.db.query_raw.call_args_list] == [[double_hashed], [double_hashed]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_only_asks_deleted_tokens_for_digests_active_keys_missed():
|
||||
found_active = hash_token("1" * 64)
|
||||
found_deleted = hash_token("2" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = _query_raw_by_table(
|
||||
active_rows=[_digest_row(found_active, "active-key", None, None)],
|
||||
deleted_rows=[_digest_row(found_deleted, "deleted-key", None, None)],
|
||||
)
|
||||
|
||||
result = await recover_double_hashed_key_metadata(mock_prisma, {found_active, found_deleted})
|
||||
|
||||
assert result[found_active]["key_alias"] == "active-key"
|
||||
assert result[found_deleted]["key_alias"] == "deleted-key"
|
||||
assert [call.args[1] for call in mock_prisma.db.query_raw.call_args_list] == [
|
||||
sorted((found_active, found_deleted)),
|
||||
[found_deleted],
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_permanent_miss_costs_two_digest_lookups_and_no_table_walk():
|
||||
double_hashed = hash_token("b" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = _query_raw_by_table(active_rows=[], deleted_rows=[])
|
||||
|
||||
result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed})
|
||||
|
||||
assert result == {}
|
||||
assert len(mock_prisma.db.query_raw.call_args_list) == 2
|
||||
mock_prisma.db.litellm_verificationtoken.find_many.assert_not_called()
|
||||
mock_prisma.db.litellm_deletedverificationtoken.find_many.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_skips_keys_that_are_not_sha256_digests():
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
|
||||
result = await recover_double_hashed_key_metadata(mock_prisma, {"sk-plain-key", "key-hash-short"})
|
||||
|
||||
assert result == {}
|
||||
mock_prisma.db.query_raw.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_returns_empty_when_digest_lookup_raises_prisma_error():
|
||||
double_hashed = hash_token("c" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(side_effect=PrismaError("db down"))
|
||||
|
||||
result = await recover_double_hashed_key_metadata(mock_prisma, {double_hashed})
|
||||
|
||||
assert result == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_missing_api_key_aliases_updates_null_alias_and_email_rows():
|
||||
double_hashed = hash_token("d" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = _query_raw_by_table(
|
||||
active_rows=[_digest_row(double_hashed, "recovered-alias", "team-9", "bob")],
|
||||
deleted_rows=[],
|
||||
)
|
||||
mock_prisma.db.litellm_usertable.find_many = AsyncMock(
|
||||
return_value=[SimpleNamespace(user_id="bob", user_email="bob@example.com")]
|
||||
)
|
||||
|
||||
rows = (
|
||||
{
|
||||
"api_key": double_hashed,
|
||||
"api_key_alias": None,
|
||||
"team_id": None,
|
||||
"user_email": None,
|
||||
"spend": 12.5,
|
||||
},
|
||||
{
|
||||
"api_key": "already-joined-token",
|
||||
"api_key_alias": "named-key",
|
||||
"team_id": "team-ok",
|
||||
"user_email": "other@example.com",
|
||||
"spend": 1.0,
|
||||
},
|
||||
)
|
||||
|
||||
filled = await fill_missing_api_key_aliases(mock_prisma, rows)
|
||||
|
||||
assert filled[0]["api_key_alias"] == "recovered-alias"
|
||||
assert filled[0]["team_id"] == "team-9"
|
||||
assert filled[0]["user_email"] == "bob@example.com"
|
||||
assert filled[0]["spend"] == 12.5
|
||||
assert filled[1]["api_key_alias"] == "named-key"
|
||||
assert mock_prisma.db.litellm_usertable.find_many.call_args.kwargs["where"] == {"user_id": {"in": ["bob"]}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_missing_api_key_aliases_leaves_rows_untouched_when_nothing_is_missing():
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
rows = ({"api_key": hash_token("e" * 64), "api_key_alias": "named", "user_email": "x@example.com"},)
|
||||
|
||||
filled = await fill_missing_api_key_aliases(mock_prisma, rows)
|
||||
|
||||
assert filled == rows
|
||||
mock_prisma.db.query_raw.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_missing_api_key_aliases_keeps_spend_user_email_when_alias_is_missing():
|
||||
double_hashed = hash_token("f" * 64)
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = _query_raw_by_table(
|
||||
active_rows=[_digest_row(double_hashed, "team-key", "team-9", "key-owner")],
|
||||
deleted_rows=[],
|
||||
)
|
||||
mock_prisma.db.litellm_usertable.find_many = AsyncMock(
|
||||
return_value=[SimpleNamespace(user_id="key-owner", user_email="owner@example.com")]
|
||||
)
|
||||
|
||||
rows = (
|
||||
{
|
||||
"api_key": double_hashed,
|
||||
"api_key_alias": None,
|
||||
"team_id": None,
|
||||
"user_email": "spender@example.com",
|
||||
"spend": 4.0,
|
||||
},
|
||||
)
|
||||
|
||||
filled = await fill_missing_api_key_aliases(mock_prisma, rows)
|
||||
|
||||
assert filled[0]["api_key_alias"] == "team-key"
|
||||
assert filled[0]["team_id"] == "team-9"
|
||||
assert filled[0]["user_email"] == "spender@example.com"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_missing_api_key_aliases_skips_named_keys_that_have_no_email():
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
rows = (
|
||||
{
|
||||
"api_key": hash_token("g" * 64),
|
||||
"api_key_alias": "service-key",
|
||||
"team_id": "team-svc",
|
||||
"user_email": None,
|
||||
},
|
||||
)
|
||||
|
||||
filled = await fill_missing_api_key_aliases(mock_prisma, rows)
|
||||
|
||||
assert filled == rows
|
||||
mock_prisma.db.query_raw.assert_not_called()
|
||||
|
|
@ -2712,6 +2712,41 @@ def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed():
|
|||
assert meta["user_api_key"] == hash_token(already_hashed)
|
||||
|
||||
|
||||
def test_get_logging_payload_batch_attribution_keeps_verification_token_hash():
|
||||
"""
|
||||
Batch cost rebuilds metadata with the managed object's already-hashed api_key.
|
||||
That hash must land in SpendLogs.api_key unchanged so Usage/CloudZero can join
|
||||
LiteLLM_VerificationToken for api_key_alias and user_email. Regression: without
|
||||
user_api_key_hash provenance, v1.99+ re-hashed the token and broke the join.
|
||||
"""
|
||||
token_hash = hash_token("sk-batch-creator-key")
|
||||
kwargs = {
|
||||
"model": "gpt-4o",
|
||||
"call_type": "aretrieve_batch",
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": token_hash,
|
||||
"user_api_key_hash": token_hash,
|
||||
"user_api_key_alias": "batch-creator",
|
||||
"user_api_key_user_id": "alice",
|
||||
"user_api_key_team_id": "team-1",
|
||||
}
|
||||
},
|
||||
}
|
||||
payload = get_logging_payload(
|
||||
kwargs=kwargs,
|
||||
response_obj={"id": "batch_123", "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}},
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
assert payload["api_key"] == token_hash
|
||||
assert payload["api_key"] != hash_token(token_hash)
|
||||
parsed_meta = json.loads(payload["metadata"])
|
||||
assert parsed_meta["user_api_key"] == token_hash
|
||||
assert parsed_meta["user_api_key_alias"] == "batch-creator"
|
||||
|
||||
|
||||
def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match():
|
||||
already_hashed = hash_token("sk-some-key")
|
||||
different_hash = hash_token("sk-other-key")
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import urllib.parse as urlparse
|
|||
|
||||
import uvicorn
|
||||
import yaml
|
||||
from uvicorn.config import LOOP_FACTORIES
|
||||
from uvicorn.importer import import_from_string
|
||||
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
|
||||
|
||||
|
|
@ -462,6 +464,12 @@ class TestProxyInitializationHelpers:
|
|||
with patch("sys.platform", "linux"):
|
||||
assert ProxyInitializationHelpers._get_loop_type() == "uvloop"
|
||||
|
||||
def test_selected_loop_factory_imports_on_this_interpreter(self):
|
||||
loop_type = ProxyInitializationHelpers._get_loop_type()
|
||||
if loop_type is None:
|
||||
pytest.skip("uvicorn picks the loop itself on this platform")
|
||||
assert callable(import_from_string(LOOP_FACTORIES[loop_type]))
|
||||
|
||||
@patch.dict(os.environ, {}, clear=True)
|
||||
def test_database_url_construction_with_special_characters(self):
|
||||
# Setup environment variables with special characters that need escaping
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@ import subprocess
|
|||
import types
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest import mock
|
||||
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, create_autospec, mock_open, patch
|
||||
|
||||
import click
|
||||
import httpx
|
||||
|
|
@ -808,6 +809,18 @@ def test_restructure_always_happens(monkeypatch):
|
|||
assert ui_path == packaged_ui_path
|
||||
|
||||
|
||||
def _mock_scheduled_proxy_config() -> MagicMock:
|
||||
config: Final = proxy_server_module.ProxyConfig()
|
||||
return MagicMock(
|
||||
spec=proxy_server_module.ProxyConfig,
|
||||
check_periodic_reloads=create_autospec(config.check_periodic_reloads),
|
||||
get_credentials=create_autospec(config.get_credentials),
|
||||
add_deployment=create_autospec(config.add_deployment),
|
||||
reload_search_tools_from_db=create_autospec(config.reload_search_tools_from_db),
|
||||
reload_mcp_servers_from_db=create_autospec(config.reload_mcp_servers_from_db),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_scheduled_jobs_credentials(monkeypatch):
|
||||
"""
|
||||
|
|
@ -823,7 +836,7 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
|
|
@ -883,7 +896,7 @@ async def test_periodic_reload_job_scheduled_without_store_model_in_db(monkeypat
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
scheduler = AsyncIOScheduler()
|
||||
|
||||
try:
|
||||
|
|
@ -924,7 +937,7 @@ async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
mock_scheduler = MagicMock()
|
||||
|
||||
configured_interval = 47
|
||||
|
|
@ -973,7 +986,7 @@ async def test_initialize_scheduled_jobs_rejects_non_positive_config_reload_inte
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
mock_scheduler = MagicMock()
|
||||
|
||||
with (
|
||||
|
|
@ -1020,7 +1033,7 @@ async def test_initialize_scheduled_jobs_hydrates_mcp_when_store_model_in_db_fal
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
|
|
@ -7370,7 +7383,7 @@ async def test_batch_cost_poller_is_confirmed_before_serving(monkeypatch):
|
|||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", AsyncMock()),
|
||||
patch("litellm.proxy.proxy_server.proxy_config", _mock_scheduled_proxy_config()),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", False),
|
||||
patch("litellm.proxy.proxy_server.llm_router", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.PROXY_BATCH_POLLING_ENABLED", True),
|
||||
|
|
@ -7412,7 +7425,7 @@ async def test_store_model_in_db_db_override_when_config_false():
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
|
|
@ -7455,7 +7468,7 @@ async def test_store_model_in_db_db_check_skipped_when_already_true(monkeypatch)
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
|
|
@ -7498,7 +7511,7 @@ async def test_store_model_in_db_db_failure_graceful(monkeypatch):
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
|
|
@ -11864,7 +11877,7 @@ async def _run_scheduled_background_jobs():
|
|||
mock_proxy_logging = MagicMock(spec=ProxyLogging)
|
||||
mock_proxy_logging.slack_alerting_instance = MagicMock()
|
||||
mock_proxy_logging.db_spend_update_writer = MagicMock()
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config = _mock_scheduled_proxy_config()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Includes file_search emulation: the flag must be forwarded on inner aresponses
|
|||
calls so routed requests do not hit a custom api_base /v1/responses endpoint.
|
||||
"""
|
||||
|
||||
from importlib import import_module
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
|
|
@ -17,11 +18,11 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
|||
class TestUseResponsesApiBridgeFlag:
|
||||
"""Test that bridge opt-in forces the chat completions path."""
|
||||
|
||||
@patch(
|
||||
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler"
|
||||
)
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
def test_bridge_used_when_use_chat_completions_api_true(
|
||||
self, mock_get_config, mock_bridge_handler
|
||||
|
|
@ -39,11 +40,11 @@ class TestUseResponsesApiBridgeFlag:
|
|||
|
||||
mock_bridge_handler.assert_called_once()
|
||||
|
||||
@patch(
|
||||
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler"
|
||||
)
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
def test_bridge_used_when_model_uses_chat_completions_prefix(
|
||||
self, mock_get_config, mock_bridge_handler
|
||||
|
|
@ -62,9 +63,9 @@ class TestUseResponsesApiBridgeFlag:
|
|||
# Model string is provider-normalized after resolution; prefix only forces the bridge.
|
||||
assert mock_bridge_handler.call_args.kwargs["model"].endswith("my-custom-model")
|
||||
|
||||
@patch("litellm.responses.main.base_llm_http_handler.response_api_handler")
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(import_module("litellm.responses.main").base_llm_http_handler, "response_api_handler")
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
def test_native_forwarding_when_flag_absent(
|
||||
self, mock_get_config, mock_native_handler
|
||||
|
|
@ -82,11 +83,11 @@ class TestUseResponsesApiBridgeFlag:
|
|||
|
||||
mock_native_handler.assert_called_once()
|
||||
|
||||
@patch(
|
||||
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler"
|
||||
)
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
def test_flag_does_not_leak_into_kwargs(self, mock_get_config, mock_bridge_handler):
|
||||
"""use_chat_completions_api should be popped and not passed to the bridge handler."""
|
||||
|
|
@ -104,11 +105,11 @@ class TestUseResponsesApiBridgeFlag:
|
|||
all_kwargs = call_kwargs.kwargs if call_kwargs.kwargs else {}
|
||||
assert "use_chat_completions_api" not in all_kwargs
|
||||
|
||||
@patch(
|
||||
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler"
|
||||
)
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
def test_bridge_used_when_provider_config_none(
|
||||
self, mock_get_config, mock_bridge_handler
|
||||
|
|
@ -127,8 +128,8 @@ class TestUseResponsesApiBridgeFlag:
|
|||
mock_bridge_handler.assert_called_once()
|
||||
|
||||
@patch("litellm.acompletion")
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
async def test_allowed_openai_params_forwarded_through_bridge(
|
||||
self, mock_get_config, mock_acompletion
|
||||
|
|
@ -164,9 +165,9 @@ class TestUseResponsesApiBridgeFlag:
|
|||
"reasoning_effort"
|
||||
]
|
||||
|
||||
@patch("litellm.responses.file_search.emulated_handler._call_aresponses")
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses")
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
async def test_bridge_flag_forwarded_to_file_search_emulation(
|
||||
self, mock_get_config, mock_call_aresponses
|
||||
|
|
@ -206,12 +207,12 @@ class TestUseResponsesApiBridgeFlag:
|
|||
call_kwargs.get("use_chat_completions_api") is True
|
||||
), "use_chat_completions_api should be forwarded to inner aresponses call"
|
||||
|
||||
@patch(
|
||||
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler"
|
||||
)
|
||||
@patch("litellm.vector_stores.main.asearch")
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
async def test_bridge_flag_prevents_native_responses_endpoint_call(
|
||||
self, mock_get_config, mock_asearch, mock_bridge_handler
|
||||
|
|
@ -280,10 +281,10 @@ class TestUseResponsesApiBridgeFlag:
|
|||
assert result is not None
|
||||
assert result.id is not None
|
||||
|
||||
@patch("litellm.responses.main.base_llm_http_handler.response_api_handler")
|
||||
@patch.object(import_module("litellm.responses.main").base_llm_http_handler, "response_api_handler")
|
||||
@patch("litellm.vector_stores.main.asearch")
|
||||
@patch(
|
||||
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config"
|
||||
@patch.object(
|
||||
import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config"
|
||||
)
|
||||
async def test_without_bridge_flag_uses_native_endpoint(
|
||||
self, mock_get_config, mock_asearch, mock_native_handler
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ in expected_responses_api_request/.
|
|||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from importlib import import_module
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -405,8 +406,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_
|
|||
|
||||
from litellm.responses.main import _aresponses_websocket
|
||||
|
||||
with patch(
|
||||
"litellm.responses.main.base_llm_http_handler.async_responses_websocket",
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue