scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

Merge remote-tracking branch 'origin/main' into litellm_deflake_20261021

Browse source
This commit is contained in:
Devin AI 2026-10-24 09:40:09 +00:00
commit cd573b41ce
392 changed files with 30643 additions and 6522 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

91
.github/workflows/interactions-spec-canary.yml vendored Normal file
View file

@ -0,0 +1,91 @@
name: Interactions spec canary
on:
schedule:
- cron: "17 7 * * *"
pull_request:
paths:
- .github/workflows/interactions-spec-canary.yml
- tests/unit/interactions/**
- litellm/types/interactions/**
workflow_dispatch:
permissions: {}
concurrency: interactions-spec-canary
jobs:
interactions-spec-canary:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
compliance: ${{ steps.compliance.outcome }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
- name: Run the compliance tests against Google's live spec
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
tracking-issue:
name: tracking-issue
needs: interactions-spec-canary
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
steps:
- name: File the tracking issue unless the compliance step passed, close it when it did
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const title = "Interactions spec canary is red";
const outcome = process.env.COMPLIANCE_OUTCOME;
const runUrl = process.env.RUN_URL;
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
const open = found.items.find((issue) => issue.title === title);
if (outcome === "success") {
if (open === undefined) return;
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
return;
}
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
if (open !== undefined) {
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
return;
}
await github.rest.issues.create({
...context.repo,
title,
assignees: ["mateo-berri"],
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
});

1
.github/workflows/test-unit.yml vendored
View file

@ -425,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"

11
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -61,6 +61,7 @@ RUN apk add --no-cache \
python-3.13-dev \
rust \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
@ -141,7 +142,7 @@ USER root
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

6
README.md
View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

2
security.md → SECURITY.md
View file

@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod
- Audit Logs with retention policy
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
For security inquiries, please contact us at support@berri.ai
For security inquiries, please contact us at security@berri.ai
#### Supported data regions for LiteLLM Cloud

8
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

4
cosign.pub
View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

7
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -59,6 +59,7 @@ RUN apk add --no-cache \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

7
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
done
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

2
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -30,7 +30,7 @@ RUN uv venv --python python && \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \

8
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

8
litellm-rust/Cargo.lock generated
View file

@ -8202,18 +8202,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
dependencies = [
"rustls-pki-types",
]

2
litellm-rust/Cargo.toml
View file

@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
strum = { version = "0.28.0", features = ["derive"] }
url = "2.5.8"
percent-encoding = "2.3"
webpki-roots = "1"
webpki-roots = "1.0.10"
time = { version = "0.3.53", features = ["parsing"] }
criterion = "0.8.2"
fancy-regex = "0.19.2"

8
litellm-rust/crates/gateway-inference/src/caching.rs
View file

@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result<Duration, Error> {
#[derive(Clone, Default)]
pub(crate) struct CacheHeaders(std::sync::Arc<std::sync::OnceLock<String>>);
impl litellm_host::interceptors::Interceptors<litellm_inference::RouteError> for CacheHeaders {
impl litellm_host::interceptors::Interceptors<litellm_inference::Error> for CacheHeaders {
async fn before_provider_request(
&self,
wire: litellm_host::interceptors::WireRequest,
_: litellm_host::interceptors::RequestContext,
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::RouteError> {
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::Error> {
Ok(wire)
}
async fn after_provider_response(
&self,
_: litellm_host::interceptors::RawResponse,
) -> Result<(), litellm_inference::RouteError> {
) -> Result<(), litellm_inference::Error> {
Ok(())
}
async fn result_ready(
&self,
facts: litellm_host::interceptors::ExecutionFacts,
) -> Result<(), litellm_inference::RouteError> {
) -> Result<(), litellm_inference::Error> {
if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source {
let _ = self.0.set(key);
}

26
litellm-rust/crates/gateway-inference/src/error.rs
View file

@ -4,7 +4,7 @@ use axum::{
response::{IntoResponse, Response},
};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error as InferenceError;
use litellm_llms::base_llm::ocr::error::Error as OcrError;
use serde_json::{Map, Value, json};
@ -19,7 +19,7 @@ pub enum Error {
)]
UnknownModel(String),
#[error(transparent)]
Route(#[from] RouteError),
Route(#[from] InferenceError),
#[error(transparent)]
Ocr(#[from] OcrError),
#[error("{0} is not implemented by the Rust gateway")]
@ -36,8 +36,8 @@ impl IntoResponse for Error {
}
}
impl From<litellm_host_http::Error<RouteError>> for Error {
fn from(error: litellm_host_http::Error<RouteError>) -> Self {
impl From<litellm_host_http::Error<InferenceError>> for Error {
fn from(error: litellm_host_http::Error<InferenceError>) -> Self {
match error {
litellm_host_http::Error::Call(error) => Self::Route(error),
litellm_host_http::Error::Protocol => Self::Internal(error.to_string()),
@ -50,7 +50,7 @@ impl Error {
match self {
Self::Auth(error) => error.status(),
Self::Unsupported(_)
| Self::Route(RouteError::Unsupported(_))
| Self::Route(InferenceError::Unsupported(_))
| Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED,
Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
Self::Ocr(
@ -64,10 +64,10 @@ impl Error {
.and_then(|status| StatusCode::from_u16(status).ok())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST,
Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => {
Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => {
StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY)
}
Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
StatusCode::UNAUTHORIZED
}
Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST,
@ -97,7 +97,9 @@ impl Error {
/// body already in that shape passes through, any other has its message extracted.
pub fn body(&self, request_id: Option<&str>) -> Value {
let raw = match self {
Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(),
Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => {
body.clone()
}
other => other.to_string(),
};
let parsed = serde_json::from_str::<Value>(&raw).ok();
@ -175,7 +177,7 @@ mod tests {
use super::*;
fn upstream(status: u16, body: &str) -> Error {
Error::Route(RouteError::Transport(TransportError::Http {
Error::Route(InferenceError::Transport(TransportError::Http {
status,
body: body.into(),
}))
@ -225,16 +227,16 @@ mod tests {
#[rstest]
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
#[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
#[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
#[case::missing_key(
Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey {
Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
})),
StatusCode::UNAUTHORIZED,
)]
#[case::lost_connection(
Error::Route(RouteError::Transport(TransportError::Network("reset".into()))),
Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))),
StatusCode::INTERNAL_SERVER_ERROR,
)]
fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) {

2
litellm-rust/crates/inference-chat/src/lib.rs
View file

@ -1,7 +1,7 @@
use litellm_host::observation::ObservationSender;
pub mod route;
pub mod types;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
mod common_utils;
pub mod constants;
pub(crate) mod handler;

10
litellm-rust/crates/inference-chat/tests/caching.rs
View file

@ -21,7 +21,7 @@ use litellm_host::{
lifecycle::{CallEvent, ExecutionEvent},
observation::observation_channel,
};
use litellm_inference::RouteError;
use litellm_inference::Error;
use rstest::{fixture, rstest};
use serde_json::{Value, json};
@ -166,12 +166,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -179,11 +179,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

2
litellm-rust/crates/inference-messages/src/lib.rs
View file

@ -14,7 +14,7 @@ use std::sync::Arc;
use crate::common_utils::messages_provider;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
messages_body,

10
litellm-rust/crates/inference-messages/tests/caching.rs
View file

@ -17,7 +17,7 @@ use litellm_host::interceptors::{
ResultSource, WireRequest,
};
use litellm_inference::{
RouteError,
Error,
caching::{CacheRequest, execute_unary},
};
use rstest::{fixture, rstest};
@ -133,12 +133,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -146,11 +146,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

2
litellm-rust/crates/inference-ocr/AGENTS.md
View file

@ -3,4 +3,4 @@
- The route returns its completed response directly
- Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`)
- Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors`
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error`
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error`

2
litellm-rust/crates/inference-ocr/src/provider_config.rs
View file

@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>(
provider: LlmProviders::Mistral,
}
}
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
Err(litellm_inference::Error::InvalidProvider(provider)) => {
return Err(Error::InvalidProvider(provider));
}
Err(error) => return Err(Error::InvalidProvider(error.to_string())),

2
litellm-rust/crates/inference-responses/src/lib.rs
View file

@ -1,5 +1,5 @@
use litellm_host::observation::ObservationSender;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
pub mod ids;
pub mod route;

10
litellm-rust/crates/inference-responses/tests/caching.rs
View file

@ -18,7 +18,7 @@ use litellm_host::interceptors::{
WireRequest,
};
use litellm_inference::{
RouteError,
Error,
caching::{CacheRequest, execute_unary},
};
use rstest::{fixture, rstest};
@ -124,12 +124,12 @@ struct ChangingHooks {
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
}
impl Interceptors<RouteError> for ChangingHooks {
impl Interceptors<Error> for ChangingHooks {
async fn before_provider_request(
&self,
mut wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if self.rewrite {
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
@ -137,11 +137,11 @@ impl Interceptors<RouteError> for ChangingHooks {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
Ok(())
}

2
litellm-rust/crates/inference-transcription/src/lib.rs
View file

@ -1,5 +1,5 @@
pub mod types;
pub use litellm_inference::RouteError as Error;
pub use litellm_inference::Error;
mod constants;
mod handler;

6
litellm-rust/crates/inference/AGENTS.md
View file

@ -2,7 +2,7 @@
- `gateway-inference` (serving) -> `inference-<fmt>` (one API's call) -> `inference` (shared base)
- This crate is the shared base every format crate builds on
- `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
- `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
- diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`)
- response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture
- Shared test helpers belong in litellm-inference-testing, used only as a dev dependency
@ -49,14 +49,14 @@
- `litellm-inference-<fmt>` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...)
- Provider code never imports from `inference` or `inference-*`
- Handlers belong in `inference-*` or `llms`, never in a host crate
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `<fmt>::Error` name
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `<fmt>::Error` name
## Error placement
- The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to
- A failure is declared once, by the lowest crate that raises it
- every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`
- `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises
- `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises
- Scope follows the concept, not the first caller
- an error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it
- a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer

26
litellm-rust/crates/inference/src/caching.rs
View file

@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value;
use tokio_util::codec::Decoder;
use crate::RouteError;
use crate::Error;
pub trait Cachable: Protocol<Error = RouteError> {
pub trait Cachable: Protocol<Error = Error> {
const SURFACE: &'static str;
fn reusable(_response: &Self::Response) -> bool {
@ -145,15 +145,15 @@ pub async fn execute_unary<P, F, Fut>(
request: CacheRequest,
cache: Option<Arc<dyn ResponseCacheService>>,
options: Option<CacheOptions>,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
provider: F,
) -> Result<P::Response, RouteError>
) -> Result<P::Response, Error>
where
P: Cachable,
P::Response: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<P::Response, RouteError>>,
Fut: Future<Output = Result<P::Response, Error>>,
{
let identity = request.identity.clone();
crate::diagnostic::provider(&identity.model, &identity.provider);
@ -191,15 +191,15 @@ pub async fn execute_streaming<P, F, Fut>(
request: CacheRequest,
cache: Option<Arc<dyn ResponseCacheService>>,
options: Option<CacheOptions>,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
provider: F,
) -> Result<OutputOf<P>, RouteError>
) -> Result<OutputOf<P>, Error>
where
P: StreamCachable,
P::Response: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<OutputOf<P>, RouteError>>,
Fut: Future<Output = Result<OutputOf<P>, Error>>,
{
let identity = request.identity.clone();
crate::diagnostic::provider(&identity.model, &identity.provider);
@ -299,9 +299,9 @@ impl<P: StreamCachable> CallCache<P> {
}
fn capture_stream<P: StreamCachable>(
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>>,
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>>,
session: CacheSession,
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>> {
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>> {
stream::try_unfold(
(chunks, Some(Vec::<u8>::new()), session),
|(mut chunks, captured, session)| async move {
@ -330,7 +330,7 @@ fn capture_stream<P: StreamCachable>(
{
session.store(entry).await;
}
Ok::<_, RouteError>(None)
Ok::<_, Error>(None)
}
}
},
@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool {
async fn publish(
facts: ExecutionFacts,
interceptors: &impl Interceptors<RouteError>,
interceptors: &impl Interceptors<Error>,
observers: Option<&ObservationSender>,
) -> Result<(), RouteError> {
) -> Result<(), Error> {
if let Some(observers) = observers {
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
facts: facts.clone(),

10
litellm-rust/crates/inference/src/context.rs
View file

@ -5,7 +5,7 @@ use litellm_host::{
observation::ObservationSender,
};
use crate::{CallOptions, RouteError};
use crate::{CallOptions, Error};
pub struct CallContext<'a, I> {
pub interceptors: &'a I,
@ -13,7 +13,7 @@ pub struct CallContext<'a, I> {
pub cache: CachePolicy,
}
impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
pub fn new(interceptors: &'a I, options: CallOptions) -> Self {
Self {
interceptors,
@ -22,7 +22,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
}
}
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
if let Some(observers) = &self.observers {
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
facts: facts.clone(),
@ -31,7 +31,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
self.interceptors.result_ready(facts).await
}
pub async fn response_received(&self, body: &str) -> Result<(), RouteError> {
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
let raw = RawResponse {
body: body.to_owned(),
};
@ -43,6 +43,6 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
self.interceptors
.after_provider_response(raw)
.await
.map_err(RouteError::post_call)
.map_err(Error::post_call)
}
}

24
litellm-rust/crates/inference/src/error.rs
View file

@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError;
use litellm_llms::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum RouteError {
pub enum Error {
#[error("expected {expected}, got {actual}")]
InvalidType {
expected: &'static str,
@ -38,10 +38,10 @@ pub enum RouteError {
#[error(transparent)]
Secret(#[from] SecretError),
#[error("post-call hook failed: {0}")]
PostCallHook(#[source] Arc<RouteError>),
PostCallHook(#[source] Arc<Error>),
}
impl From<litellm_host::machine::MachineFault> for RouteError {
impl From<litellm_host::machine::MachineFault> for Error {
fn from(fault: litellm_host::machine::MachineFault) -> Self {
use litellm_host::machine::MachineFault;
Self::InvalidRequest(match fault {
@ -51,7 +51,7 @@ impl From<litellm_host::machine::MachineFault> for RouteError {
}
}
impl RouteError {
impl Error {
pub fn post_call(error: Self) -> Self {
Self::PostCallHook(Arc::new(error))
}
@ -76,7 +76,7 @@ impl RouteError {
}
}
impl From<LlmError> for RouteError {
impl From<LlmError> for Error {
fn from(error: LlmError) -> Self {
match error {
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
@ -99,7 +99,7 @@ impl SecretError {
}
}
impl From<litellm_secrets::Error> for RouteError {
impl From<litellm_secrets::Error> for Error {
fn from(error: litellm_secrets::Error) -> Self {
Self::Secret(SecretError(Arc::new(error)))
}
@ -115,22 +115,22 @@ impl Eq for SecretError {}
#[cfg(test)]
mod tests {
use super::RouteError;
use super::Error;
use litellm_llms::{Error as LlmError, ErrorDetail};
use rstest::rstest;
#[test]
fn a_missing_api_key_is_the_environment_not_the_request() {
assert!(
!RouteError::Auth(litellm_auth::Error::MissingApiKey {
!Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
})
.is_request()
);
assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request());
assert!(RouteError::InvalidRequest("top_k".into()).is_request());
assert!(!RouteError::InvalidResponse("bad json".into()).is_request());
assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request());
assert!(Error::InvalidRequest("top_k".into()).is_request());
assert!(!Error::InvalidResponse("bad json".into()).is_request());
}
#[rstest]
#[case::request(true)]
@ -139,7 +139,7 @@ mod tests {
let source = serde_json::from_str::<serde_json::Value>("{").unwrap_err();
let source_message = source.to_string();
let detail = ErrorDetail::invalid("test payload", source);
let error = RouteError::from(if request {
let error = Error::from(if request {
LlmError::InvalidRequest(detail)
} else {
LlmError::InvalidResponse(detail)

2
litellm-rust/crates/inference/src/lib.rs
View file

@ -7,7 +7,7 @@ pub mod outbound;
pub mod provider;
pub mod resources;
pub use error::RouteError;
pub use error::Error;
#[derive(Clone, Default)]
pub struct CallOptions {

2
litellm-rust/crates/inference/src/provider.rs
View file

@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
use crate::error::RouteError as Error;
use crate::error::Error;
#[derive(Debug)]
pub struct ResolvedProvider<'a> {

24
litellm-rust/crates/inference/tests/caching.rs
View file

@ -25,7 +25,7 @@ use litellm_host::{
protocol::Protocol,
};
use litellm_inference::{
RouteError,
Error,
caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary},
};
use rstest::{fixture, rstest};
@ -36,7 +36,7 @@ struct TestRoute;
impl Protocol for TestRoute {
type Request = Value;
type Response = Value;
type Error = RouteError;
type Error = Error;
type HostCall = Infallible;
type Chunk = Bytes;
type StreamHead = ();
@ -204,7 +204,7 @@ async fn streamed(
.chunks(3)
.map(|bytes| Ok(Bytes::copy_from_slice(bytes)))
.collect::<Vec<_>>();
let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure")));
let ending = fail.then_some(Err(Error::Unsupported("test transport failure")));
Ok(CallOutput::Stream {
head: (),
chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(),
@ -215,7 +215,7 @@ async fn streamed(
.unwrap()
}
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, RouteError> {
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, Error> {
let CallOutput::Stream { chunks, .. } = output else {
panic!("expected a stream");
};
@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc<dyn ResponseCac
None,
|| async {
calls.fetch_add(1, Ordering::SeqCst);
Err(RouteError::Unsupported("test provider failure"))
Err(Error::Unsupported("test provider failure"))
},
)
.await;
@ -467,8 +467,8 @@ struct CacheHitAccounting {
reject: bool,
}
impl Interceptors<RouteError> for CacheHitAccounting {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
impl Interceptors<Error> for CacheHitAccounting {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.calls.fetch_add(1, Ordering::SeqCst);
let ResultSource::Cache { key } = facts.source else {
panic!("expected cache source")
@ -482,7 +482,7 @@ impl Interceptors<RouteError> for CacheHitAccounting {
);
self.key.set(key).unwrap();
if self.reject {
return Err(RouteError::Unsupported("cache accounting rejected"));
return Err(Error::Unsupported("cache accounting rejected"));
}
Ok(())
}
@ -491,11 +491,11 @@ impl Interceptors<RouteError> for CacheHitAccounting {
&self,
wire: WireRequest,
_: RequestContext,
) -> Result<WireRequest, RouteError> {
) -> Result<WireRequest, Error> {
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
}
@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
if reject {
assert!(matches!(
result,
Err(RouteError::Unsupported("cache accounting rejected"))
Err(Error::Unsupported("cache accounting rejected"))
));
} else {
assert_eq!(result.unwrap(), expected);
@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
impl Protocol for UnaryTestRoute {
type Request = Value;
type Response = Value;
type Error = RouteError;
type Error = Error;
type HostCall = Infallible;
type Chunk = Infallible;
type StreamHead = Infallible;

6
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

28
litellm-rust/crates/python-bridge/src/errors.rs
View file

@ -1,5 +1,5 @@
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error;
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
@ -19,9 +19,9 @@ pyo3::create_exception!(
"The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response."
);
pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr {
pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
match error {
RouteError::Transport(TransportError::Http { status, body }) => {
Error::Transport(TransportError::Http { status, body }) => {
RustUpstreamError::new_err((status, body))
}
other => by_fault(other.is_request(), other.to_string()),
@ -42,13 +42,13 @@ mod tests {
use super::*;
#[rstest::rstest]
#[case::unsupported(RouteError::Unsupported("test capability"), true)]
#[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)]
#[case::unsupported(Error::Unsupported("test capability"), true)]
#[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)]
#[case::connection(TransportError::Connect("unreachable".into()).into(), false)]
#[case::network(TransportError::Network("timed out".into()).into(), false)]
#[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)]
fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) {
#[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)]
fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) {
Python::initialize();
Python::attach(|py| {
let failure = route_error_to_pyerr(error);
@ -86,14 +86,12 @@ mod tests {
fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() {
Python::initialize();
Python::attach(|py| {
let missing =
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
}));
let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "Anthropic",
environment_variable: "ANTHROPIC_API_KEY",
}));
assert!(missing.is_instance_of::<PyRuntimeError>(py));
let invalid =
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader));
let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader));
assert!(invalid.is_instance_of::<PyValueError>(py));
});
}

10
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_host_python::{PythonBinding, PythonHostCalls, from_py};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_inference::Error;
use litellm_secrets::source::SecretSource;
use pyo3::{
exceptions::PyValueError,
@ -119,14 +119,14 @@ impl InferenceHost {
public_response(py, self.module, response)
}
pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult<PyErr> {
if let RouteError::Secret(source) = &error
pub fn error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
{
return Ok(original);
}
let native = match error {
RouteError::Transport(TransportError::Http { status, body }) => {
Error::Transport(TransportError::Http { status, body }) => {
let error = RustUpstreamError::new_err((status, body));
error
.value(py)
@ -144,7 +144,7 @@ impl InferenceHost {
}
pub(super) trait InferenceRoute: Sized + 'static {
type Protocol: Protocol<Error = RouteError>;
type Protocol: Protocol<Error = Error>;
const OPERATION: LoggingOperation;
const SYNC_CALL_TYPE: &'static str;
const ASYNC_CALL_TYPE: &'static str;

2
litellm/anthropic_beta_headers_config.json
View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

3
litellm/anthropic_beta_headers_manager.py
View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -145,7 +146,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()

4
litellm/cost_calculator.py
View file

@ -2264,6 +2264,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2292,6 +2294,8 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,

4
litellm/integrations/deepeval/types.py
View file

@ -1,6 +1,6 @@
# Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py
from enum import Enum
from typing import Any, ClassVar, Literal
from typing import ClassVar, Literal
from pydantic import ConfigDict, Field
@ -55,7 +55,7 @@ class TraceApi(LiteLLMBaseModel):
tool_spans: list[BaseApiSpan] = Field(alias="toolSpans")
start_time: str = Field(alias="startTime")
end_time: str = Field(alias="endTime")
metadata: dict[str, Any] | None = Field(None)
metadata: dict[str, object] | None = Field(None)
tags: list[str] | None = Field(None)
environment: str | None = Field(None)

7
litellm/integrations/otel/mount.py
View file

@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
is unavailable. This MUST be called at app-creation time — once the lifespan
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
middleware after an application has started".
runs, the middleware stack is frozen and middleware added later never runs, so
a late call is skipped without registering anything.
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
``ProxyTracerProvider``; the proxy publishes the real provider as the global
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
try:
if not is_otel_v2_enabled():
return
if app.middleware_stack is not None:
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
return
# Lazy: only the V2-enabled path needs the optional
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

52
litellm/integrations/otel/plumbing/otlp_json.py
View file

@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
import base64
import json
import logging
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
@ -13,9 +14,13 @@ from typing import Final, TypeAlias
import requests
from google.protobuf.json_format import MessageToDict
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
_logger: Final = logging.getLogger(__name__)
JSON_CONTENT_TYPE: Final = "application/json"
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
def __init__(
self,
endpoint: str | None,
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
headers: Mapping[str, str],
certificate_file: str | None = None,
session: "requests.Session | None" = None,
*,
max_request_size: int | None = None,
) -> None:
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
super().__init__(
endpoint=endpoint,
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
certificate_file=certificate_file,
session=session,
max_request_size=max_request_size,
)
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
return encode_spans_json(spans)
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
if self._shutdown:
_logger.warning("Exporter already shutdown, ignoring batch")
return SpanExportResult.FAILURE
with self._metrics.export_operation(len(spans)) as result:
try:
serialized_data: Final = encode_spans_json(spans)
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
_logger.error("Failed to encode span batch: %s", error)
result.error = error
return SpanExportResult.FAILURE
if 0 < self._max_request_size < len(serialized_data):
result.error = RequestPayloadTooLargeError(
f"Serialized span request size {len(serialized_data)} bytes "
f"exceeds max_request_size {self._max_request_size} bytes."
)
_logger.warning("Dropping span batch: %s", result.error)
return SpanExportResult.FAILURE
export_result: Final = self._client.export(serialized_data)
if not export_result.success:
result.error = export_result.error
result.error_attrs = (
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
if export_result.status_code is not None
else None
)
return SpanExportResult.FAILURE
return SpanExportResult.SUCCESS

9
litellm/integrations/otel/presets/agentops.py
View file

@ -76,6 +76,7 @@ def agentops_preset(
def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
import requests
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
super().__init__(endpoint=endpoint)
self._agentops_session = requests.Session()
super().__init__(endpoint=endpoint, session=self._agentops_session)
self._agentops_api_key = api_key
self._auth_resolved = False
@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
try:
token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
if token:
# ``_session`` is the requests.Session the base exporter
# POSTs through; updating its Authorization header is how the
# minted JWT reaches every subsequent export.
self._session.headers["Authorization"] = f"Bearer {token}"
self._agentops_session.headers["Authorization"] = f"Bearer {token}"
except Exception as e:
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)

6
litellm/interactions/litellm_responses_transformation/streaming_iterator.py
View file

@ -3,8 +3,8 @@ Streaming iterator for transforming Responses API stream to Interactions API str
"""
from collections import deque
from collections.abc import AsyncIterator, Iterator
from typing import Any, Final, cast
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import Final, cast
from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
@ -44,7 +44,7 @@ class LiteLLMResponsesInteractionsStreamingIterator:
request_input: InteractionInput | None,
optional_params: InteractionsAPIOptionalRequestParams,
custom_llm_provider: str | None = None,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
):
self.model = model
self.responses_stream_iterator = litellm_custom_stream_wrapper

1
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -102,6 +102,7 @@ class ExceptionCheckers:
"`inputs` tokens + `max_new_tokens` must be",
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:

3
litellm/litellm_core_utils/get_blog_posts.py
View file

@ -19,6 +19,7 @@ import httpx
from pydantic import ConfigDict
from litellm import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.types.llms.base import LiteLLMBaseModel
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
@ -129,7 +130,7 @@ class GetBlogPosts:
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
Fetches from ``url`` otherwise, falling back to local backup on failure.
"""
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
return cls.load_local_blog_posts()
now: Final = time.time()

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info(
if dynamic_api_key is None and api_key is not None:
dynamic_api_key = api_key
return model, custom_llm_provider, dynamic_api_key, api_base
def get_provider_affinity_header(api_base: str | None) -> str | None:
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)

5
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -39,6 +39,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.fallback_generalizations import (
set_fallback_generalizations,
)
from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
@ -456,7 +457,7 @@ async def refetch_model_cost_map(
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
map they already have.
"""
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
@ -740,7 +741,7 @@ def get_model_cost_map(
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True

5
litellm/litellm_core_utils/litellm_logging.py
View file

@ -161,6 +161,7 @@ from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostPerToken,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass):
"call_type": self.call_type,
"optional_params": self.optional_params,
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,

7
litellm/litellm_core_utils/offline_mode.py Normal file
View file

@ -0,0 +1,7 @@
import os
def is_offline_mode() -> bool:
"""True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather
than from a destination the operator configured, must check this and use its bundled copy or skip instead"""
return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true"

8
litellm/litellm_core_utils/provider_affinity.py
View file

@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None:
return getattr(value, key, None)
def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None:
def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None:
header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None
if header is None:
return None
return default_header
if not isinstance(header, str):
raise TypeError("provider_affinity_header must be a string")
return validate_provider_affinity_header_name(header)
@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
def add_provider_affinity_header(
headers: Mapping[str, object], litellm_params: object | None
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
header_name: Final = _get_provider_affinity_header_name(litellm_params)
header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header)
if header_name is None or any(key.lower() == header_name.lower() for key in headers):
return dict(headers)

12
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
_messages_carry_output_config = messages_carry_output_config
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
if "stop_sequence" not in response:
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
return response
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> AnthropicMessagesResponse:
"""
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
"""
try:
raw_response_json: Final = raw_response.json()
except Exception:
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
return AnthropicMessagesResponse(**raw_response_json)
response: Final = AnthropicMessagesResponse(**raw_response_json)
return _ensure_stop_sequence(response)
def get_async_streaming_response_iterator(
self,

17
litellm/llms/aws_polly/text_to_speech/transformation.py
View file

@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
else:
mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
# Extract engine from model name (e.g., "aws_polly/neural" -> "neural")
# model arrives without the "aws_polly/" prefix, e.g. "standard"
engine: Final = self._extract_engine_from_model(model)
mapped_params["engine"] = engine
@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
"""
Extract engine from model name.
speech() strips the "aws_polly/" prefix before the config is called,
so the bare engine name is what normally arrives here.
Examples:
- aws_polly/neural -> neural
- aws_polly/standard -> standard
- standard -> standard
- aws_polly/long-form -> long-form
- aws_polly -> neural (default)
"""
if "/" in model:
parts: Final = model.split("/")
if len(parts) >= 2:
engine: Final = parts[1].lower()
if engine in self.VALID_ENGINES:
return engine
engine: Final = model.rsplit("/", 1)[-1].lower()
if engine in self.VALID_ENGINES:
return engine
return self.DEFAULT_ENGINE
def validate_environment(

15
litellm/llms/azure/responses/transformation.py
View file

@ -3,6 +3,7 @@ from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from urllib.parse import urlsplit
import httpx
from openai.types.responses import ResponseReasoningItem
@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params)
parsed_base: Final = urlsplit(resolved_api_base or "")
path: Final = parsed_base.path.rstrip("/")
if path.endswith(("/openai/v1", "/openai/v1/responses")):
responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path
return BaseAzureLLM.get_base_azure_url(
api_base=parsed_base._replace(path=responses_path).geturl(),
litellm_params=responses_params,
route="/openai/v1/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
return BaseAzureLLM.get_base_azure_url(
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
litellm_params=responses_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

3
litellm/llms/azure_ai/chat/transformation.py
View file

@ -12,6 +12,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
audio_or_image_in_message_content,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig):
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(messages)
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:

6
litellm/llms/base_llm/ocr/transformation.py
View file

@ -4,7 +4,7 @@ Base OCR types shared by the Rust OCR route and Python consumers.
import builtins
from collections.abc import Mapping
from typing import Any, Final, Literal
from typing import Final, Literal
from pydantic import PrivateAttr
@ -45,7 +45,7 @@ class OCRPageImage(LiteLLMPydanticObjectBase):
"""Image extracted from OCR page."""
image_base64: str | None = None
bbox: dict[str, Any] | None = None
bbox: dict[str, builtins.object] | None = None
model_config = {"extra": "allow"}
@ -80,7 +80,7 @@ class OCRResponse(LiteLLMPydanticObjectBase):
pages: list[OCRPage]
model: str
document_annotation: Any | None = None
document_annotation: builtins.object | None = None
usage_info: OCRUsageInfo | None = None
content: str | None = None
tables: list[dict[str, builtins.object]] | None = None

6
litellm/llms/base_llm/sandbox/transformation.py
View file

@ -7,7 +7,7 @@ run code -> delete container; `code_interpreter_tool` combines all three.
"""
import builtins
from typing import Any, Final
from typing import Final
import httpx
from pydantic import Field, PrivateAttr
@ -42,8 +42,8 @@ class CodeExecutionResult(LiteLLMPydanticObjectBase):
stdout: str = ""
stderr: str = ""
results: list[dict[str, Any]] = Field(default_factory=list)
error: dict[str, Any] | None = None
results: list[dict[str, object]] = Field(default_factory=list)
error: dict[str, object] | None = None
execution_count: int | None = None
object: str = "code_execution"

15
litellm/llms/bedrock/count_tokens/kimi_k3.py
View file

@ -27,6 +27,7 @@ from typing_extensions import Self
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
@ -67,12 +68,14 @@ class KimiK3TokenizerSource:
def _hugging_face_ranks_path() -> str:
import huggingface_hub
return huggingface_hub.hf_hub_download(
repo_id=KIMI_K3_REPO,
filename=KIMI_K3_RANKS_FILE,
revision=KIMI_K3_REVISION,
local_files_only=litellm.disable_hf_tokenizer_download is True,
)
if litellm.disable_hf_tokenizer_download is True or is_offline_mode():
cached: Final = huggingface_hub.try_to_load_from_cache(
repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION
)
if not isinstance(cached, str):
raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache")
return cached
return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION)
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)

27
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.realtime_errors import (
close_after_upstream_handshake_refusal,
realtime_error_event,
@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import (
)
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -2435,13 +2438,13 @@ class BaseLLMHTTPHandler:
# Also check for extra_headers in kwargs (from config or direct calls)
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
# Merge all header sources: forwarded < extra_headers < provider_specific
merged_headers: Final = {}
if forwarded_headers:
merged_headers.update(forwarded_headers)
if extra_headers_from_kwargs:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
{
**(forwarded_headers or {}),
**(extra_headers_from_kwargs or {}),
**(provider_specific_headers or {}),
}
)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
@ -2452,11 +2455,19 @@ class BaseLLMHTTPHandler:
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
affinity_api_base: Final = (
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
(
headers,
api_base,
) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment(
headers=merged_headers or {},
headers=(
add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header)
if default_affinity_header is not None
else dict(merged_headers)
),
model=model,
messages=messages,
optional_params=request_params,

4
litellm/llms/github_copilot/embedding/transformation.py
View file

@ -202,7 +202,9 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig):
optional_params[param] = value
return optional_params
def get_error_class(self, error_message: str, status_code: int, headers: Any) -> BaseLLMException:
def get_error_class(
self, error_message: str, status_code: int, headers: dict[str, str] | httpx.Headers
) -> BaseLLMException:
from litellm.llms.openai.openai import OpenAIConfig
return OpenAIConfig().get_error_class(error_message=error_message, status_code=status_code, headers=headers)

4
litellm/llms/ollama/completion/handler.py
View file

@ -87,7 +87,7 @@ async def ollama_aembeddings(
model: str,
prompts: list[str],
model_response: EmbeddingResponse,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
encoding: TokenEncoder | None,
):
@ -110,7 +110,7 @@ def ollama_embeddings(
api_base: str,
model: str,
prompts: list[str],
optional_params: dict,
optional_params: Mapping[str, object],
model_response: EmbeddingResponse,
logging_obj: object,
encoding: TokenEncoder | None = None,

10
litellm/llms/opencode/common_utils.py Normal file
View file

@ -0,0 +1,10 @@
import re
def is_opencode_api_base(api_base: str | None) -> bool:
return bool(
api_base
and re.fullmatch(
r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base
)
)

6
litellm/llms/opencode/transformation.py Normal file
View file

@ -0,0 +1,6 @@
from litellm.llms.opencode.common_utils import is_opencode_api_base
class OpenCodeProviderAdapter:
def get_provider_affinity_header(self, api_base: str | None) -> str | None:
return "x-opencode-session" if is_opencode_api_base(api_base) else None

17
litellm/main.py
View file

@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import (
with_control_options,
)
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_provider_affinity_header,
invalid_provider_routing_hints_message,
)
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
VertexAIModelRoute,
@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import (
resolve_chat_reasoning,
)
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.completion import (
CompletionDispatchContext,
_CompletionDispatchResult,
@ -169,7 +172,6 @@ from litellm.utils import (
ProviderConfigManager,
TextChoices,
TextCompletionResponse,
TextCompletionStreamWrapper,
TranscriptionResponse,
Usage,
add_provider_specific_params_to_optional_params,
@ -5990,19 +5992,28 @@ def completion(
},
)
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
if litellm_params.get("provider_affinity_header") is not None:
affinity_api_base: Final = (
str(client.base_url)
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
else get_openai_credentials(api_base=api_base).api_base
if custom_llm_provider == "openai"
else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None:
try:
headers = add_provider_affinity_header(
headers=headers or litellm.headers or MappingProxyType({}),
litellm_params=MappingProxyType(
{
"provider_affinity_header": litellm_params["provider_affinity_header"],
"provider_affinity_header": litellm_params.get("provider_affinity_header"),
"litellm_session_id": kwargs.get("litellm_session_id"),
"session_id": kwargs.get("session_id"),
"metadata": metadata,
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
except ValueError as affinity_error:
raise litellm.BadRequestError(

1
litellm/messages/dispatch.py
View file

@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target:
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**detectors.COMMON,
Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity,
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
Feature.RESPONSE_CACHE: detectors.response_cache,
Feature.MODIFY_PARAMS: detectors.modify_params,

276
litellm/model_prices_and_context_window_backup.json
View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -36861,17 +36957,17 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-08,
"cache_creation_input_token_cost_priority": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_decisions": 1e-07,
"cache_read_input_token_cost_decisions": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
"cache_read_input_token_cost_batches": 5e-09,
@ -42337,6 +42433,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -45623,6 +45847,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45648,6 +45873,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -48499,6 +48725,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -49352,6 +49589,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49371,6 +49609,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -52906,12 +53145,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53167,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -59917,6 +60160,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -59942,6 +60186,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -59967,6 +60212,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -59990,6 +60236,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -67841,6 +68088,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -67860,6 +68108,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68138,6 +68387,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68157,6 +68407,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68545,6 +68796,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68564,6 +68816,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68704,6 +68957,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68723,6 +68977,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -73330,6 +73585,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,

8
litellm/models/managed_files.py
View file

@ -6,7 +6,7 @@ Canonical definitions for the ``litellm_managed*`` tables. Re-exported from
"""
from datetime import datetime
from typing import Any, Literal
from typing import Literal
from litellm.types.llms.base import LiteLLMPydanticObjectBase
from litellm.types.llms.openai import OpenAIFileObject, ResponsesAPIResponse
@ -39,7 +39,7 @@ class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase):
"""Table for managing vector stores with target_model_names support."""
unified_resource_id: str
resource_object: Any | None = None
resource_object: object | None = None
model_mappings: dict[str, str]
flat_model_resource_ids: list[str]
created_by: str | None = None
@ -54,11 +54,11 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase):
custom_llm_provider: str
vector_store_name: str | None = None
vector_store_description: str | None = None
vector_store_metadata: dict[str, Any] | None = None
vector_store_metadata: dict[str, object] | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
litellm_credential_name: str | None = None
litellm_params: dict[str, Any] | None = None
litellm_params: dict[str, object] | None = None
team_id: str | None = None
user_id: str | None = None
is_config: bool = False

BIN
litellm/proxy/_experimental/out/assets/aria-agent.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 685 KiB

27
litellm/proxy/_experimental/out/assets/logos/claude.svg Normal file
View file

@ -0,0 +1,27 @@
<!--
Source: https://github.com/lobehub/lobe-icons/blob/master/packages/static-svg/icons/claude-color.svg
Retrieved: 2026-10-22
MIT License
Copyright (c) 2023 LobeHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-->
<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z" fill="#D97757" fill-rule="nonzero"></path></svg>

5
litellm/proxy/_lazy_features.py
View file

@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
path_prefixes=("/vantage",),
),
LazyFeature(
name="usage_ai",
module_path="litellm.proxy.management_endpoints.usage_endpoints",
path_prefixes=("/usage/ai",),
),
LazyFeature(
name="prompts",
module_path="litellm.proxy.prompts.prompt_endpoints",

163
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -11407,7 +11407,7 @@
},
"/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_systemone_post",
"requestBody": {
"content": {
@ -11813,7 +11813,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -12814,7 +12813,7 @@
},
"/v1/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_v1_systemone_post",
"requestBody": {
"content": {
@ -13220,7 +13219,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -57900,163 +57898,6 @@
}
}
},
"usage_ai": {
"components": {
"schemas": {
"ChatMessage": {
"properties": {
"content": {
"title": "Content",
"type": "string"
},
"role": {
"enum": [
"user",
"assistant"
],
"title": "Role",
"type": "string"
}
},
"required": [
"role",
"content"
],
"title": "ChatMessage",
"type": "object"
},
"HTTPValidationError": {
"properties": {
"detail": {
"items": {
"$ref": "#/components/schemas/ValidationError"
},
"title": "Detail",
"type": "array"
}
},
"title": "HTTPValidationError",
"type": "object"
},
"UsageAIChatRequest": {
"properties": {
"messages": {
"description": "Chat messages (user/assistant history)",
"items": {
"$ref": "#/components/schemas/ChatMessage"
},
"title": "Messages",
"type": "array"
},
"model": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "Model to use for AI chat",
"title": "Model"
}
},
"required": [
"messages"
],
"title": "UsageAIChatRequest",
"type": "object"
},
"ValidationError": {
"properties": {
"ctx": {
"title": "Context",
"type": "object"
},
"input": {
"title": "Input"
},
"loc": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
},
"title": "Location",
"type": "array"
},
"msg": {
"title": "Message",
"type": "string"
},
"type": {
"title": "Error Type",
"type": "string"
}
},
"required": [
"loc",
"msg",
"type"
],
"title": "ValidationError",
"type": "object"
}
}
},
"paths": {
"/usage/ai/chat": {
"post": {
"description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.",
"operationId": "usage_ai_chat_usage_ai_chat_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UsageAIChatRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Usage Ai Chat",
"tags": [
"usage_ai"
]
}
}
}
},
"vantage": {
"components": {
"schemas": {

7
litellm/proxy/_types.py
View file

@ -2717,6 +2717,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"hidden model can still be called."
),
)
return_wildcard_routes: bool | None = Field(
None,
description=(
"When true, `/v1/models` lists wildcard routes like `openai/*` unless the request passes "
"`return_wildcard_routes=false`."
),
)
alerting: list | None = Field(
None,
description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL",

6
litellm/proxy/a2a/endpoints.py
View file

@ -9,7 +9,7 @@ admin pick which ones to expose through the proxy. The actual merge into a
LiteLLM-fronted card happens when the agent is saved via ``POST /v1/agents``.
"""
from typing import Any, Final
from typing import Final
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
@ -50,7 +50,7 @@ class DiscoverAgentRequest(LiteLLMBaseModel):
"query parameter."
),
)
params: dict[str, Any] | None = Field(
params: dict[str, object] | None = Field(
default=None,
description=(
"Mode-specific parameters. ``langgraph_platform`` requires "
@ -61,7 +61,7 @@ class DiscoverAgentRequest(LiteLLMBaseModel):
class DiscoverAgentResponse(LiteLLMBaseModel):
url: str
agent_card: dict[str, Any]
agent_card: dict[str, object]
@router.post(

2
litellm/proxy/auth/auth_exception_handler.py
View file

@ -41,7 +41,7 @@ DB_UNAVAILABLE_FALLBACK_USER_ID: Final = "__db_unavailable_fallback__"
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any

3
litellm/proxy/auth/password_policy.py
View file

@ -21,6 +21,7 @@ from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm._version import version
from litellm.constants import HIBP_RANGE_API_BASE
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
return general_settings.get("password_policy_check_breached_passwords", True) is not False
return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False
async def is_password_breached(

300
litellm/proxy/db/db_spend_update_writer.py
View file

@ -12,7 +12,6 @@ import json
import os
import random
import time
import traceback
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
from contextlib import asynccontextmanager
from contextvars import ContextVar
@ -90,6 +89,7 @@ from litellm.proxy.spend_tracking.savings import (
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
@ -313,6 +313,38 @@ def _spend_tables_left_to_send(
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None:
spend_log_error(
"Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s",
step_name,
payload.get("request_id"),
payload.get("api_key"),
exc=exc,
)
_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata
def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata:
"""Malformed metadata costs the request its metadata-derived fields, never its daily rollup row."""
raw: Final = payload.get("metadata")
if not isinstance(raw, str):
return _EMPTY_SPEND_LOG_METADATA
try:
parsed: Final[SpendLogsMetadata] = json.loads(raw)
except json.JSONDecodeError as e:
spend_log_error(
"Spend tracking - unreadable spend log metadata, rollups for this request are written without "
"metadata-derived fields. request_id=%s api_key=%s",
payload.get("request_id"),
payload.get("api_key"),
exc=e,
)
return _EMPTY_SPEND_LOG_METADATA
return parsed
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
@ -647,8 +679,8 @@ class DBSpendUpdateWriter:
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
)
# Single task replaces 11 create_task() calls
asyncio.create_task(
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=response_cost,
user_id=user_id,
@ -661,7 +693,8 @@ class DBSpendUpdateWriter:
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
request_model_access_groups=get_request_model_access_groups(kwargs),
)
),
task_name="spend rollups",
)
self._enqueue_tool_registry_upsert(
@ -1086,91 +1119,52 @@ class DBSpendUpdateWriter:
project_id: str | None = None,
):
"""
Runs all 13 spend-update helpers sequentially inside a single asyncio task.
Each helper is wrapped in try/except so one failure doesn't prevent the others.
The deepcopy runs here, off the awaited request path, so the daily spend
helpers get a payload isolated from the spend-log queue entry and the caller.
Runs every spend-update helper for one request. Each helper goes through
_run_spend_update_step so one failure never skips the sibling writes. The deepcopy
isolates the daily spend helpers' payload from the spend-log queue entry and the caller.
"""
payload_copy: Final = copy.deepcopy(payload)
request_tags: Final = payload_copy.get("request_tags")
try:
await self._update_user_db(
await self._run_spend_update_step(
"_update_user_db",
self._update_user_db(
response_cost=response_cost,
user_id=user_id,
prisma_client=prisma_client,
litellm_proxy_budget_name=litellm_proxy_budget_name,
end_user_id=end_user_id,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_user_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_key_db(
response_cost=response_cost,
hashed_token=hashed_token,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_key_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_team_db(
response_cost=response_cost,
team_id=team_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_team_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_org_db(
response_cost=response_cost,
org_id=org_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_org_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_project_db(
response_cost=response_cost,
project_id=project_id,
prisma_client=prisma_client,
)
except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _update_project_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_tag_db(
response_cost=response_cost,
request_tags=request_tags,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_tag_db failed: %s",
traceback.format_exc(),
)
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_key_db",
self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_team_db",
self._update_team_db(
response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_org_db",
self._update_org_db(
response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_project_db",
self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_tag_db",
self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client),
payload=payload_copy,
)
await self._update_model_access_group_db(
response_cost=response_cost,
request_model_access_groups=request_model_access_groups,
@ -1178,96 +1172,64 @@ class DBSpendUpdateWriter:
prisma_client=prisma_client,
router=get_llm_router(),
)
_agent_id_for_spend: Final = payload_copy.get("agent_id")
try:
await self._update_agent_db(
response_cost=response_cost,
agent_id=_agent_id_for_spend,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_agent_db failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_team_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s",
traceback.format_exc(),
)
await self._run_spend_update_step(
"_update_agent_db",
self._update_agent_db(
response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_user_transaction",
self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_end_user_transaction",
self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_agent_transaction",
self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_team_transaction",
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
try:
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _record_request_error failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy,
org_id=org_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_tag_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s",
traceback.format_exc(),
)
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_org_transaction",
self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy, org_id=org_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_tag_transaction",
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
@staticmethod
async def _run_spend_update_step(
step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload
) -> None:
try:
await step
except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes
_report_spend_update_step_failure(step_name, payload=payload, exc=e)
async def _update_key_db(
self,
response_cost: float | None,
@ -2810,7 +2772,7 @@ class DBSpendUpdateWriter:
date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "")
if not date:
return
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
metadata: Final = _parse_spend_log_metadata(payload)
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
request_error_accumulator.record(
payload=payload,
@ -2861,7 +2823,7 @@ class DBSpendUpdateWriter:
# spend, token and per-entity columns written here stay either way.
request_status: Final = prisma_client.get_request_status(payload)
verbose_proxy_logger.debug("Logged request status: %s", request_status)
_metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
_metadata: Final = _parse_spend_log_metadata(payload)
usage_obj: Final = _metadata.get("usage_object", {}) or {}
if isinstance(payload["startTime"], datetime):
start_time: Final = payload["startTime"].isoformat()

2
litellm/proxy/decisions_endpoints/endpoints.py
View file

@ -228,7 +228,7 @@ async def systemone(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,

6
litellm/proxy/litellm_pre_call_utils.py
View file

@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = (
"/v1/messages",
"responses",
"files",
"interactions",
)
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
LlmProviders.VERTEX_AI.value,
)
)
_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
)
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request(
custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers
),
ProviderSpecificHeader(
custom_llm_provider=LlmProviders.DATABRICKS.value,
custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS,
extra_headers=anthropic_api_headers,
native_anthropic_messages_only=True,
),

50
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2093,7 +2093,9 @@ async def get_user_key_counts(
return result
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
order_by: Final[dict[str, str]] = {}
if sort_by is None:
@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
"spend",
"user_alias",
"user_role",
"max_budget",
"updated_at",
"sso_user_id",
"status",
"key_count",
]
if sort_by not in valid_columns:
raise HTTPException(
@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
if sort_by in {"status", "key_count"}:
return None
if sort_by in {"max_budget", "updated_at", "sso_user_id"}:
nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last"
return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"})
order_by[sort_by] = sort_order.lower()
return order_by
@ -2338,16 +2350,40 @@ async def get_users(
# Build order_by conditions
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
where=where,
skip=skip,
take=page_size,
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
computed_sort: Final = sort_by in ("status", "key_count")
sorted_ids: Final = (
await sorted_user_ids(
prisma_client,
sort_by="status" if sort_by == "status" else "key_count",
descending=sort_order.lower() == "desc",
skip=skip,
take=page_size,
role=role,
user_ids=user_ids,
user_email=user_email,
team=team,
sso_user_ids=sso_user_ids,
organization_ids=organization_ids,
search=search,
)
if computed_sort
else ()
)
page_users: Final = await UserRepository(prisma_client).table.find_many(
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"},
)
positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)}
users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users
# Get total count of user rows
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)

29
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -7397,20 +7397,26 @@ async def key_aliases(
)
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
order_by: Final[dict[str, str]] = {}
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
if sort_by is None:
return None
# Validate sort_by is a valid column
valid_columns: Final = [
valid_columns: Final = (
"spend",
"max_budget",
"created_at",
"updated_at",
"token",
"key_alias",
]
"expires",
"last_active",
"budget_reset_at",
"total_spend",
"tpm_limit",
"rpm_limit",
)
if sort_by not in valid_columns:
raise HTTPException(
status_code=400,
@ -7424,9 +7430,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
order_by[sort_by] = sort_order.lower()
return order_by
direction: Final = sort_order.lower()
nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit")
if sort_by not in (*nullable_sort_columns, "total_spend"):
return {sort_by: direction}
return [
{sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction},
{"token": "asc"},
]
def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]:
@ -7668,7 +7679,7 @@ async def list_key_helper(
verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size)
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)

46
litellm/proxy/management_endpoints/management_v1/budgets.py
View file

@ -3,6 +3,7 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Final
@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...])
SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields)
# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt.
_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰０𐒠𐴰𐵀𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑛐𑛚𑜰𑣠𑥐𑯰𑱐𑵐𑶠\U00011de0𑽐𖄰𖩠𖫀𖭐𖵰𜳰𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞗱𞥐🯰"
_DECIMAL_DIGITS: Final = "".join(
chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES)
)
_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES)
_BUDGET_DURATION_SECONDS: Final = f"""(
SELECT EXTRACT(EPOCH FROM (
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +
(parts[1] || ' ' || CASE parts[2]
WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours'
WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval
- (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
)) FROM regexp_match(
CASE lower(trim(budget_duration))
WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h'
WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d'
ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration
ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END,
'^([0-9]+)(mo|[smhdw])'
) AS duration(parts)
)"""
@dataclass(frozen=True, slots=True)
class PrismaBudgetListExecutor:
"""The database half of the budgets list. Every caller-supplied value is bound to a
@ -91,10 +118,16 @@ class PrismaBudgetListExecutor:
async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]:
clauses, params = where_sql(plan.where)
ordering: Final = ", ".join(
order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST")
if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")
else order_by_sql((key,))
for key in plan.order
).replace('"budget_duration"', _BUDGET_DURATION_SECONDS)
sql: Final = (
f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}"
+ (f" WHERE {clauses}" if clauses else "")
+ f" ORDER BY {order_by_sql(plan.order)}"
+ f" ORDER BY {ordering}"
+ f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip))
@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope:
return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}")
# budget_duration is deliberately absent from `sortable`: the column holds strings
# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d".
BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
{
"budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))),
@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
resource="budgets",
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
sortable=frozenset(
("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration")
),
searchable=frozenset(("budget_id",)),
filters=BUDGET_FILTERS,
default_sort=(SortKey(field="created_at", descending=True),),
@ -154,8 +187,9 @@ async def list_budgets(
way to page, sort or filter it.
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. `budget_id` is appended to every sort as the
`rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months
measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
`page_size` defaults to 50 and is capped at 100. Filters are
`filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]`

3
litellm/proxy/management_endpoints/policy_endpoints/endpoints.py
View file

@ -28,6 +28,7 @@ from litellm.constants import (
MAX_COMPETITOR_NAMES,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.openai.chat.guardrail_translation.handler import (
OpenAIChatCompletionsHandler,
)
@ -657,7 +658,7 @@ async def get_policy_templates(
Fetches from GitHub with automatic fallback to local backup on failure.
Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.
"""
use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
"true",
"1",
"yes",

73
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -6109,13 +6109,52 @@ async def list_team_v2(
"total_pages": 0,
}
# Build order_by conditions
valid_sort_columns: Final = ["team_id", "team_alias", "created_at"]
order_by = None
if sort_by and sort_by in valid_sort_columns:
if sort_order.lower() not in ["asc", "desc"]:
sort_order = "asc"
order_by = {sort_by: sort_order.lower()}
from litellm.proxy.management_helpers.team_list_sorting import (
COMPUTED_TEAM_SORTS,
UnsupportedTeamFilter,
sorted_team_ids,
)
valid_sort_columns: Final = (
"team_id",
"team_alias",
"created_at",
"spend",
"max_budget",
"tpm_limit",
"rpm_limit",
"updated_at",
)
sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc"
nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit")
order_by: Final = (
(
{sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}},
{"team_id": "asc"},
)
if nullable_sort
else ({sort_by: sort_direction}, {"team_id": "asc"})
if sort_by in ("spend", "updated_at")
else {sort_by: sort_direction}
if sort_by in valid_sort_columns
else None
)
computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS
sorted_ids: Final = (
await sorted_team_ids(
prisma_client,
where=where_conditions,
sort_by=sort_by or "",
descending=sort_direction == "desc",
skip=skip,
take=page_size,
)
if computed_sort
else ()
)
if isinstance(sorted_ids, UnsupportedTeamFilter):
raise HTTPException(status_code=500, detail="Unsupported team sorting filter")
# Get teams with pagination
if use_deleted_table:
@ -6130,26 +6169,32 @@ async def list_team_v2(
total_count = await _deleted_team_db(prisma_client).count(where=where_conditions)
else:
teams = await _team_db(prisma_client).find_many(
where=where_conditions,
skip=skip,
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"}, # Default sort
order=order_by if order_by else {"created_at": "desc"},
include=_INCLUDE_MODEL_TABLE,
)
# Get total count for pagination
total_count = await _team_db(prisma_client).count(where=where_conditions)
positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)}
sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams
# Calculate total pages
total_pages: Final = -(-total_count // page_size) # Ceiling division
# Aggregate virtual-key counts per team for the current page. The deleted
# table does not carry keys_count, so it is skipped.
keys_count_by_team: dict[str, int] = {}
if not use_deleted_table:
keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams)
keys_count_by_team: Final = (
await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {}
)
# Convert Prisma models to response models with members_count and keys_count
team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team)
team_list: Final = _convert_teams_to_response_models(
sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team
)
# Resolve resources inherited from access groups (single batch query)
if not use_deleted_table:

4
litellm/proxy/management_endpoints/types.py
View file

@ -5,7 +5,7 @@ Might include fastapi/proxy requirements.txt related imports
"""
from collections.abc import Iterable, Sequence
from typing import Any, Final, cast
from typing import Final, cast
from fastapi_sso.sso.base import OpenID
@ -94,4 +94,4 @@ def get_litellm_user_role(role_str: object) -> LitellmUserRoles | None:
class CustomOpenID(OpenID):
team_ids: list[str]
user_role: LitellmUserRoles | None = None
extra_fields: dict[str, Any] | None = None
extra_fields: dict[str, object] | None = None

59
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -152,7 +152,7 @@ from litellm.proxy.utils import (
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import SSOConfigRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.user_repository import UserRepository, free_sso_user_creation
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
@ -1074,25 +1074,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token(
)
async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None:
"""Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license."""
if premium_user is True:
return
if prisma_client is None:
raise ProxyException(
message=CommonProxyErrors.db_not_connected_error.value,
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
billable_users: Final = await UserRepository(prisma_client).count_billable_users()
if billable_users and billable_users > 5:
raise ProxyException(
message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it",
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
class _SSOUserAdmissionError(ProxyException):
pass
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
@ -1112,8 +1095,6 @@ async def google_login(
from litellm.proxy.proxy_server import (
cli_sso_session_cache,
general_settings,
premium_user,
prisma_client,
saml_sso_cache,
user_custom_ui_sso_sign_in_handler,
)
@ -1128,15 +1109,6 @@ async def google_login(
if admin_ui_is_disabled and not is_cli_sso_login:
return admin_ui_disabled()
####### Check if user is a Enterprise / Premium User #######
if (
microsoft_client_id is not None
or google_client_id is not None
or generic_client_id is not None
or SAMLAuthHandler.is_saml_configured()
):
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
####### Detect DB + MASTER KEY in .env #######
missing_env_vars: Final = show_missing_vars_in_env()
if missing_env_vars is not None:
@ -1963,6 +1935,8 @@ async def get_user_info_from_db(
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e)
@ -2328,7 +2302,6 @@ async def saml_callback(request: Request):
general_settings,
jwt_handler,
master_key,
premium_user,
prisma_client,
saml_sso_cache,
)
@ -2352,8 +2325,6 @@ async def saml_callback(request: Request):
user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data)
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
ui_access_mode: Final = general_settings.get("ui_access_mode", None)
relay_state: Final = post_data.get("RelayState")
cp_return_to: Final[str | None] = (
@ -2821,12 +2792,20 @@ async def insert_sso_user(
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
response: Final = await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
from litellm.proxy.proxy_server import premium_user
return response
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
try:
return await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
except ProxyException as error:
if premium_user is True:
raise
raise _SSOUserAdmissionError(
message=error.message, type=error.type, param=error.param, code=error.code
) from error
DefaultTeamJoin: TypeAlias = Callable[
@ -3567,6 +3546,8 @@ class SSOAuthenticationHandler:
prisma_client=prisma_client,
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if (
user_info is None

9
litellm/proxy/management_endpoints/usage_endpoints/__init__.py
View file

@ -1,9 +0,0 @@
"""
Usage endpoints package.
Re-exports the router from endpoints module.
"""
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401
router,
)

627
litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py
View file

@ -1,627 +0,0 @@
"""
AI Usage Chat - uses LLM tool calling to answer questions about
usage/spend data by querying the aggregated daily activity endpoints.
"""
import json
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
from datetime import date
from typing import Final, Literal, NamedTuple, Protocol, cast, overload
from fastapi import HTTPException
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.proxy._types import CommonProxyErrors
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
from litellm.types.utils import ChatCompletionMessageToolCall
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
USAGE_AI_TEMPERATURE: Final = 0.2
TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend"
TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend"
TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend"
ENTITY_FIELD_USER: Final = "user_id"
ENTITY_FIELD_TEAM: Final = "team_id"
ENTITY_FIELD_TAG: Final = "tag"
PAGINATED_PAGE_SIZE: Final = 200
MAX_CHAT_MESSAGES: Final = 20
TOP_N_MODELS: Final = 15
TOP_N_PROVIDERS: Final = 10
TOP_N_KEYS: Final = 10
# ---------------------------------------------------------------------------
# Types
# ---------------------------------------------------------------------------
class SSEStatusEvent(TypedDict):
type: Literal["status"]
message: str
class SSEToolCallEvent(TypedDict, total=False):
type: Literal["tool_call"]
tool_name: str
tool_label: str
arguments: dict[str, str]
status: Literal["running", "complete", "error"]
error: str
class SSEChunkEvent(TypedDict):
type: Literal["chunk"]
content: str
class SSEDoneEvent(TypedDict):
type: Literal["done"]
class SSEErrorEvent(TypedDict):
type: Literal["error"]
message: str
SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent
class _EntityEntry(TypedDict, total=False):
metrics: ReadOnly[Mapping[str, float]]
metadata: ReadOnly[Mapping[str, str]]
class _DayDump(TypedDict, total=False):
breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]]
class _EntityTotal(NamedTuple):
"""Running per-entity totals accumulated while summarising a usage dump."""
alias: str
spend: float
requests: float
tokens: float
class _UsageDump(Protocol):
@overload
def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ...
@overload
def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ...
class _ToolFunctionDef(TypedDict):
name: ReadOnly[str]
description: ReadOnly[str]
parameters: ReadOnly[Mapping[str, object]]
class _ToolDef(TypedDict):
type: ReadOnly[str]
function: ReadOnly[_ToolFunctionDef]
class ToolHandler(TypedDict):
fetch: Callable[..., Awaitable[_UsageDump]]
summarise: Callable[[_UsageDump], str]
label: str
# ---------------------------------------------------------------------------
# Tool definitions (OpenAI function-calling schema)
# ---------------------------------------------------------------------------
_DATE_PARAMS: Final = {
"start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"},
"end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"},
}
_TOOL_USAGE: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_usage_data",
"description": (
"Fetch aggregated global usage/spend data. Returns daily spend, "
"token counts, request counts, and breakdowns by model, provider, "
"and API key. Use for overall spend, top models, top providers."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"user_id": {
"type": "string",
"description": "Optional user ID filter. Omit for global view.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TEAM: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_team_usage_data",
"description": (
"Fetch usage/spend data broken down by team. Use for questions "
"like 'which team spends the most' or 'show me team X usage'."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"team_ids": {
"type": "string",
"description": "Optional comma-separated team IDs. Omit for all teams.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TAG: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_tag_usage_data",
"description": (
"Fetch usage/spend data broken down by tag. Tags are labels "
"attached to requests (features, environments, credentials)."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"tags": {
"type": "string",
"description": "Optional comma-separated tag names. Omit for all tags.",
},
},
"required": ["start_date", "end_date"],
},
},
}
TOOLS_BASE: Final = [_TOOL_USAGE]
TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG]
def get_tools_for_role(is_admin: bool) -> list[_ToolDef]:
"""Return the tool list appropriate for the user's role."""
return TOOLS_ADMIN if is_admin else TOOLS_BASE
_SYSTEM_PROMPT_BASE: Final = (
"You are an AI assistant embedded in the LiteLLM Usage dashboard. "
"You help users understand their LLM API spend and usage data.\n\n"
"ALWAYS call the appropriate tool(s) first to fetch data before answering. "
"You may call multiple tools if the question spans different dimensions.\n\n"
"Guidelines:\n"
"- Be concise and specific. Use exact numbers from the data.\n"
"- Format costs as dollar amounts (e.g. $12.34).\n"
"- When comparing entities, show a ranked list.\n"
"- If data is empty or no results found, say so clearly.\n"
"- Do not hallucinate data — only use what the tools return.\n"
"- Today's date will be provided below. Use it to interpret relative dates "
"like 'this week', 'this month', 'last 7 days', etc."
)
_TOOL_DESCRIPTIONS_ADMIN: Final = (
"You have access to these tools:\n"
"- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n"
"- `get_team_usage_data`: Team-level usage breakdown\n"
"- `get_tag_usage_data`: Tag-level usage breakdown\n\n"
)
_TOOL_DESCRIPTIONS_BASE: Final = (
"You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n"
)
def _build_system_prompt(is_admin: bool) -> str:
"""Build role-appropriate system prompt with today's date."""
tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE
return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}"
# keep a public reference for test assertions
SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE
# ---------------------------------------------------------------------------
# Data fetchers
# ---------------------------------------------------------------------------
def _parse_csv_ids(raw: str | None) -> list[str] | None:
if not raw:
return None
return [t.strip() for t in raw.split(",") if t.strip()]
async def _query_activity(
table_name: str,
entity_id_field: str,
entity_id: str | list[str] | None,
start_date: str,
end_date: str,
*,
use_aggregated: bool = False,
) -> SpendAnalyticsPaginatedResponse:
"""Shared helper that calls the daily activity query layer."""
from litellm.proxy.management_endpoints.common_daily_activity import (
daily_activity_repository,
daily_activity_scope,
get_daily_activity,
get_daily_activity_aggregated,
)
from litellm.proxy.proxy_server import prisma_client
if use_aggregated:
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
repository: Final = daily_activity_repository(prisma_client)
scope: Final = daily_activity_scope(
table_name,
entity_id_field,
entity_id,
None,
None,
start_date,
end_date,
None,
None,
)
return await get_daily_activity_aggregated(
repository,
scope,
)
return await get_daily_activity(
prisma_client=prisma_client,
table_name=table_name,
entity_id_field=entity_id_field,
entity_id=entity_id,
entity_metadata_field=None,
start_date=start_date,
end_date=end_date,
model=None,
api_key=None,
page=1,
page_size=PAGINATED_PAGE_SIZE,
)
async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_USER_SPEND,
ENTITY_FIELD_USER,
user_id,
start_date,
end_date,
use_aggregated=True,
)
return resp.model_dump(mode="json")
async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TEAM_SPEND,
ENTITY_FIELD_TEAM,
_parse_csv_ids(team_ids),
start_date,
end_date,
)
return resp.model_dump(mode="json")
async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TAG_SPEND,
ENTITY_FIELD_TAG,
_parse_csv_ids(tags),
start_date,
end_date,
)
return resp.model_dump(mode="json")
# ---------------------------------------------------------------------------
# Summarisers — convert raw JSON to concise text the LLM can reason over
# ---------------------------------------------------------------------------
def _accumulate_breakdown(
results: Sequence[_DayDump], dimension: str, fields: Sequence[str]
) -> dict[str, dict[str, float]]:
"""Aggregate a single breakdown dimension across days."""
totals: Final[dict[str, dict[str, float]]] = {}
for day in results:
for key, entry in day.get("breakdown", {}).get(dimension, {}).items():
if key not in totals:
totals[key] = {f: 0.0 for f in fields}
m = entry.get("metrics", {})
for f in fields:
totals[key][f] += m.get(f, 0)
return totals
def _ranked_lines(
totals: dict[str, dict[str, float]],
fmt: Callable[[str, dict[str, float]], str],
limit: int,
) -> list[str]:
"""Sort by spend descending, format each entry, and truncate."""
return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]]
def _summarise_usage_data(data: _UsageDump) -> str:
meta: Final = data.get("metadata", {})
results: Final = data.get("results", [])
header: Final = (
f"Total Spend: ${meta.get('total_spend', 0):.4f}\n"
f"Total Requests: {meta.get('total_api_requests', 0)}\n"
f"Successful: {meta.get('total_successful_requests', 0)} | "
f"Failed: {meta.get('total_failed_requests', 0)}\n"
f"Total Tokens: {meta.get('total_tokens', 0)}"
)
models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"])
providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"])
model_lines: Final = _ranked_lines(
models,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)",
TOP_N_MODELS,
)
provider_lines: Final = _ranked_lines(
providers,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)",
TOP_N_PROVIDERS,
)
sections = [header, ""]
sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""]
sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"])
return "\n".join(sections)
def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str:
"""Summarise team/tag entity usage data."""
results: Final = data.get("results", [])
if not results:
return f"No {entity_label} usage data found for the given date range."
totals: Final[dict[str, _EntityTotal]] = {}
for day in results:
for eid, entry in day.get("breakdown", {}).get("entities", {}).items():
previous = totals.get(eid)
m = entry.get("metrics", {})
totals[eid] = _EntityTotal(
alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid),
spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0),
requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0),
tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0),
)
lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""]
for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend):
label = d.alias if d.alias != eid else eid
lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Tool dispatch registry
# ---------------------------------------------------------------------------
TOOL_HANDLERS: Final[dict[str, ToolHandler]] = {
"get_usage_data": ToolHandler(
fetch=_fetch_usage_data,
summarise=_summarise_usage_data,
label="global usage data",
),
"get_team_usage_data": ToolHandler(
fetch=_fetch_team_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Team"),
label="team usage data",
),
"get_tag_usage_data": ToolHandler(
fetch=_fetch_tag_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Tag"),
label="tag usage data",
),
}
# ---------------------------------------------------------------------------
# SSE streaming
# ---------------------------------------------------------------------------
def _sse(event: SSEEvent) -> str:
return f"data: {json.dumps(event)}\n\n"
def _resolve_fetch_kwargs(
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> dict[str, str]:
"""Build keyword arguments for a tool's fetch function."""
start_date: Final = fn_args.get("start_date", "")
end_date: Final = fn_args.get("end_date", "")
if not start_date or not end_date:
raise ValueError("Missing required start_date or end_date from tool arguments")
kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date}
if fn_name == "get_usage_data":
if not is_admin:
if user_id is None:
# Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
# should have already rejected this. If we ever reach here it means
# a future caller invoked the helper without scoping — fail loudly
# rather than issuing an unfiltered global query.
raise ValueError(
"Non-admin caller has user_id=None; refusing to issue an "
"unscoped query. Endpoint-level guard missing."
)
kwargs["user_id"] = user_id
elif fn_args.get("user_id"):
kwargs["user_id"] = fn_args["user_id"]
elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"):
kwargs["team_ids"] = fn_args["team_ids"]
elif fn_name == "get_tag_usage_data" and fn_args.get("tags"):
kwargs["tags"] = fn_args["tags"]
return kwargs
async def _execute_tool_call(
handler: ToolHandler,
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> str:
"""Run a single tool and return the summarised result text, or the reason the query
layer rejected its dates so the model can correct them."""
kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin)
try:
raw_data: Final = await handler["fetch"](**kwargs)
except HTTPException as e:
reason: Final = _invalid_date_reason(e)
if reason is None:
raise
return f"Invalid date range: {reason}"
return handler["summarise"](raw_data)
def _invalid_date_reason(error: HTTPException) -> str | None:
"""The reason in the daily activity query layer's 400 for invalid or reversed dates."""
detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict
match detail:
case {"error": str() as reason} if error.status_code == 400:
return reason
case _:
return None
async def _process_tool_call(
tc: ChatCompletionMessageToolCall,
chat_messages: list[Mapping[str, object]],
user_id: str | None,
is_admin: bool,
) -> AsyncIterator[str]:
"""Execute a single tool call, yielding SSE events for status."""
fn_name: Final = tc.function.name
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None
if fn_name is None or fn_name not in allowed_names or not handler:
chat_messages.append(
{
"role": "tool",
"tool_call_id": tc.id,
"content": f"Tool not available: {fn_name}",
}
)
return
tool_event_base: Final = {
"type": "tool_call",
"tool_name": fn_name,
"tool_label": handler["label"],
"arguments": fn_args,
}
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"}))
try:
tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin)
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"}))
except Exception as e:
verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e)
tool_result = f"Error fetching {handler['label']}. Please try again."
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"}))
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
"""Stream the final LLM response after tool results are appended."""
yield _sse({"type": "status", "message": "Analyzing results..."})
response: Final = await litellm.acompletion(
model=model,
messages=chat_messages,
stream=True,
temperature=USAGE_AI_TEMPERATURE,
)
async for chunk in response:
delta = chunk.choices[0].delta.content
if delta:
yield _sse({"type": "chunk", "content": delta})
async def stream_usage_ai_chat(
messages: list[dict[str, str]],
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
chat_messages: Final[list[Mapping[str, object]]] = [
{"role": "system", "content": _build_system_prompt(is_admin)},
*truncated,
]
try:
yield _sse({"type": "status", "message": "Thinking..."})
tools: Final = get_tools_for_role(is_admin)
response: Final = await litellm.acompletion(
model=resolved_model,
messages=chat_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
)
choice: Final = response.choices[0]
if not choice.message.tool_calls:
if choice.message.content:
yield _sse({"type": "chunk", "content": choice.message.content})
yield _sse({"type": "done"})
return
chat_messages.append(choice.message.model_dump())
for tc in choice.message.tool_calls:
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
yield event
async for event in _stream_final_response(resolved_model, chat_messages):
yield event
yield _sse({"type": "done"})
except Exception as e:
verbose_proxy_logger.error("AI usage chat failed: %s", e)
yield _sse(
{
"type": "error",
"message": "An internal error occurred. Please try again.",
}
)

77
litellm/proxy/management_endpoints/usage_endpoints/endpoints.py
View file

@ -1,77 +0,0 @@
"""
USAGE AI CHAT ENDPOINTS
/usage/ai/chat - Stream AI chat responses about usage data
"""
from typing import Final, Literal
from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from pydantic import Field
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
wrap_sse_stream_with_keepalive_pings,
)
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
class ChatMessage(LiteLLMBaseModel):
role: Literal["user", "assistant"]
content: str
class UsageAIChatRequest(LiteLLMBaseModel):
messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)")
model: str | None = Field(default=None, description="Model to use for AI chat")
@router.post(
"/usage/ai/chat",
tags=["Budget & Spend Tracking"],
dependencies=[Depends(user_api_key_auth)],
)
async def usage_ai_chat(
data: UsageAIChatRequest,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
AI chat about usage data. Streams SSE events with the AI response.
The AI agent has access to tools that query aggregated daily activity data.
"""
from litellm.proxy.management_endpoints.common_utils import (
require_caller_user_id_for_non_admin,
user_api_key_has_admin_view,
)
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
stream_usage_ai_chat,
)
is_admin: Final = user_api_key_has_admin_view(user_api_key_dict)
if is_admin:
user_id = user_api_key_dict.user_id
else:
user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
return StreamingResponse(
wrap_sse_stream_with_keepalive_pings(
stream_usage_ai_chat(
messages=messages,
model=data.model,
user_id=user_id,
is_admin=is_admin,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)

109
litellm/proxy/management_helpers/team_list_sorting.py Normal file
View file

@ -0,0 +1,109 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from litellm.proxy.utils import PrismaClient
COMPUTED_TEAM_SORTS: Final = MappingProxyType(
{
"organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') "
'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), '
"NULLIF(t.organization_id, ''))",
"members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' "
"THEN jsonb_array_length(t.members_with_roles) ELSE 0 END",
"models": "cardinality(t.models)",
"keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)',
}
)
_WHERE: Final = TypeAdapter(dict[str, JsonValue])
@dataclass(frozen=True, slots=True)
class UnsupportedTeamFilter:
field: str
def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
if field not in ("team_id", "team_alias", "organization_id"):
return UnsupportedTeamFilter(field)
column: Final = f't."{field}"'
match value:
case str(text):
return f"{column} = ${index}", (text,)
case {"in": list(values)}:
return f"{column} = ANY(${index}::text[])", (values,)
case {"contains": str(text), "mode": "insensitive"}:
return f"{column} ILIKE ${index}", (f"%{text}%",)
case {"startsWith": str(text)}:
return f"{column} LIKE ${index}", (f"{text}%",)
case _:
return UnsupportedTeamFilter(field)
def _group_tokens(
field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue]
) -> tuple[str | tuple[str, JsonValue], ...]:
children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value)
if not children:
return ("FALSE" if field == "OR" else "TRUE",)
return (
"(",
*chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)),
")",
)
def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal
fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying
params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order
while pending:
match pending.pop():
case str(fragment):
fragments.append(fragment)
case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)):
pending.extend(reversed(_group_tokens(field, value)))
case (field, value):
match _field_sql(field, value, len(params) + 1):
case UnsupportedTeamFilter() as failure:
return failure
case (fragment, values):
fragments.append(fragment)
params.extend(values)
return "".join(fragments), tuple(params)
class _TeamId(BaseModel):
model_config = ConfigDict(frozen=True)
team_id: str
_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...])
async def sorted_team_ids(
prisma_client: PrismaClient,
*,
where: Mapping[str, object],
sort_by: str,
descending: bool,
skip: int,
take: int,
) -> Sequence[str] | UnsupportedTeamFilter:
result: Final = _where_sql(_WHERE.validate_python(where))
if isinstance(result, UnsupportedTeamFilter):
return result
predicate, params = result
expression: Final = COMPUTED_TEAM_SORTS[sort_by]
direction: Final = "DESC" if descending else "ASC"
sql: Final = (
f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} '
f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC "
f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip))
return tuple(row.team_id for row in rows)

77
litellm/proxy/management_helpers/user_list_sorting.py Normal file
View file

@ -0,0 +1,77 @@
from collections.abc import Sequence
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy.utils import PrismaClient
class _UserId(BaseModel):
model_config = ConfigDict(frozen=True)
user_id: str
_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...])
async def sorted_user_ids(
prisma_client: PrismaClient,
*,
sort_by: Literal["status", "key_count"],
descending: bool,
skip: int,
take: int,
role: str | None,
user_ids: str | None,
user_email: str | None,
team: str | None,
sso_user_ids: str | None,
organization_ids: str | None,
search: str | None,
) -> Sequence[str]:
ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip())
sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip())
org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip())
expression: Final = (
"CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END"
if sort_by == "status"
else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k '
"WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)"
)
direction: Final = "DESC" if descending else "ASC"
sql: Final = f"""
SELECT u.user_id FROM "LiteLLM_UserTable" u
WHERE ($1::text IS NULL OR u.user_role = $1)
AND ($2::text[] IS NULL
OR CASE WHEN cardinality($2::text[]) = 1
THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%'
ELSE u.user_id = ANY($2::text[]) END)
AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%')
AND ($4::text IS NULL OR $4 = ANY(u.teams))
AND ($5::boolean OR u.sso_user_id = ANY($6::text[]))
AND (cardinality($7::text[]) = 0 OR EXISTS (
SELECT 1 FROM "LiteLLM_OrganizationMembership" m
WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[])))
AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%'
OR u.user_email ILIKE '%' || $8 || '%')
ORDER BY {expression} {direction}, u.user_id ASC
LIMIT $9 OFFSET $10
"""
rows: Final = _USER_IDS.validate_python(
await prisma_client.db.query_raw(
sql,
role or None,
ids if user_ids else None,
user_email,
team,
sso_user_ids is None,
sso_ids,
org_ids,
search or None,
take,
skip,
*((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()),
)
)
return tuple(row.user_id for row in rows)

13
litellm/proxy/proxy_server.py
View file

@ -12210,7 +12210,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
async def model_list(
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
return_wildcard_routes: bool | None = False,
return_wildcard_routes: bool | None = None,
team_id: str | None = None,
include_model_access_groups: bool | None = False,
only_model_access_groups: bool | None = False,
@ -12226,6 +12226,9 @@ async def model_list(
This is just for compatibility with openai projects like aider.
Query Parameters:
- return_wildcard_routes: When true, also list wildcard routes such as `openai/*`.
Defaults to `general_settings.return_wildcard_routes`, and an explicit
value on the request wins over it
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
provider's model catalog. When present, the response is Codex's own
catalog shape (`{"models": [...]}`) built from the same listing, with
@ -12254,6 +12257,9 @@ async def model_list(
global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
settings: Final = cast(dict[str, object], general_settings) # any-ok: legacy settings
include_wildcard_routes: Final = (
return_wildcard_routes if return_wildcard_routes is not None else settings.get("return_wildcard_routes") is True
)
from litellm.llms.anthropic.common_utils import (
create_anthropic_model_list_response,
@ -12343,7 +12349,7 @@ async def model_list(
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=return_wildcard_routes or False,
return_wildcard_routes=include_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups or False,
@ -12411,7 +12417,7 @@ async def model_list(
team_id=team_id,
include_model_access_groups=include_model_access_groups or False,
only_model_access_groups=only_model_access_groups or False,
return_wildcard_routes=return_wildcard_routes or False,
return_wildcard_routes=include_wildcard_routes,
user_api_key_cache=user_api_key_cache,
)
@ -19657,6 +19663,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro
"proxy_config_reload_interval_seconds": "Integer",
"pass_through_endpoints": "PydanticModel",
"store_model_in_db": "Boolean",
"return_wildcard_routes": "Boolean",
"store_prompts_in_spend_logs": "Boolean",
"disable_requester_ip_address_logging": "Boolean",
"disable_fallbacks_on_per_model_rate_limits": "Boolean",

3
litellm/proxy/public_endpoints/public_endpoints.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import (
get_blog_posts,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy._types import (
CommonProxyErrors,
)
@ -564,7 +565,7 @@ async def _resolve_autorouter_presets(
url: str,
fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],
) -> Mapping[str, AutoRouterPresetRecord]:
if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
return _load_bundled_autorouter_presets()
try:
return await fetch(url)

4
litellm/proxy/spend_tracking/vantage_endpoints.py
View file

@ -122,7 +122,9 @@ async def _get_vantage_settings():
encrypted_api_key: Final = settings.get("api_key")
if encrypted_api_key:
decrypted_api_key = decrypt_value_helper(encrypted_api_key, key="vantage_api_key", exception_type="error")
decrypted_api_key: Final = decrypt_value_helper(
encrypted_api_key, key="vantage_api_key", exception_type="error"
)
if decrypted_api_key is None:
raise HTTPException(
status_code=500,

78
litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py
View file

@ -1,5 +1,7 @@
import asyncio
import os
import re
import xml.etree.ElementTree as ET
from collections import Counter
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass
@ -12,12 +14,14 @@ from pydantic import ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest"
CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/"
CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml"
RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}"
LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5
LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60
@ -26,7 +30,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info"
_RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S")
_NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b")
_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/")
_Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"]
_PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"})
@ -45,6 +49,16 @@ class LatestReleaseUnavailable:
reason: str
@dataclass(frozen=True, slots=True)
class ChangelogEntry:
version: tuple[int, int, int]
url: str
@property
def tag(self) -> str:
return "v" + ".".join(str(part) for part in self.version)
class _GitHubRelease(LiteLLMBaseModel):
tag_name: str
html_url: str
@ -76,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock:
return _latest_release_fetch_lock
def _default_release_check_enabled() -> bool:
return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true"
def _bucket_for(line: str) -> _Bucket | None:
if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None:
return None
@ -91,9 +109,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]:
return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None))
def _stable_version(tag_name: str) -> tuple[int, int, int] | None:
match: Final = _STABLE_TAG_PATTERN.match(tag_name)
return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3)))
def _changelog_entry(link: str) -> ChangelogEntry | None:
match: Final = _CHANGELOG_PAGE_PATTERN.match(link)
if match is None:
return None
return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link)
def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None:
"""The highest stable release with a page in the changelog feed. Release candidate pages don't count."""
links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link"))
entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None)
return max(entries, key=lambda entry: entry.version, default=None)
def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
@ -102,7 +129,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch))
def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo:
def summarize_release_line(
entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]
) -> LatestReleaseInfo:
"""A patch release's notes only list its backports, so count the whole minor line it belongs to."""
counts: Final = sum(
(Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)),
@ -113,7 +142,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease
new_features=counts.get("new_features", 0),
bug_fixes=counts.get("bug_fixes", 0),
other_updates=counts.get("other_updates", 0),
release_url=latest.html_url,
release_url=entry.url,
)
@ -137,25 +166,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe
return release
async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable:
response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
if response.status_code != 200:
return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}")
try:
entry: Final = latest_changelog_entry(response.content)
except ET.ParseError as e:
return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}")
return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry
async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable:
try:
entry: Final = await _fetch_changelog_entry(client)
if isinstance(entry, LatestReleaseUnavailable):
return entry
latest: Final = parse_github_release(
await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
)
if isinstance(latest, LatestReleaseUnavailable):
return latest
version: Final = _stable_version(latest.tag_name)
if version is None:
return summarize_release_line(latest, ())
earlier: Final = await asyncio.gather(
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version))
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version))
)
except httpx.HTTPError as e:
return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}")
unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None)
if unavailable is not None:
return unavailable
return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)))
return summarize_release_line(
entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))
)
async def get_latest_release_info(
@ -188,12 +230,16 @@ async def latest_release_info(
client: Annotated[_AsyncGetClient, Depends(_default_client)],
cache: Annotated[InMemoryCache, Depends(_default_cache)],
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)],
) -> LatestReleaseInfo | None:
"""
Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
split into new features, bug fixes and other updates.
Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request.
"""
if not release_check_enabled:
return None
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
if isinstance(result, LatestReleaseUnavailable):
verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason)

15
litellm/proxy/utils.py
View file

@ -1294,6 +1294,7 @@ def _failure_usage_to_lift(
provider_billed: bool = False,
deployment_id: str | None = None,
base_model: str | None = None,
dispatched_call_type: str | None = None,
) -> tuple[object, object] | None:
current_attempt: Final = _current_failure_usage_to_lift(
model_call_details,
@ -1302,6 +1303,7 @@ def _failure_usage_to_lift(
provider_billed,
deployment_id,
base_model,
dispatched_call_type=dispatched_call_type,
)
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ()))
if not failed_attempts:
@ -6030,13 +6032,12 @@ class PrismaClient:
elif table_name == "user":
db_data = self.jsonify_object(data=data)
try:
new_user_row: Final = await UserRepository(self).table.upsert(
where={"user_id": data["user_id"]},
data={
"create": {**db_data},
"update": {}, # don't do anything if it already exists
},
)
new_user_row: Final = await UserRepository(self).upsert_user(db_data)
if new_user_row is None:
raise HTTPException(
status_code=403,
detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.",
)
except Exception as e:
if (
"Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`"

60
litellm/repositories/user_repository.py
View file

@ -2,8 +2,11 @@
User repository for database operations on LiteLLM_UserTable.
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Generator, Mapping, Sequence
from contextlib import AbstractAsyncContextManager, contextmanager
from contextvars import ContextVar
from itertools import chain
from typing import TYPE_CHECKING, Final, Protocol
@ -35,6 +38,16 @@ ORDER BY p.user_id
"""
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None)
@contextmanager
def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]:
token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None)
try:
yield
finally:
_FREE_SSO_CREATION.reset(token)
class _UserDb(Protocol):
@ -42,6 +55,12 @@ class _UserDb(Protocol):
def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ...
class _UserTransaction(_UserDb, Protocol):
async def execute_raw(self, query: str) -> int: ...
async def query_raw(self, query: str) -> object: ...
class _PrismaClientView(Protocol):
@property
def db(self) -> _UserDb: ...
@ -49,6 +68,16 @@ class _PrismaClientView(Protocol):
@property
def writer_db(self) -> _UserDb: ...
def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ...
async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int:
from prisma import Json
total: Final = await table.count()
deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Repository for user database operations."""
@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
null, or true all count, so seats are counted as total users minus
the deactivated ones.
"""
from prisma import Json # pyright: ignore[reportUnknownVariableType]
return await _count_billable_users(self.table)
total: Final = await self.count()
deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None":
"""Return the user row, or None when a new free SSO user would exceed capacity."""
scope: Final = _FREE_SSO_CREATION.get()
if (
scope is None
or (scope[0] is not None and scope[0] != data["user_id"])
or scope[1] is not asyncio.current_task()
):
return await self.table.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
client: Final[_PrismaClientView] = self.prisma_client
async with client.tx() as transaction:
await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)")
existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]})
if existing is not None:
return existing
if await _count_billable_users(transaction.litellm_usertable) >= 5:
return None
return await transaction.litellm_usertable.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
async def create_user(
self,

68
litellm/responses/main.py
View file

@ -6,10 +6,10 @@ from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
import httpx
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from typing_extensions import (
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
assert_never,
@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.litellm_completion_transformation.compaction_items import (
with_compaction_items_as_summaries,
@ -85,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.types.responses.main import *
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
CallTypes,
CostPerToken,
CustomPricingLiteLLMParams,
all_litellm_params,
)
from litellm.utils import (
ProviderConfigManager,
client,
@ -544,6 +552,47 @@ async def aresponses_api_with_mcp(
return response
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
return MappingProxyType(
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
)
class _DeploymentModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: object = None
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
)
def _is_router_deployment(deployment_model_info: object) -> bool:
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
return None
return _COST_PER_TOKEN.validate_python(
litellm_params.model_dump(
include={
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
},
exclude_none=True,
)
)
def _bridges_to_chat_completions(
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
) -> bool:
@ -1466,6 +1515,12 @@ def responses(
# get llm provider logic
litellm_params: Final = GenericLiteLLMParams(**kwargs)
affinity_api_base: Final = (
get_openai_credentials(api_base=litellm_params.api_base).api_base
if custom_llm_provider == "openai"
else litellm_params.api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
try:
effective_extra_headers: Final = (
add_provider_affinity_header(
@ -1479,8 +1534,9 @@ def responses(
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
if litellm_params.provider_affinity_header is not None
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
else extra_headers
)
except ValueError as affinity_error:
@ -1664,6 +1720,7 @@ def responses(
optional_params=dict(responses_api_request_params),
litellm_params={
**responses_api_request_params,
**_per_call_custom_pricing(litellm_params),
"aresponses": _is_async,
"litellm_call_id": litellm_call_id,
"model_info": kwargs.get("model_info"),
@ -1674,6 +1731,9 @@ def responses(
custom_llm_provider=custom_llm_provider,
)
if not _is_router_deployment(deployment_model_info):
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
input = with_compaction_items_as_summaries(
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))

55
litellm/router.py
View file

@ -14186,6 +14186,43 @@ class Router:
deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
)
@staticmethod
def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool:
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
declared: Final = model_info.get("supports_vision")
if declared is not None:
return declared is False
raw_params: Final = plain_deployment.get("litellm_params") or {}
params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts
if not params:
return False
base_model: Final = model_info.get("base_model") or params.get("base_model")
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return litellm.utils.is_vision_explicitly_disabled(
base_model if isinstance(base_model, str) else model, custom_llm_provider
)
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
verbose_router_logger.debug(
"litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
)
return False
@staticmethod
def _request_carries_image(
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
request_kwargs: Mapping[str, object],
) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
structured: Final = resolve_structured_messages(
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
)
return bool(structured) and request_contains_image_content(structured)
@staticmethod
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
@ -14212,6 +14249,7 @@ class Router:
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- request carries an image and the deployment declares supports_vision false, unless every candidate does
- [TODO] function call and model doesn't support function calling
"""
@ -14221,7 +14259,7 @@ class Router:
# We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
_returned_deployments = list(healthy_deployments)
invalid_model_indices: Final = set() # Use set for O(1) membership checks
invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks
_context_window_error = False
_potential_error_str = ""
@ -14371,9 +14409,20 @@ class Router:
model=model,
llm_provider="",
)
if len(excluded_indices) > 0:
remaining: Final = tuple(
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
)
text_only_indices: Final[frozenset[int]] = (
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
if remaining and self._request_carries_image(messages, request_settings)
else frozenset()
)
dropped_indices: Final[frozenset[int]] = excluded_indices | (
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
)
if len(dropped_indices) > 0:
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
return _returned_deployments

1
litellm/rust_bridge/catalog.py
View file

@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
}
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.PROVIDER_AFFINITY,
Feature.SAFEGUARDS_CLASSIFIER,
Feature.PRE_REQUEST_HOOK,
Feature.WEBSEARCH_INTERCEPTION,

2
litellm/rust_bridge/detectors.py
View file

@ -128,7 +128,7 @@ def _custom_loggers(callbacks: Iterable[object]) -> tuple[CustomLogger, ...]:
def overridden(method: str, callbacks: Iterable[object]) -> bool:
base: Final = getattr(CustomLogger, method)
base: Final[object] = getattr(CustomLogger, method)
return any(getattr(type(callback), method, base) is not base for callback in _custom_loggers(callbacks))

7
litellm/rust_bridge/messages/detectors.py
View file

@ -5,10 +5,15 @@ from __future__ import annotations
from typing import Final
import litellm
from litellm.rust_bridge.detectors import overridden, present, streaming
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
def provider_affinity(request: NativeCall) -> bool:
return is_opencode_api_base(api_base(request))
def anthropic_workload_identity(request: NativeCall) -> bool:
"""Python mints an Anthropic federation token when no static credential is configured."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo

Some files were not shown because too many files have changed in this diff Show more