chore: merge upstream main into prometheus budget metrics

Signed-off-by: amarrtech <272048731+amarrtech@users.noreply.github.com>
This commit is contained in:
amarrtech 2026-10-24 10:55:26 -07:00
commit c2b842d8e2
345 changed files with 31781 additions and 6585 deletions

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

View file

@ -0,0 +1,91 @@
name: Interactions spec canary
on:
schedule:
- cron: "17 7 * * *"
pull_request:
paths:
- .github/workflows/interactions-spec-canary.yml
- tests/unit/interactions/**
- litellm/types/interactions/**
workflow_dispatch:
permissions: {}
concurrency: interactions-spec-canary
jobs:
interactions-spec-canary:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
compliance: ${{ steps.compliance.outcome }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
- name: Run the compliance tests against Google's live spec
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
tracking-issue:
name: tracking-issue
needs: interactions-spec-canary
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
steps:
- name: File the tracking issue unless the compliance step passed, close it when it did
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const title = "Interactions spec canary is red";
const outcome = process.env.COMPLIANCE_OUTCOME;
const runUrl = process.env.RUN_URL;
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
const open = found.items.find((issue) => issue.title === title);
if (outcome === "success") {
if (open === undefined) return;
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
return;
}
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
if (open !== undefined) {
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
return;
}
await github.rest.issues.create({
...context.repo,
title,
assignees: ["mateo-berri"],
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
});

View file

@ -425,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -61,6 +61,7 @@ RUN apk add --no-cache \
python-3.13-dev \
rust \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
@ -141,7 +142,7 @@ USER root
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -59,6 +59,7 @@ RUN apk add --no-cache \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app

View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
done
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

View file

@ -30,7 +30,7 @@ RUN uv venv --python python && \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

View file

@ -8202,18 +8202,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.9"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
dependencies = [
"rustls-pki-types",
]

View file

@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
strum = { version = "0.28.0", features = ["derive"] }
url = "2.5.8"
percent-encoding = "2.3"
webpki-roots = "1"
webpki-roots = "1.0.10"
time = { version = "0.3.53", features = ["parsing"] }
criterion = "0.8.2"
fancy-regex = "0.19.2"

View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -145,7 +146,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()

View file

@ -2264,6 +2264,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2292,6 +2294,8 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
@ -2875,6 +2879,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -3008,6 +3013,7 @@ def _batch_prompt_cost(
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
@ -3022,7 +3028,9 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
)

View file

@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
is unavailable. This MUST be called at app-creation time — once the lifespan
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
middleware after an application has started".
runs, the middleware stack is frozen and middleware added later never runs, so
a late call is skipped without registering anything.
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
``ProxyTracerProvider``; the proxy publishes the real provider as the global
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
try:
if not is_otel_v2_enabled():
return
if app.middleware_stack is not None:
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
return
# Lazy: only the V2-enabled path needs the optional
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

View file

@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
import base64
import json
import logging
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
@ -13,9 +14,13 @@ from typing import Final, TypeAlias
import requests
from google.protobuf.json_format import MessageToDict
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
_logger: Final = logging.getLogger(__name__)
JSON_CONTENT_TYPE: Final = "application/json"
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
def __init__(
self,
endpoint: str | None,
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
headers: Mapping[str, str],
certificate_file: str | None = None,
session: "requests.Session | None" = None,
*,
max_request_size: int | None = None,
) -> None:
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
super().__init__(
endpoint=endpoint,
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
certificate_file=certificate_file,
session=session,
max_request_size=max_request_size,
)
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
return encode_spans_json(spans)
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
if self._shutdown:
_logger.warning("Exporter already shutdown, ignoring batch")
return SpanExportResult.FAILURE
with self._metrics.export_operation(len(spans)) as result:
try:
serialized_data: Final = encode_spans_json(spans)
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
_logger.error("Failed to encode span batch: %s", error)
result.error = error
return SpanExportResult.FAILURE
if 0 < self._max_request_size < len(serialized_data):
result.error = RequestPayloadTooLargeError(
f"Serialized span request size {len(serialized_data)} bytes "
f"exceeds max_request_size {self._max_request_size} bytes."
)
_logger.warning("Dropping span batch: %s", result.error)
return SpanExportResult.FAILURE
export_result: Final = self._client.export(serialized_data)
if not export_result.success:
result.error = export_result.error
result.error_attrs = (
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
if export_result.status_code is not None
else None
)
return SpanExportResult.FAILURE
return SpanExportResult.SUCCESS

View file

@ -76,6 +76,7 @@ def agentops_preset(
def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
import requests
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
super().__init__(endpoint=endpoint)
self._agentops_session = requests.Session()
super().__init__(endpoint=endpoint, session=self._agentops_session)
self._agentops_api_key = api_key
self._auth_resolved = False
@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
try:
token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
if token:
# ``_session`` is the requests.Session the base exporter
# POSTs through; updating its Authorization header is how the
# minted JWT reaches every subsequent export.
self._session.headers["Authorization"] = f"Bearer {token}"
self._agentops_session.headers["Authorization"] = f"Bearer {token}"
except Exception as e:
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)

View file

@ -102,6 +102,7 @@ class ExceptionCheckers:
"`inputs` tokens + `max_new_tokens` must be",
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:

View file

@ -19,6 +19,7 @@ import httpx
from pydantic import ConfigDict
from litellm import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.types.llms.base import LiteLLMBaseModel
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
@ -129,7 +130,7 @@ class GetBlogPosts:
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
Fetches from ``url`` otherwise, falling back to local backup on failure.
"""
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
return cls.load_local_blog_posts()
now: Final = time.time()

View file

@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info(
if dynamic_api_key is None and api_key is not None:
dynamic_api_key = api_key
return model, custom_llm_provider, dynamic_api_key, api_base
def get_provider_affinity_header(api_base: str | None) -> str | None:
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)

View file

@ -39,6 +39,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.fallback_generalizations import (
set_fallback_generalizations,
)
from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
@ -456,7 +457,7 @@ async def refetch_model_cost_map(
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
map they already have.
"""
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
@ -740,7 +741,7 @@ def get_model_cost_map(
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True

View file

@ -161,6 +161,7 @@ from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostPerToken,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass):
"call_type": self.call_type,
"optional_params": self.optional_params,
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,

View file

@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
reservation logic so neither has to depend on the other.
"""
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, SupportsFloat
from pydantic import TypeAdapter, ValidationError
_TIER_RANGE: Final = TypeAdapter(tuple[float, float])
def _coerce_cost_per_token(value: float | str | None) -> float:
def _coerce_cost_per_token(value: object) -> float:
"""
Coerce a per-token cost into a float.
Model cost values loaded from YAML config may arrive as strings (e.g.
scientific notation like "4e-07"), which would break arithmetic.
"""
if value is None:
return 0.0
if isinstance(value, str):
try:
return float(value)
except ValueError:
return 0.0
return float(value)
if isinstance(value, SupportsFloat):
return float(value)
return 0.0
def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None:
try:
return _TIER_RANGE.validate_python(tier.get("range"))
except ValidationError:
return None
def select_tier_for_input(
tiered_pricing: list[dict],
tiered_pricing: Sequence[Mapping[str, object]],
input_tokens: int,
) -> dict | None:
) -> Mapping[str, object] | None:
"""
Select the pricing tier for a request based on its total input token count.
@ -44,19 +56,21 @@ def select_tier_for_input(
if not tiered_pricing or input_tokens <= 0:
return None
sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
if not valid_tiers:
ranged_tiers: Final = sorted(
((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None),
key=lambda ranged: ranged[0][0],
)
if not ranged_tiers:
return None
matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end)
if matching:
return matching[0]
return valid_tiers[-1]
return ranged_tiers[-1][1]
def tier_rate(
tier: dict,
tier: Mapping[str, object],
cost_key: str,
fallback_cost_key: str | None = None,
) -> float:
@ -68,4 +82,6 @@ def tier_rate(
primary: Final = tier.get(cost_key)
if primary is not None:
return _coerce_cost_per_token(primary)
if fallback_cost_key is None:
return 0.0
return _coerce_cost_per_token(tier.get(fallback_cost_key, 0))

View file

@ -3,7 +3,7 @@
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone, tzinfo
from types import MappingProxyType
from typing import Final, Literal, TypedDict, cast
@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost"
_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
{
"5m": _BASE_CACHE_WRITE_RATE_KEY,
"30m": _BASE_CACHE_WRITE_RATE_KEY,
"1hr": "cache_creation_input_token_cost_above_1hr",
"12hr": "cache_creation_input_token_cost_above_12hr",
}
)
_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
{ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY}
)
_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
"input_cost_per_token",
"output_cost_per_token",
@ -356,7 +372,21 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
)
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None:
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
output_rate: float
cache_read_rate: float
cache_creation_rate: float
reasoning_rate: float | None
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None:
tiered_pricing: Final = model_info.get("tiered_pricing")
if not isinstance(tiered_pricing, list) or not tiered_pricing:
return None
@ -376,7 +406,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N
return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None:
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None:
"""
Resolve the base rates from a model's ``tiered_pricing`` table, if it has one.
@ -391,22 +421,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float,
if tier is None:
return None
cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
completion_cost: Final = (
tier_rate(tier, "output_cost_per_token")
if "output_cost_per_token" in tier
else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0
)
return (
tier_rate(tier, "input_cost_per_token"),
completion_cost,
cache_creation_cost,
tier_rate(tier, "cache_creation_input_token_cost_above_1hr")
if "cache_creation_input_token_cost_above_1hr" in tier
and tier["cache_creation_input_token_cost_above_1hr"] is not None
else cache_creation_cost,
tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
rates: Final = TokenRates(
input_rate=tier_rate(tier, "input_cost_per_token"),
output_rate=completion_cost,
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate=cache_creation_rate,
reasoning_rate=None,
)
return rates, tier
def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool:
@ -532,19 +560,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
return False
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
output_rate: float
cache_read_rate: float
cache_creation_rate: float
reasoning_rate: float | None
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def resolve_explicit_cache_read_cost_rate(
model_info: ModelInfo,
usage: Usage,
@ -649,7 +664,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
if off_peak is None:
return rates
off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
return TokenRates(
return replace(
rates,
input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
@ -661,32 +677,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
def _apply_off_peak_to_base_costs(
model_info: ModelInfo,
current_time: datetime | None,
base_costs: tuple[float, float, float, float | None, float],
) -> tuple[float, float, float, float, float]:
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
present one passes through untouched and an absent one resolves to the applied
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
base_rates: TokenRates,
tier: Mapping[str, object],
) -> TokenRates:
"""Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no
field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier
that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to
_resolve_billed_reasoning_rate.
"""
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=prompt,
output_rate=completion,
cache_read_rate=cache_read,
cache_creation_rate=cache_creation,
reasoning_rate=None,
rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates)
return replace(
rates,
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key)
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
}
),
)
return (
rates.input_rate,
rates.output_rate,
rates.cache_creation_rate,
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
rates.cache_read_rate,
)
def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str:
return preferred_key if model_info.get(preferred_key) is not None else fallback_key
def _get_token_base_cost(
@ -696,7 +708,7 @@ def _get_token_base_cost(
current_time: datetime | None = None,
*,
threshold_is_inclusive: bool = False,
) -> tuple[float, float, float, float, float]:
) -> TokenRates:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
@ -709,15 +721,12 @@ def _get_token_base_cost(
An absent cache-creation or cache-read rate always resolves to the resolved input
rate, the way the tiered table and custom deployment pricing already do, since a
provider that publishes no cache price bills cached tokens as ordinary input. An
absent 1h write rate resolves to the cache-creation rate, off-peak included. An
absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An
explicit 0.0 stays a real price for all of them.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage)
if tiered_base_costs is not None:
return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs)
return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs)
# Get service tier aware cost keys
input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
@ -734,13 +743,13 @@ def _get_token_base_cost(
# For image generation models that don't have output_cost_per_token,
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
if completion_base_cost == 0.0 or completion_base_cost is None:
if completion_base_cost == 0.0:
output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None)
if output_image_cost is not None:
completion_base_cost = output_image_cost
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
cache_creation_cost_above_1hr = get_cost_per_unit(
model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType(
{ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()}
)
cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
@ -811,14 +820,6 @@ def _get_token_base_cost(
if service_tier
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
)
cache_creation_1hr_tiered_key = (
get_service_tier_cost_key(
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
service_tier,
)
if service_tier
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
)
cache_read_tiered_key = (
get_service_tier_cost_key(
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
@ -830,10 +831,19 @@ def _get_token_base_cost(
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
cache_creation_cost_above_1hr = get_cost_per_unit(
model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
cache_creation_keys_by_ttl = MappingProxyType(
{
ttl: _first_present_key(
model_info,
get_service_tier_cost_key(
f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens",
service_tier,
),
key,
)
for ttl, key in cache_creation_keys_by_ttl.items()
}
)
cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
except (IndexError, ValueError):
continue
@ -849,16 +859,26 @@ def _get_token_base_cost(
resolved_cache_creation_cost: Final = (
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
)
return _apply_off_peak_to_base_costs(
base_rates: Final = apply_off_peak_pricing(
model_info,
current_time,
(
prompt_base_cost,
completion_base_cost,
resolved_cache_creation_cost,
cache_creation_cost_above_1hr,
resolved_cache_read_cost,
TokenRates(
input_rate=prompt_base_cost,
output_rate=completion_base_cost,
cache_read_rate=resolved_cache_read_cost,
cache_creation_rate=resolved_cache_creation_cost,
reasoning_rate=None,
),
)
return replace(
base_rates,
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: base_rates.cache_creation_rate
if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None
else rate
for ttl, key in cache_creation_keys_by_ttl.items()
}
),
)
@ -975,25 +995,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M
def calculate_cache_writing_cost(
cache_creation_tokens: int,
cache_creation_token_details: CacheCreationTokenDetails | None,
cache_creation_cost_above_1hr: float,
cache_creation_cost: float,
cache_creation_cost_by_ttl: Mapping[str, float],
) -> float:
"""
Adjust cost of cache creation tokens based on the cache creation token details.
Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts.
Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate.
"""
total_cost: float = 0.0
if cache_creation_token_details is not None:
# get the number of 5m and 1h cache creation tokens
cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens
cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0
total_cost += (
cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0
)
else:
total_cost += cache_creation_tokens * cache_creation_cost
return total_cost
if cache_creation_token_details is None:
return cache_creation_tokens * cache_creation_cost
tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType(
{
"5m": cache_creation_token_details.ephemeral_5m_input_tokens,
"30m": cache_creation_token_details.ephemeral_30m_input_tokens,
"1hr": cache_creation_token_details.ephemeral_1h_input_tokens,
"12hr": cache_creation_token_details.ephemeral_12h_input_tokens,
}
)
unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0)
return unattributed_tokens * cache_creation_cost + sum(
(tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost)
for ttl, tokens in tokens_by_ttl.items()
)
class PromptTokensDetailsResult(TypedDict):
@ -1178,7 +1201,7 @@ def _calculate_input_cost(
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_above_1hr: float,
cache_creation_cost_by_ttl: Mapping[str, float],
service_tier: str | None = None,
) -> float:
"""
@ -1248,8 +1271,8 @@ def _calculate_input_cost(
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
)
### CHARACTER COST
@ -1510,34 +1533,31 @@ def generic_cost_per_token(
)
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=resolved_model_info,
usage=usage,
service_tier=service_tier,
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
completion_base_cost: Final = base_costs.output_rate
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
model_info=resolved_model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
)
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=resolved_model_info,
prompt_base_cost=prompt_base_cost,
prompt_base_cost=base_costs.input_rate,
cache_read_cost=effective_cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=base_costs.cache_creation_rate,
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
service_tier=service_tier,
)
@ -1640,7 +1660,7 @@ class BilledTokenRates:
cache_read_input_token_cost: float
cache_read_input_audio_token_cost: float
cache_creation_input_token_cost: float
cache_creation_input_token_cost_above_1hr: float
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
output_cost_per_reasoning_token: float
def scaled(self, multiplier: float) -> "BilledTokenRates":
@ -1652,7 +1672,9 @@ class BilledTokenRates:
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
cache_creation_input_token_cost_by_ttl=MappingProxyType(
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
),
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
)
@ -1702,7 +1724,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
cache_read_input_token_cost=cache_read_rate,
cache_read_input_audio_token_cost=cache_read_rate,
cache_creation_input_token_cost=cache_creation_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
cache_creation_input_token_cost_by_ttl=MappingProxyType({}),
output_cost_per_reasoning_token=output_rate,
)
@ -1717,13 +1739,7 @@ def _cost_map_billed_rates(
current_time: datetime | None,
) -> BilledTokenRates:
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost_rate,
cache_creation_cost_above_1hr_rate,
cache_read_cost_rate,
) = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=model_info,
usage=usage,
service_tier=service_tier,
@ -1737,14 +1753,14 @@ def _cost_map_billed_rates(
service_tier=service_tier,
)
effective_cache_read_cost_rate: Final = (
cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
)
reasoning_rate: Final = _resolve_billed_reasoning_rate(
model_info=model_info,
usage=usage,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
completion_base_cost=base_costs.output_rate,
current_time=billing_time,
)
audio_cache_read_rate: Final = get_cost_per_unit(
@ -1759,14 +1775,14 @@ def _cost_map_billed_rates(
* get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
)
return BilledTokenRates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
input_cost_per_token=base_costs.input_rate,
output_cost_per_token=base_costs.output_rate,
cache_read_input_token_cost=effective_cache_read_cost_rate,
cache_read_input_audio_token_cost=(
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
),
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
cache_creation_input_token_cost=base_costs.cache_creation_rate,
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
output_cost_per_reasoning_token=reasoning_rate,
).scaled(multiplier)
@ -1854,8 +1870,8 @@ def get_token_type_cost_breakdown(
else calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
cache_creation_cost=rates.cache_creation_input_token_cost,
cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl,
)
)
return TokenTypeCostBreakdown(
@ -1886,7 +1902,7 @@ def calculate_prompt_caching_savings(
biller saw them rather than at the later spend write.
"""
model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider)
prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=model_info_with_cache_read_default,
usage=usage,
service_tier=service_tier,
@ -1899,9 +1915,11 @@ def calculate_prompt_caching_savings(
current_time=billed_at,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost
write_rate: Final = cache_creation_cost or prompt_base_cost
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
prompt_base_cost: Final = base_costs.input_rate
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost
)
write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
@ -1909,7 +1927,9 @@ def calculate_prompt_caching_savings(
cache_creation_details: Final = (
CacheCreationTokenDetails(
ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0),
ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0),
ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0),
ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0),
)
if details is not None
else None
@ -1918,8 +1938,10 @@ def calculate_prompt_caching_savings(
write_premium: Final = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_details,
cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost,
cache_creation_cost=write_rate - prompt_base_cost,
cache_creation_cost_by_ttl=MappingProxyType(
{ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
),
)
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
model_info, vertex_location

View file

@ -5,6 +5,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total"
_TEXT_INPUT_RATE: Final = "input_cost_per_token"
_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token"
_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost"
_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost"
_TEXT_OUTPUT_RATE: Final = "output_cost_per_token"
_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token"
_RATE_KEY_MARKERS: Final = ("cost", "pricing")
@ -33,19 +36,38 @@ def _tokens(value: object) -> int:
def used_pricing_keys(usage: Usage) -> tuple[str, ...]:
prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details)
prompt_details: Final = parse_prompt_tokens_details(usage)
prompt_audio: Final = prompt_details["audio_tokens"]
completion_audio: Final = _audio_tokens(usage.completion_tokens_details)
prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio
prompt_text: Final = max(
_tokens(usage.prompt_tokens)
- prompt_details["cache_hit_tokens"]
- prompt_audio
- prompt_details["image_tokens"]
- prompt_details["video_tokens"],
0,
)
cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0)
completion_text: Final = _tokens(usage.completion_tokens) - completion_audio
components: Final = (
(_TEXT_INPUT_RATE, prompt_text),
(_AUDIO_INPUT_RATE, prompt_audio),
(_CACHE_READ_INPUT_RATE, cached_text),
(_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]),
(_TEXT_OUTPUT_RATE, completion_text),
(_AUDIO_OUTPUT_RATE, completion_audio),
)
return tuple(key for key, count in components if count > 0)
def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str:
if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None:
return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE
if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None:
return _TEXT_INPUT_RATE
return key
def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None:
try:
return _NESTED_PRICING.validate_python(value)
@ -89,7 +111,8 @@ def diagnose_zero_cost(
pricing_entry: Mapping[str, object],
calculation_failed: bool,
) -> StandardLoggingZeroCostDiagnostic | None:
used_keys: Final = used_pricing_keys(usage)
effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage))
used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index])
if not used_keys:
return None
missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None)

View file

@ -0,0 +1,7 @@
import os
def is_offline_mode() -> bool:
"""True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather
than from a destination the operator configured, must check this and use its bundled copy or skip instead"""
return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true"

View file

@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None:
return getattr(value, key, None)
def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None:
def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None:
header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None
if header is None:
return None
return default_header
if not isinstance(header, str):
raise TypeError("provider_affinity_header must be a string")
return validate_provider_affinity_header_name(header)
@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
def add_provider_affinity_header(
headers: Mapping[str, object], litellm_params: object | None
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
header_name: Final = _get_provider_affinity_header_name(litellm_params)
header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header)
if header_name is None or any(key.lower() == header_name.lower() for key in headers):
return dict(headers)

View file

@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
_messages_carry_output_config = messages_carry_output_config
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
if "stop_sequence" not in response:
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
return response
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> AnthropicMessagesResponse:
"""
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
"""
try:
raw_response_json: Final = raw_response.json()
except Exception:
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
return AnthropicMessagesResponse(**raw_response_json)
response: Final = AnthropicMessagesResponse(**raw_response_json)
return _ensure_stop_sequence(response)
def get_async_streaming_response_iterator(
self,

View file

@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
else:
mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
# Extract engine from model name (e.g., "aws_polly/neural" -> "neural")
# model arrives without the "aws_polly/" prefix, e.g. "standard"
engine: Final = self._extract_engine_from_model(model)
mapped_params["engine"] = engine
@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
"""
Extract engine from model name.
speech() strips the "aws_polly/" prefix before the config is called,
so the bare engine name is what normally arrives here.
Examples:
- aws_polly/neural -> neural
- aws_polly/standard -> standard
- standard -> standard
- aws_polly/long-form -> long-form
- aws_polly -> neural (default)
"""
if "/" in model:
parts: Final = model.split("/")
if len(parts) >= 2:
engine: Final = parts[1].lower()
if engine in self.VALID_ENGINES:
return engine
engine: Final = model.rsplit("/", 1)[-1].lower()
if engine in self.VALID_ENGINES:
return engine
return self.DEFAULT_ENGINE
def validate_environment(

View file

@ -3,6 +3,7 @@ from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from urllib.parse import urlsplit
import httpx
from openai.types.responses import ResponseReasoningItem
@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params)
parsed_base: Final = urlsplit(resolved_api_base or "")
path: Final = parsed_base.path.rstrip("/")
if path.endswith(("/openai/v1", "/openai/v1/responses")):
responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path
return BaseAzureLLM.get_base_azure_url(
api_base=parsed_base._replace(path=responses_path).geturl(),
litellm_params=responses_params,
route="/openai/v1/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
return BaseAzureLLM.get_base_azure_url(
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
litellm_params=responses_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

View file

@ -12,6 +12,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
audio_or_image_in_message_content,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig):
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(messages)
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:

View file

@ -27,6 +27,7 @@ from typing_extensions import Self
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
@ -67,12 +68,14 @@ class KimiK3TokenizerSource:
def _hugging_face_ranks_path() -> str:
import huggingface_hub
return huggingface_hub.hf_hub_download(
repo_id=KIMI_K3_REPO,
filename=KIMI_K3_RANKS_FILE,
revision=KIMI_K3_REVISION,
local_files_only=litellm.disable_hf_tokenizer_download is True,
)
if litellm.disable_hf_tokenizer_download is True or is_offline_mode():
cached: Final = huggingface_hub.try_to_load_from_cache(
repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION
)
if not isinstance(cached, str):
raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache")
return cached
return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION)
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)

View file

@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.realtime_errors import (
close_after_upstream_handshake_refusal,
realtime_error_event,
@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import (
)
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -2435,13 +2438,13 @@ class BaseLLMHTTPHandler:
# Also check for extra_headers in kwargs (from config or direct calls)
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
# Merge all header sources: forwarded < extra_headers < provider_specific
merged_headers: Final = {}
if forwarded_headers:
merged_headers.update(forwarded_headers)
if extra_headers_from_kwargs:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
{
**(forwarded_headers or {}),
**(extra_headers_from_kwargs or {}),
**(provider_specific_headers or {}),
}
)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
@ -2452,11 +2455,19 @@ class BaseLLMHTTPHandler:
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
affinity_api_base: Final = (
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
(
headers,
api_base,
) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment(
headers=merged_headers or {},
headers=(
add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header)
if default_affinity_header is not None
else dict(merged_headers)
),
model=model,
messages=messages,
optional_params=request_params,

View file

@ -7,6 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
"""
from collections.abc import Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Final
@ -72,7 +73,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates:
)
def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates:
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
# spelling out only input rates would serve every completion for free, so there the model's
# own output rates stand in

View file

@ -0,0 +1,10 @@
import re
def is_opencode_api_base(api_base: str | None) -> bool:
return bool(
api_base
and re.fullmatch(
r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base
)
)

View file

@ -0,0 +1,6 @@
from litellm.llms.opencode.common_utils import is_opencode_api_base
class OpenCodeProviderAdapter:
def get_provider_affinity_header(self, api_base: str | None) -> str | None:
return "x-opencode-session" if is_opencode_api_base(api_base) else None

View file

@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_httpx_client,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
from litellm.types.llms.openai import (
AllMessageValues,
@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif modality == "DOCUMENT":
cached_text_tokens = (cached_text_tokens or 0) + token_count
## Calculate non-cached tokens by subtracting cached from total (per modality)
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
## See: https://github.com/BerriAI/litellm/issues/18750
if cached_text_tokens is not None and prompt_text_tokens is not None:
# Explicit caching: subtract cached tokens per modality from cacheTokensDetails
prompt_text_tokens = prompt_text_tokens - cached_text_tokens
elif (
if (
cached_tokens is not None
and prompt_text_tokens is not None
and cached_text_tokens is None
and "cacheTokensDetails" not in usage_metadata
):
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
# Subtract from text tokens since implicit caching is primarily for text content
# See: https://github.com/BerriAI/litellm/issues/16341
prompt_text_tokens = prompt_text_tokens - cached_tokens
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
if cached_image_tokens is not None and prompt_image_tokens is not None:
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
if cached_video_tokens is not None and prompt_video_tokens is not None:
prompt_video_tokens = prompt_video_tokens - cached_video_tokens
@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
billable_text_tokens: Final = (
prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens
)
cached_tokens_details_fields: Final = MappingProxyType(
{
key: token_count
for key, token_count in (
("text_tokens", cached_text_tokens),
("audio_tokens", cached_audio_tokens),
("image_tokens", cached_image_tokens),
)
if token_count is not None
}
)
cached_tokens_details: Final = (
CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
cached_tokens_details=cached_tokens_details,
audio_tokens=prompt_audio_tokens,
text_tokens=billable_text_tokens,
image_tokens=prompt_image_tokens,

View file

@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import (
with_control_options,
)
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_provider_affinity_header,
invalid_provider_routing_hints_message,
)
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
VertexAIModelRoute,
@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import (
resolve_chat_reasoning,
)
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.completion import (
CompletionDispatchContext,
_CompletionDispatchResult,
@ -169,7 +172,6 @@ from litellm.utils import (
ProviderConfigManager,
TextChoices,
TextCompletionResponse,
TextCompletionStreamWrapper,
TranscriptionResponse,
Usage,
add_provider_specific_params_to_optional_params,
@ -5990,19 +5992,28 @@ def completion(
},
)
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
if litellm_params.get("provider_affinity_header") is not None:
affinity_api_base: Final = (
str(client.base_url)
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
else get_openai_credentials(api_base=api_base).api_base
if custom_llm_provider == "openai"
else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None:
try:
headers = add_provider_affinity_header(
headers=headers or litellm.headers or MappingProxyType({}),
litellm_params=MappingProxyType(
{
"provider_affinity_header": litellm_params["provider_affinity_header"],
"provider_affinity_header": litellm_params.get("provider_affinity_header"),
"litellm_session_id": kwargs.get("litellm_session_id"),
"session_id": kwargs.get("session_id"),
"metadata": metadata,
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
except ValueError as affinity_error:
raise litellm.BadRequestError(

View file

@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target:
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**detectors.COMMON,
Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity,
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
Feature.RESPONSE_CACHE: detectors.response_cache,
Feature.MODIFY_PARAMS: detectors.modify_params,

View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -31041,11 +31137,15 @@
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31370,11 +31470,15 @@
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
"cache_read_input_audio_token_cost_batches": 3e-08,
"cache_read_input_audio_token_cost_flex": 3e-08,
"cache_read_input_audio_token_cost_priority": 5.4e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 1e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_priority": 1.8e-08,
"input_cost_per_audio_token": 3e-07,
"input_cost_per_audio_token_flex": 1.5e-07,
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31664,12 +31768,16 @@
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_audio_token_cost_batches": 2.5e-08,
"cache_read_input_audio_token_cost_flex": 2.5e-08,
"cache_read_input_audio_token_cost_priority": 9e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"deprecation_date": "2027-05-07",
"input_cost_per_audio_token": 5e-07,
"input_cost_per_audio_token_flex": 2.5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
@ -31790,10 +31898,14 @@
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 5e-08,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -36699,6 +36811,12 @@
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
@ -36784,6 +36902,12 @@
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -36861,17 +36985,23 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-08,
"cache_creation_input_token_cost_priority": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_decisions": 1e-07,
"cache_read_input_token_cost_decisions": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
"cache_read_input_token_cost_batches": 5e-09,
@ -36947,6 +37077,12 @@
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37021,6 +37157,12 @@
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37097,6 +37239,12 @@
},
"gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -37172,6 +37320,12 @@
},
"gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_12hr": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
@ -42337,6 +42491,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -45623,6 +45905,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45648,6 +45931,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -48499,6 +48783,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -49352,6 +49647,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49371,6 +49667,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -52906,12 +53203,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53225,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -59917,6 +60218,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -59942,6 +60244,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -59967,6 +60270,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -59990,6 +60294,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -67841,6 +68146,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -67860,6 +68166,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68138,6 +68445,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68157,6 +68465,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68545,6 +68854,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68564,6 +68874,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68704,6 +69015,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68723,6 +69035,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -73330,6 +73643,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
@ -74677,6 +74991,12 @@
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,

View file

@ -1368,7 +1368,7 @@ class MCPRequestHandler:
if auth is None:
return None
refreshed = auth.model_copy()
refreshed: Final = auth.model_copy(update={"requires_fresh_policy": True})
if (auth.mcp_admitted_user_subject or auth.is_session_token) and auth.user_id:
# A session credential (``lite login`` or a redeemed ID-JAG) has no key row, its authority is
# sealed in the token: refresh the live user policy and keep the token's team, routes and models.
@ -1386,7 +1386,12 @@ class MCPRequestHandler:
refreshed.project_id = current.project_id
refreshed.user_id = current.user_id
refreshed.access_group_ids = current.access_group_ids
refreshed.requires_fresh_policy = True
else:
return refreshed
if refreshed.mcp_toolset_id is not None:
from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope
return await apply_toolset_scope(refreshed, refreshed.mcp_toolset_id)
return refreshed
@staticmethod

View file

@ -1797,7 +1797,7 @@ if MCP_AVAILABLE:
return acting.model_copy(update={"mcp_toolset_id": toolset_id})
tool_permissions = await operations.global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=[toolset_id]
toolset_ids=[toolset_id], requires_fresh_policy=acting.requires_fresh_policy
)
server_ids: Final = list(tool_permissions.keys())
existing_op: Final = acting.object_permission

Binary file not shown.

After

Width:  |  Height:  |  Size: 685 KiB

View file

@ -0,0 +1,27 @@
<!--
Source: https://github.com/lobehub/lobe-icons/blob/master/packages/static-svg/icons/claude-color.svg
Retrieved: 2026-10-22
MIT License
Copyright (c) 2023 LobeHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-->
<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z" fill="#D97757" fill-rule="nonzero"></path></svg>

View file

@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
path_prefixes=("/vantage",),
),
LazyFeature(
name="usage_ai",
module_path="litellm.proxy.management_endpoints.usage_endpoints",
path_prefixes=("/usage/ai",),
),
LazyFeature(
name="prompts",
module_path="litellm.proxy.prompts.prompt_endpoints",

View file

@ -11407,7 +11407,7 @@
},
"/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_systemone_post",
"requestBody": {
"content": {
@ -11813,7 +11813,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -12814,7 +12813,7 @@
},
"/v1/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_v1_systemone_post",
"requestBody": {
"content": {
@ -13220,7 +13219,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -22860,6 +22858,25 @@
"title": "Body_image_edit_api_openai_deployments__model__images_edits_post",
"type": "object"
},
"CacheWriteTokensDetails": {
"additionalProperties": true,
"properties": {
"ttl_12h_tokens": {
"title": "Ttl 12H Tokens",
"type": "integer"
},
"ttl_30m_tokens": {
"title": "Ttl 30M Tokens",
"type": "integer"
}
},
"required": [
"ttl_30m_tokens",
"ttl_12h_tokens"
],
"title": "CacheWriteTokensDetails",
"type": "object"
},
"CachedTokensDetails": {
"properties": {
"audio_tokens": {
@ -24354,6 +24371,27 @@
],
"title": "Audio Tokens"
},
"cache_write_tokens": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Cache Write Tokens"
},
"cache_write_tokens_details": {
"anyOf": [
{
"$ref": "#/components/schemas/CacheWriteTokensDetails"
},
{
"type": "null"
}
]
},
"cached_tokens": {
"default": 0,
"title": "Cached Tokens",
@ -57900,163 +57938,6 @@
}
}
},
"usage_ai": {
"components": {
"schemas": {
"ChatMessage": {
"properties": {
"content": {
"title": "Content",
"type": "string"
},
"role": {
"enum": [
"user",
"assistant"
],
"title": "Role",
"type": "string"
}
},
"required": [
"role",
"content"
],
"title": "ChatMessage",
"type": "object"
},
"HTTPValidationError": {
"properties": {
"detail": {
"items": {
"$ref": "#/components/schemas/ValidationError"
},
"title": "Detail",
"type": "array"
}
},
"title": "HTTPValidationError",
"type": "object"
},
"UsageAIChatRequest": {
"properties": {
"messages": {
"description": "Chat messages (user/assistant history)",
"items": {
"$ref": "#/components/schemas/ChatMessage"
},
"title": "Messages",
"type": "array"
},
"model": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "Model to use for AI chat",
"title": "Model"
}
},
"required": [
"messages"
],
"title": "UsageAIChatRequest",
"type": "object"
},
"ValidationError": {
"properties": {
"ctx": {
"title": "Context",
"type": "object"
},
"input": {
"title": "Input"
},
"loc": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
},
"title": "Location",
"type": "array"
},
"msg": {
"title": "Message",
"type": "string"
},
"type": {
"title": "Error Type",
"type": "string"
}
},
"required": [
"loc",
"msg",
"type"
],
"title": "ValidationError",
"type": "object"
}
}
},
"paths": {
"/usage/ai/chat": {
"post": {
"description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.",
"operationId": "usage_ai_chat_usage_ai_chat_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UsageAIChatRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Usage Ai Chat",
"tags": [
"usage_ai"
]
}
}
}
},
"vantage": {
"components": {
"schemas": {

View file

@ -2717,6 +2717,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"hidden model can still be called."
),
)
return_wildcard_routes: bool | None = Field(
None,
description=(
"When true, `/v1/models` lists wildcard routes like `openai/*` unless the request passes "
"`return_wildcard_routes=false`."
),
)
alerting: list | None = Field(
None,
description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL",

View file

@ -21,6 +21,7 @@ from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm._version import version
from litellm.constants import HIBP_RANGE_API_BASE
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
return general_settings.get("password_policy_check_breached_passwords", True) is not False
return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False
async def is_password_breached(

View file

@ -12,7 +12,6 @@ import json
import os
import random
import time
import traceback
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
from contextlib import asynccontextmanager
from contextvars import ContextVar
@ -90,6 +89,7 @@ from litellm.proxy.spend_tracking.savings import (
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
@ -313,6 +313,38 @@ def _spend_tables_left_to_send(
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None:
spend_log_error(
"Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s",
step_name,
payload.get("request_id"),
payload.get("api_key"),
exc=exc,
)
_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata
def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata:
"""Malformed metadata costs the request its metadata-derived fields, never its daily rollup row."""
raw: Final = payload.get("metadata")
if not isinstance(raw, str):
return _EMPTY_SPEND_LOG_METADATA
try:
parsed: Final[SpendLogsMetadata] = json.loads(raw)
except json.JSONDecodeError as e:
spend_log_error(
"Spend tracking - unreadable spend log metadata, rollups for this request are written without "
"metadata-derived fields. request_id=%s api_key=%s",
payload.get("request_id"),
payload.get("api_key"),
exc=e,
)
return _EMPTY_SPEND_LOG_METADATA
return parsed
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
@ -647,8 +679,8 @@ class DBSpendUpdateWriter:
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
)
# Single task replaces 11 create_task() calls
asyncio.create_task(
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=response_cost,
user_id=user_id,
@ -661,7 +693,8 @@ class DBSpendUpdateWriter:
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
request_model_access_groups=get_request_model_access_groups(kwargs),
)
),
task_name="spend rollups",
)
self._enqueue_tool_registry_upsert(
@ -1086,91 +1119,52 @@ class DBSpendUpdateWriter:
project_id: str | None = None,
):
"""
Runs all 13 spend-update helpers sequentially inside a single asyncio task.
Each helper is wrapped in try/except so one failure doesn't prevent the others.
The deepcopy runs here, off the awaited request path, so the daily spend
helpers get a payload isolated from the spend-log queue entry and the caller.
Runs every spend-update helper for one request. Each helper goes through
_run_spend_update_step so one failure never skips the sibling writes. The deepcopy
isolates the daily spend helpers' payload from the spend-log queue entry and the caller.
"""
payload_copy: Final = copy.deepcopy(payload)
request_tags: Final = payload_copy.get("request_tags")
try:
await self._update_user_db(
await self._run_spend_update_step(
"_update_user_db",
self._update_user_db(
response_cost=response_cost,
user_id=user_id,
prisma_client=prisma_client,
litellm_proxy_budget_name=litellm_proxy_budget_name,
end_user_id=end_user_id,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_user_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_key_db(
response_cost=response_cost,
hashed_token=hashed_token,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_key_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_team_db(
response_cost=response_cost,
team_id=team_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_team_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_org_db(
response_cost=response_cost,
org_id=org_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_org_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_project_db(
response_cost=response_cost,
project_id=project_id,
prisma_client=prisma_client,
)
except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _update_project_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_tag_db(
response_cost=response_cost,
request_tags=request_tags,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_tag_db failed: %s",
traceback.format_exc(),
)
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_key_db",
self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_team_db",
self._update_team_db(
response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_org_db",
self._update_org_db(
response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_project_db",
self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_tag_db",
self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client),
payload=payload_copy,
)
await self._update_model_access_group_db(
response_cost=response_cost,
request_model_access_groups=request_model_access_groups,
@ -1178,96 +1172,64 @@ class DBSpendUpdateWriter:
prisma_client=prisma_client,
router=get_llm_router(),
)
_agent_id_for_spend: Final = payload_copy.get("agent_id")
try:
await self._update_agent_db(
response_cost=response_cost,
agent_id=_agent_id_for_spend,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_agent_db failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_team_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s",
traceback.format_exc(),
)
await self._run_spend_update_step(
"_update_agent_db",
self._update_agent_db(
response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_user_transaction",
self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_end_user_transaction",
self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_agent_transaction",
self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_team_transaction",
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
try:
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _record_request_error failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy,
org_id=org_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_tag_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s",
traceback.format_exc(),
)
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_org_transaction",
self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy, org_id=org_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_tag_transaction",
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
@staticmethod
async def _run_spend_update_step(
step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload
) -> None:
try:
await step
except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes
_report_spend_update_step_failure(step_name, payload=payload, exc=e)
async def _update_key_db(
self,
response_cost: float | None,
@ -2810,7 +2772,7 @@ class DBSpendUpdateWriter:
date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "")
if not date:
return
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
metadata: Final = _parse_spend_log_metadata(payload)
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
request_error_accumulator.record(
payload=payload,
@ -2861,7 +2823,7 @@ class DBSpendUpdateWriter:
# spend, token and per-entity columns written here stay either way.
request_status: Final = prisma_client.get_request_status(payload)
verbose_proxy_logger.debug("Logged request status: %s", request_status)
_metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
_metadata: Final = _parse_spend_log_metadata(payload)
usage_obj: Final = _metadata.get("usage_object", {}) or {}
if isinstance(payload["startTime"], datetime):
start_time: Final = payload["startTime"].isoformat()

View file

@ -228,7 +228,7 @@ async def systemone(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,

View file

@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = (
"/v1/messages",
"responses",
"files",
"interactions",
)
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
LlmProviders.VERTEX_AI.value,
)
)
_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
)
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request(
custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers
),
ProviderSpecificHeader(
custom_llm_provider=LlmProviders.DATABRICKS.value,
custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS,
extra_headers=anthropic_api_headers,
native_anthropic_messages_only=True,
),

View file

@ -2093,7 +2093,9 @@ async def get_user_key_counts(
return result
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
order_by: Final[dict[str, str]] = {}
if sort_by is None:
@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
"spend",
"user_alias",
"user_role",
"max_budget",
"updated_at",
"sso_user_id",
"status",
"key_count",
]
if sort_by not in valid_columns:
raise HTTPException(
@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
if sort_by in {"status", "key_count"}:
return None
if sort_by in {"max_budget", "updated_at", "sso_user_id"}:
nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last"
return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"})
order_by[sort_by] = sort_order.lower()
return order_by
@ -2338,16 +2350,40 @@ async def get_users(
# Build order_by conditions
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
where=where,
skip=skip,
take=page_size,
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
computed_sort: Final = sort_by in ("status", "key_count")
sorted_ids: Final = (
await sorted_user_ids(
prisma_client,
sort_by="status" if sort_by == "status" else "key_count",
descending=sort_order.lower() == "desc",
skip=skip,
take=page_size,
role=role,
user_ids=user_ids,
user_email=user_email,
team=team,
sso_user_ids=sso_user_ids,
organization_ids=organization_ids,
search=search,
)
if computed_sort
else ()
)
page_users: Final = await UserRepository(prisma_client).table.find_many(
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"},
)
positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)}
users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users
# Get total count of user rows
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)

View file

@ -7397,20 +7397,26 @@ async def key_aliases(
)
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
order_by: Final[dict[str, str]] = {}
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
if sort_by is None:
return None
# Validate sort_by is a valid column
valid_columns: Final = [
valid_columns: Final = (
"spend",
"max_budget",
"created_at",
"updated_at",
"token",
"key_alias",
]
"expires",
"last_active",
"budget_reset_at",
"total_spend",
"tpm_limit",
"rpm_limit",
)
if sort_by not in valid_columns:
raise HTTPException(
status_code=400,
@ -7424,9 +7430,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
order_by[sort_by] = sort_order.lower()
return order_by
direction: Final = sort_order.lower()
nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit")
if sort_by not in (*nullable_sort_columns, "total_spend"):
return {sort_by: direction}
return [
{sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction},
{"token": "asc"},
]
def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]:
@ -7668,7 +7679,7 @@ async def list_key_helper(
verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size)
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)

View file

@ -3,6 +3,7 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Final
@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...])
SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields)
# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt.
_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰0𐒠𐴰𐵀𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑛐𑛚𑜰𑣠𑥐𑯰𑱐𑵐𑶠\U00011de0𑽐𖄰𖩠𖫀𖭐𖵰𜳰𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞗱𞥐🯰"
_DECIMAL_DIGITS: Final = "".join(
chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES)
)
_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES)
_BUDGET_DURATION_SECONDS: Final = f"""(
SELECT EXTRACT(EPOCH FROM (
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +
(parts[1] || ' ' || CASE parts[2]
WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours'
WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval
- (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
)) FROM regexp_match(
CASE lower(trim(budget_duration))
WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h'
WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d'
ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration
ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END,
'^([0-9]+)(mo|[smhdw])'
) AS duration(parts)
)"""
@dataclass(frozen=True, slots=True)
class PrismaBudgetListExecutor:
"""The database half of the budgets list. Every caller-supplied value is bound to a
@ -91,10 +118,16 @@ class PrismaBudgetListExecutor:
async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]:
clauses, params = where_sql(plan.where)
ordering: Final = ", ".join(
order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST")
if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")
else order_by_sql((key,))
for key in plan.order
).replace('"budget_duration"', _BUDGET_DURATION_SECONDS)
sql: Final = (
f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}"
+ (f" WHERE {clauses}" if clauses else "")
+ f" ORDER BY {order_by_sql(plan.order)}"
+ f" ORDER BY {ordering}"
+ f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip))
@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope:
return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}")
# budget_duration is deliberately absent from `sortable`: the column holds strings
# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d".
BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
{
"budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))),
@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
resource="budgets",
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
sortable=frozenset(
("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration")
),
searchable=frozenset(("budget_id",)),
filters=BUDGET_FILTERS,
default_sort=(SortKey(field="created_at", descending=True),),
@ -154,8 +187,9 @@ async def list_budgets(
way to page, sort or filter it.
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. `budget_id` is appended to every sort as the
`rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months
measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
`page_size` defaults to 50 and is capped at 100. Filters are
`filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]`

View file

@ -28,6 +28,7 @@ from litellm.constants import (
MAX_COMPETITOR_NAMES,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.openai.chat.guardrail_translation.handler import (
OpenAIChatCompletionsHandler,
)
@ -657,7 +658,7 @@ async def get_policy_templates(
Fetches from GitHub with automatic fallback to local backup on failure.
Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.
"""
use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
"true",
"1",
"yes",

View file

@ -6109,13 +6109,52 @@ async def list_team_v2(
"total_pages": 0,
}
# Build order_by conditions
valid_sort_columns: Final = ["team_id", "team_alias", "created_at"]
order_by = None
if sort_by and sort_by in valid_sort_columns:
if sort_order.lower() not in ["asc", "desc"]:
sort_order = "asc"
order_by = {sort_by: sort_order.lower()}
from litellm.proxy.management_helpers.team_list_sorting import (
COMPUTED_TEAM_SORTS,
UnsupportedTeamFilter,
sorted_team_ids,
)
valid_sort_columns: Final = (
"team_id",
"team_alias",
"created_at",
"spend",
"max_budget",
"tpm_limit",
"rpm_limit",
"updated_at",
)
sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc"
nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit")
order_by: Final = (
(
{sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}},
{"team_id": "asc"},
)
if nullable_sort
else ({sort_by: sort_direction}, {"team_id": "asc"})
if sort_by in ("spend", "updated_at")
else {sort_by: sort_direction}
if sort_by in valid_sort_columns
else None
)
computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS
sorted_ids: Final = (
await sorted_team_ids(
prisma_client,
where=where_conditions,
sort_by=sort_by or "",
descending=sort_direction == "desc",
skip=skip,
take=page_size,
)
if computed_sort
else ()
)
if isinstance(sorted_ids, UnsupportedTeamFilter):
raise HTTPException(status_code=500, detail="Unsupported team sorting filter")
# Get teams with pagination
if use_deleted_table:
@ -6130,26 +6169,32 @@ async def list_team_v2(
total_count = await _deleted_team_db(prisma_client).count(where=where_conditions)
else:
teams = await _team_db(prisma_client).find_many(
where=where_conditions,
skip=skip,
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"}, # Default sort
order=order_by if order_by else {"created_at": "desc"},
include=_INCLUDE_MODEL_TABLE,
)
# Get total count for pagination
total_count = await _team_db(prisma_client).count(where=where_conditions)
positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)}
sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams
# Calculate total pages
total_pages: Final = -(-total_count // page_size) # Ceiling division
# Aggregate virtual-key counts per team for the current page. The deleted
# table does not carry keys_count, so it is skipped.
keys_count_by_team: dict[str, int] = {}
if not use_deleted_table:
keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams)
keys_count_by_team: Final = (
await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {}
)
# Convert Prisma models to response models with members_count and keys_count
team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team)
team_list: Final = _convert_teams_to_response_models(
sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team
)
# Resolve resources inherited from access groups (single batch query)
if not use_deleted_table:

View file

@ -152,7 +152,7 @@ from litellm.proxy.utils import (
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import SSOConfigRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.user_repository import UserRepository, free_sso_user_creation
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
@ -1074,25 +1074,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token(
)
async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None:
"""Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license."""
if premium_user is True:
return
if prisma_client is None:
raise ProxyException(
message=CommonProxyErrors.db_not_connected_error.value,
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
billable_users: Final = await UserRepository(prisma_client).count_billable_users()
if billable_users and billable_users > 5:
raise ProxyException(
message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it",
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
class _SSOUserAdmissionError(ProxyException):
pass
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
@ -1112,8 +1095,6 @@ async def google_login(
from litellm.proxy.proxy_server import (
cli_sso_session_cache,
general_settings,
premium_user,
prisma_client,
saml_sso_cache,
user_custom_ui_sso_sign_in_handler,
)
@ -1128,15 +1109,6 @@ async def google_login(
if admin_ui_is_disabled and not is_cli_sso_login:
return admin_ui_disabled()
####### Check if user is a Enterprise / Premium User #######
if (
microsoft_client_id is not None
or google_client_id is not None
or generic_client_id is not None
or SAMLAuthHandler.is_saml_configured()
):
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
####### Detect DB + MASTER KEY in .env #######
missing_env_vars: Final = show_missing_vars_in_env()
if missing_env_vars is not None:
@ -1963,6 +1935,8 @@ async def get_user_info_from_db(
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e)
@ -2328,7 +2302,6 @@ async def saml_callback(request: Request):
general_settings,
jwt_handler,
master_key,
premium_user,
prisma_client,
saml_sso_cache,
)
@ -2352,8 +2325,6 @@ async def saml_callback(request: Request):
user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data)
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
ui_access_mode: Final = general_settings.get("ui_access_mode", None)
relay_state: Final = post_data.get("RelayState")
cp_return_to: Final[str | None] = (
@ -2821,12 +2792,20 @@ async def insert_sso_user(
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
response: Final = await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
from litellm.proxy.proxy_server import premium_user
return response
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
try:
return await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
except ProxyException as error:
if premium_user is True:
raise
raise _SSOUserAdmissionError(
message=error.message, type=error.type, param=error.param, code=error.code
) from error
DefaultTeamJoin: TypeAlias = Callable[
@ -3567,6 +3546,8 @@ class SSOAuthenticationHandler:
prisma_client=prisma_client,
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if (
user_info is None

View file

@ -1,9 +0,0 @@
"""
Usage endpoints package.
Re-exports the router from endpoints module.
"""
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401
router,
)

View file

@ -1,627 +0,0 @@
"""
AI Usage Chat - uses LLM tool calling to answer questions about
usage/spend data by querying the aggregated daily activity endpoints.
"""
import json
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
from datetime import date
from typing import Final, Literal, NamedTuple, Protocol, cast, overload
from fastapi import HTTPException
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.proxy._types import CommonProxyErrors
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
from litellm.types.utils import ChatCompletionMessageToolCall
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
USAGE_AI_TEMPERATURE: Final = 0.2
TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend"
TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend"
TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend"
ENTITY_FIELD_USER: Final = "user_id"
ENTITY_FIELD_TEAM: Final = "team_id"
ENTITY_FIELD_TAG: Final = "tag"
PAGINATED_PAGE_SIZE: Final = 200
MAX_CHAT_MESSAGES: Final = 20
TOP_N_MODELS: Final = 15
TOP_N_PROVIDERS: Final = 10
TOP_N_KEYS: Final = 10
# ---------------------------------------------------------------------------
# Types
# ---------------------------------------------------------------------------
class SSEStatusEvent(TypedDict):
type: Literal["status"]
message: str
class SSEToolCallEvent(TypedDict, total=False):
type: Literal["tool_call"]
tool_name: str
tool_label: str
arguments: dict[str, str]
status: Literal["running", "complete", "error"]
error: str
class SSEChunkEvent(TypedDict):
type: Literal["chunk"]
content: str
class SSEDoneEvent(TypedDict):
type: Literal["done"]
class SSEErrorEvent(TypedDict):
type: Literal["error"]
message: str
SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent
class _EntityEntry(TypedDict, total=False):
metrics: ReadOnly[Mapping[str, float]]
metadata: ReadOnly[Mapping[str, str]]
class _DayDump(TypedDict, total=False):
breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]]
class _EntityTotal(NamedTuple):
"""Running per-entity totals accumulated while summarising a usage dump."""
alias: str
spend: float
requests: float
tokens: float
class _UsageDump(Protocol):
@overload
def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ...
@overload
def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ...
class _ToolFunctionDef(TypedDict):
name: ReadOnly[str]
description: ReadOnly[str]
parameters: ReadOnly[Mapping[str, object]]
class _ToolDef(TypedDict):
type: ReadOnly[str]
function: ReadOnly[_ToolFunctionDef]
class ToolHandler(TypedDict):
fetch: Callable[..., Awaitable[_UsageDump]]
summarise: Callable[[_UsageDump], str]
label: str
# ---------------------------------------------------------------------------
# Tool definitions (OpenAI function-calling schema)
# ---------------------------------------------------------------------------
_DATE_PARAMS: Final = {
"start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"},
"end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"},
}
_TOOL_USAGE: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_usage_data",
"description": (
"Fetch aggregated global usage/spend data. Returns daily spend, "
"token counts, request counts, and breakdowns by model, provider, "
"and API key. Use for overall spend, top models, top providers."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"user_id": {
"type": "string",
"description": "Optional user ID filter. Omit for global view.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TEAM: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_team_usage_data",
"description": (
"Fetch usage/spend data broken down by team. Use for questions "
"like 'which team spends the most' or 'show me team X usage'."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"team_ids": {
"type": "string",
"description": "Optional comma-separated team IDs. Omit for all teams.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TAG: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_tag_usage_data",
"description": (
"Fetch usage/spend data broken down by tag. Tags are labels "
"attached to requests (features, environments, credentials)."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"tags": {
"type": "string",
"description": "Optional comma-separated tag names. Omit for all tags.",
},
},
"required": ["start_date", "end_date"],
},
},
}
TOOLS_BASE: Final = [_TOOL_USAGE]
TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG]
def get_tools_for_role(is_admin: bool) -> list[_ToolDef]:
"""Return the tool list appropriate for the user's role."""
return TOOLS_ADMIN if is_admin else TOOLS_BASE
_SYSTEM_PROMPT_BASE: Final = (
"You are an AI assistant embedded in the LiteLLM Usage dashboard. "
"You help users understand their LLM API spend and usage data.\n\n"
"ALWAYS call the appropriate tool(s) first to fetch data before answering. "
"You may call multiple tools if the question spans different dimensions.\n\n"
"Guidelines:\n"
"- Be concise and specific. Use exact numbers from the data.\n"
"- Format costs as dollar amounts (e.g. $12.34).\n"
"- When comparing entities, show a ranked list.\n"
"- If data is empty or no results found, say so clearly.\n"
"- Do not hallucinate data — only use what the tools return.\n"
"- Today's date will be provided below. Use it to interpret relative dates "
"like 'this week', 'this month', 'last 7 days', etc."
)
_TOOL_DESCRIPTIONS_ADMIN: Final = (
"You have access to these tools:\n"
"- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n"
"- `get_team_usage_data`: Team-level usage breakdown\n"
"- `get_tag_usage_data`: Tag-level usage breakdown\n\n"
)
_TOOL_DESCRIPTIONS_BASE: Final = (
"You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n"
)
def _build_system_prompt(is_admin: bool) -> str:
"""Build role-appropriate system prompt with today's date."""
tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE
return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}"
# keep a public reference for test assertions
SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE
# ---------------------------------------------------------------------------
# Data fetchers
# ---------------------------------------------------------------------------
def _parse_csv_ids(raw: str | None) -> list[str] | None:
if not raw:
return None
return [t.strip() for t in raw.split(",") if t.strip()]
async def _query_activity(
table_name: str,
entity_id_field: str,
entity_id: str | list[str] | None,
start_date: str,
end_date: str,
*,
use_aggregated: bool = False,
) -> SpendAnalyticsPaginatedResponse:
"""Shared helper that calls the daily activity query layer."""
from litellm.proxy.management_endpoints.common_daily_activity import (
daily_activity_repository,
daily_activity_scope,
get_daily_activity,
get_daily_activity_aggregated,
)
from litellm.proxy.proxy_server import prisma_client
if use_aggregated:
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
repository: Final = daily_activity_repository(prisma_client)
scope: Final = daily_activity_scope(
table_name,
entity_id_field,
entity_id,
None,
None,
start_date,
end_date,
None,
None,
)
return await get_daily_activity_aggregated(
repository,
scope,
)
return await get_daily_activity(
prisma_client=prisma_client,
table_name=table_name,
entity_id_field=entity_id_field,
entity_id=entity_id,
entity_metadata_field=None,
start_date=start_date,
end_date=end_date,
model=None,
api_key=None,
page=1,
page_size=PAGINATED_PAGE_SIZE,
)
async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_USER_SPEND,
ENTITY_FIELD_USER,
user_id,
start_date,
end_date,
use_aggregated=True,
)
return resp.model_dump(mode="json")
async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TEAM_SPEND,
ENTITY_FIELD_TEAM,
_parse_csv_ids(team_ids),
start_date,
end_date,
)
return resp.model_dump(mode="json")
async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TAG_SPEND,
ENTITY_FIELD_TAG,
_parse_csv_ids(tags),
start_date,
end_date,
)
return resp.model_dump(mode="json")
# ---------------------------------------------------------------------------
# Summarisers — convert raw JSON to concise text the LLM can reason over
# ---------------------------------------------------------------------------
def _accumulate_breakdown(
results: Sequence[_DayDump], dimension: str, fields: Sequence[str]
) -> dict[str, dict[str, float]]:
"""Aggregate a single breakdown dimension across days."""
totals: Final[dict[str, dict[str, float]]] = {}
for day in results:
for key, entry in day.get("breakdown", {}).get(dimension, {}).items():
if key not in totals:
totals[key] = {f: 0.0 for f in fields}
m = entry.get("metrics", {})
for f in fields:
totals[key][f] += m.get(f, 0)
return totals
def _ranked_lines(
totals: dict[str, dict[str, float]],
fmt: Callable[[str, dict[str, float]], str],
limit: int,
) -> list[str]:
"""Sort by spend descending, format each entry, and truncate."""
return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]]
def _summarise_usage_data(data: _UsageDump) -> str:
meta: Final = data.get("metadata", {})
results: Final = data.get("results", [])
header: Final = (
f"Total Spend: ${meta.get('total_spend', 0):.4f}\n"
f"Total Requests: {meta.get('total_api_requests', 0)}\n"
f"Successful: {meta.get('total_successful_requests', 0)} | "
f"Failed: {meta.get('total_failed_requests', 0)}\n"
f"Total Tokens: {meta.get('total_tokens', 0)}"
)
models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"])
providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"])
model_lines: Final = _ranked_lines(
models,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)",
TOP_N_MODELS,
)
provider_lines: Final = _ranked_lines(
providers,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)",
TOP_N_PROVIDERS,
)
sections = [header, ""]
sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""]
sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"])
return "\n".join(sections)
def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str:
"""Summarise team/tag entity usage data."""
results: Final = data.get("results", [])
if not results:
return f"No {entity_label} usage data found for the given date range."
totals: Final[dict[str, _EntityTotal]] = {}
for day in results:
for eid, entry in day.get("breakdown", {}).get("entities", {}).items():
previous = totals.get(eid)
m = entry.get("metrics", {})
totals[eid] = _EntityTotal(
alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid),
spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0),
requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0),
tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0),
)
lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""]
for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend):
label = d.alias if d.alias != eid else eid
lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Tool dispatch registry
# ---------------------------------------------------------------------------
TOOL_HANDLERS: Final[dict[str, ToolHandler]] = {
"get_usage_data": ToolHandler(
fetch=_fetch_usage_data,
summarise=_summarise_usage_data,
label="global usage data",
),
"get_team_usage_data": ToolHandler(
fetch=_fetch_team_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Team"),
label="team usage data",
),
"get_tag_usage_data": ToolHandler(
fetch=_fetch_tag_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Tag"),
label="tag usage data",
),
}
# ---------------------------------------------------------------------------
# SSE streaming
# ---------------------------------------------------------------------------
def _sse(event: SSEEvent) -> str:
return f"data: {json.dumps(event)}\n\n"
def _resolve_fetch_kwargs(
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> dict[str, str]:
"""Build keyword arguments for a tool's fetch function."""
start_date: Final = fn_args.get("start_date", "")
end_date: Final = fn_args.get("end_date", "")
if not start_date or not end_date:
raise ValueError("Missing required start_date or end_date from tool arguments")
kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date}
if fn_name == "get_usage_data":
if not is_admin:
if user_id is None:
# Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
# should have already rejected this. If we ever reach here it means
# a future caller invoked the helper without scoping — fail loudly
# rather than issuing an unfiltered global query.
raise ValueError(
"Non-admin caller has user_id=None; refusing to issue an "
"unscoped query. Endpoint-level guard missing."
)
kwargs["user_id"] = user_id
elif fn_args.get("user_id"):
kwargs["user_id"] = fn_args["user_id"]
elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"):
kwargs["team_ids"] = fn_args["team_ids"]
elif fn_name == "get_tag_usage_data" and fn_args.get("tags"):
kwargs["tags"] = fn_args["tags"]
return kwargs
async def _execute_tool_call(
handler: ToolHandler,
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> str:
"""Run a single tool and return the summarised result text, or the reason the query
layer rejected its dates so the model can correct them."""
kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin)
try:
raw_data: Final = await handler["fetch"](**kwargs)
except HTTPException as e:
reason: Final = _invalid_date_reason(e)
if reason is None:
raise
return f"Invalid date range: {reason}"
return handler["summarise"](raw_data)
def _invalid_date_reason(error: HTTPException) -> str | None:
"""The reason in the daily activity query layer's 400 for invalid or reversed dates."""
detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict
match detail:
case {"error": str() as reason} if error.status_code == 400:
return reason
case _:
return None
async def _process_tool_call(
tc: ChatCompletionMessageToolCall,
chat_messages: list[Mapping[str, object]],
user_id: str | None,
is_admin: bool,
) -> AsyncIterator[str]:
"""Execute a single tool call, yielding SSE events for status."""
fn_name: Final = tc.function.name
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None
if fn_name is None or fn_name not in allowed_names or not handler:
chat_messages.append(
{
"role": "tool",
"tool_call_id": tc.id,
"content": f"Tool not available: {fn_name}",
}
)
return
tool_event_base: Final = {
"type": "tool_call",
"tool_name": fn_name,
"tool_label": handler["label"],
"arguments": fn_args,
}
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"}))
try:
tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin)
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"}))
except Exception as e:
verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e)
tool_result = f"Error fetching {handler['label']}. Please try again."
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"}))
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
"""Stream the final LLM response after tool results are appended."""
yield _sse({"type": "status", "message": "Analyzing results..."})
response: Final = await litellm.acompletion(
model=model,
messages=chat_messages,
stream=True,
temperature=USAGE_AI_TEMPERATURE,
)
async for chunk in response:
delta = chunk.choices[0].delta.content
if delta:
yield _sse({"type": "chunk", "content": delta})
async def stream_usage_ai_chat(
messages: list[dict[str, str]],
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
chat_messages: Final[list[Mapping[str, object]]] = [
{"role": "system", "content": _build_system_prompt(is_admin)},
*truncated,
]
try:
yield _sse({"type": "status", "message": "Thinking..."})
tools: Final = get_tools_for_role(is_admin)
response: Final = await litellm.acompletion(
model=resolved_model,
messages=chat_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
)
choice: Final = response.choices[0]
if not choice.message.tool_calls:
if choice.message.content:
yield _sse({"type": "chunk", "content": choice.message.content})
yield _sse({"type": "done"})
return
chat_messages.append(choice.message.model_dump())
for tc in choice.message.tool_calls:
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
yield event
async for event in _stream_final_response(resolved_model, chat_messages):
yield event
yield _sse({"type": "done"})
except Exception as e:
verbose_proxy_logger.error("AI usage chat failed: %s", e)
yield _sse(
{
"type": "error",
"message": "An internal error occurred. Please try again.",
}
)

View file

@ -1,77 +0,0 @@
"""
USAGE AI CHAT ENDPOINTS
/usage/ai/chat - Stream AI chat responses about usage data
"""
from typing import Final, Literal
from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from pydantic import Field
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
wrap_sse_stream_with_keepalive_pings,
)
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
class ChatMessage(LiteLLMBaseModel):
role: Literal["user", "assistant"]
content: str
class UsageAIChatRequest(LiteLLMBaseModel):
messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)")
model: str | None = Field(default=None, description="Model to use for AI chat")
@router.post(
"/usage/ai/chat",
tags=["Budget & Spend Tracking"],
dependencies=[Depends(user_api_key_auth)],
)
async def usage_ai_chat(
data: UsageAIChatRequest,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
AI chat about usage data. Streams SSE events with the AI response.
The AI agent has access to tools that query aggregated daily activity data.
"""
from litellm.proxy.management_endpoints.common_utils import (
require_caller_user_id_for_non_admin,
user_api_key_has_admin_view,
)
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
stream_usage_ai_chat,
)
is_admin: Final = user_api_key_has_admin_view(user_api_key_dict)
if is_admin:
user_id = user_api_key_dict.user_id
else:
user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
return StreamingResponse(
wrap_sse_stream_with_keepalive_pings(
stream_usage_ai_chat(
messages=messages,
model=data.model,
user_id=user_id,
is_admin=is_admin,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)

View file

@ -0,0 +1,109 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from litellm.proxy.utils import PrismaClient
COMPUTED_TEAM_SORTS: Final = MappingProxyType(
{
"organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') "
'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), '
"NULLIF(t.organization_id, ''))",
"members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' "
"THEN jsonb_array_length(t.members_with_roles) ELSE 0 END",
"models": "cardinality(t.models)",
"keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)',
}
)
_WHERE: Final = TypeAdapter(dict[str, JsonValue])
@dataclass(frozen=True, slots=True)
class UnsupportedTeamFilter:
field: str
def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
if field not in ("team_id", "team_alias", "organization_id"):
return UnsupportedTeamFilter(field)
column: Final = f't."{field}"'
match value:
case str(text):
return f"{column} = ${index}", (text,)
case {"in": list(values)}:
return f"{column} = ANY(${index}::text[])", (values,)
case {"contains": str(text), "mode": "insensitive"}:
return f"{column} ILIKE ${index}", (f"%{text}%",)
case {"startsWith": str(text)}:
return f"{column} LIKE ${index}", (f"{text}%",)
case _:
return UnsupportedTeamFilter(field)
def _group_tokens(
field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue]
) -> tuple[str | tuple[str, JsonValue], ...]:
children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value)
if not children:
return ("FALSE" if field == "OR" else "TRUE",)
return (
"(",
*chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)),
")",
)
def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal
fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying
params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order
while pending:
match pending.pop():
case str(fragment):
fragments.append(fragment)
case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)):
pending.extend(reversed(_group_tokens(field, value)))
case (field, value):
match _field_sql(field, value, len(params) + 1):
case UnsupportedTeamFilter() as failure:
return failure
case (fragment, values):
fragments.append(fragment)
params.extend(values)
return "".join(fragments), tuple(params)
class _TeamId(BaseModel):
model_config = ConfigDict(frozen=True)
team_id: str
_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...])
async def sorted_team_ids(
prisma_client: PrismaClient,
*,
where: Mapping[str, object],
sort_by: str,
descending: bool,
skip: int,
take: int,
) -> Sequence[str] | UnsupportedTeamFilter:
result: Final = _where_sql(_WHERE.validate_python(where))
if isinstance(result, UnsupportedTeamFilter):
return result
predicate, params = result
expression: Final = COMPUTED_TEAM_SORTS[sort_by]
direction: Final = "DESC" if descending else "ASC"
sql: Final = (
f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} '
f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC "
f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip))
return tuple(row.team_id for row in rows)

View file

@ -0,0 +1,77 @@
from collections.abc import Sequence
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy.utils import PrismaClient
class _UserId(BaseModel):
model_config = ConfigDict(frozen=True)
user_id: str
_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...])
async def sorted_user_ids(
prisma_client: PrismaClient,
*,
sort_by: Literal["status", "key_count"],
descending: bool,
skip: int,
take: int,
role: str | None,
user_ids: str | None,
user_email: str | None,
team: str | None,
sso_user_ids: str | None,
organization_ids: str | None,
search: str | None,
) -> Sequence[str]:
ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip())
sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip())
org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip())
expression: Final = (
"CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END"
if sort_by == "status"
else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k '
"WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)"
)
direction: Final = "DESC" if descending else "ASC"
sql: Final = f"""
SELECT u.user_id FROM "LiteLLM_UserTable" u
WHERE ($1::text IS NULL OR u.user_role = $1)
AND ($2::text[] IS NULL
OR CASE WHEN cardinality($2::text[]) = 1
THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%'
ELSE u.user_id = ANY($2::text[]) END)
AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%')
AND ($4::text IS NULL OR $4 = ANY(u.teams))
AND ($5::boolean OR u.sso_user_id = ANY($6::text[]))
AND (cardinality($7::text[]) = 0 OR EXISTS (
SELECT 1 FROM "LiteLLM_OrganizationMembership" m
WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[])))
AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%'
OR u.user_email ILIKE '%' || $8 || '%')
ORDER BY {expression} {direction}, u.user_id ASC
LIMIT $9 OFFSET $10
"""
rows: Final = _USER_IDS.validate_python(
await prisma_client.db.query_raw(
sql,
role or None,
ids if user_ids else None,
user_email,
team,
sso_user_ids is None,
sso_ids,
org_ids,
search or None,
take,
skip,
*((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()),
)
)
return tuple(row.user_id for row in rows)

View file

@ -12210,7 +12210,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
async def model_list(
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
return_wildcard_routes: bool | None = False,
return_wildcard_routes: bool | None = None,
team_id: str | None = None,
include_model_access_groups: bool | None = False,
only_model_access_groups: bool | None = False,
@ -12226,6 +12226,9 @@ async def model_list(
This is just for compatibility with openai projects like aider.
Query Parameters:
- return_wildcard_routes: When true, also list wildcard routes such as `openai/*`.
Defaults to `general_settings.return_wildcard_routes`, and an explicit
value on the request wins over it
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
provider's model catalog. When present, the response is Codex's own
catalog shape (`{"models": [...]}`) built from the same listing, with
@ -12254,6 +12257,9 @@ async def model_list(
global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
settings: Final = cast(dict[str, object], general_settings) # any-ok: legacy settings
include_wildcard_routes: Final = (
return_wildcard_routes if return_wildcard_routes is not None else settings.get("return_wildcard_routes") is True
)
from litellm.llms.anthropic.common_utils import (
create_anthropic_model_list_response,
@ -12343,7 +12349,7 @@ async def model_list(
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=return_wildcard_routes or False,
return_wildcard_routes=include_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups or False,
@ -12411,7 +12417,7 @@ async def model_list(
team_id=team_id,
include_model_access_groups=include_model_access_groups or False,
only_model_access_groups=only_model_access_groups or False,
return_wildcard_routes=return_wildcard_routes or False,
return_wildcard_routes=include_wildcard_routes,
user_api_key_cache=user_api_key_cache,
)
@ -19657,6 +19663,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro
"proxy_config_reload_interval_seconds": "Integer",
"pass_through_endpoints": "PydanticModel",
"store_model_in_db": "Boolean",
"return_wildcard_routes": "Boolean",
"store_prompts_in_spend_logs": "Boolean",
"disable_requester_ip_address_logging": "Boolean",
"disable_fallbacks_on_per_model_rate_limits": "Boolean",

View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import (
get_blog_posts,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy._types import (
CommonProxyErrors,
)
@ -564,7 +565,7 @@ async def _resolve_autorouter_presets(
url: str,
fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],
) -> Mapping[str, AutoRouterPresetRecord]:
if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
return _load_bundled_autorouter_presets()
try:
return await fetch(url)

View file

@ -1,5 +1,7 @@
import asyncio
import os
import re
import xml.etree.ElementTree as ET
from collections import Counter
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass
@ -12,12 +14,14 @@ from pydantic import ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest"
CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/"
CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml"
RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}"
LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5
LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60
@ -26,7 +30,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info"
_RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S")
_NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b")
_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/")
_Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"]
_PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"})
@ -45,6 +49,16 @@ class LatestReleaseUnavailable:
reason: str
@dataclass(frozen=True, slots=True)
class ChangelogEntry:
version: tuple[int, int, int]
url: str
@property
def tag(self) -> str:
return "v" + ".".join(str(part) for part in self.version)
class _GitHubRelease(LiteLLMBaseModel):
tag_name: str
html_url: str
@ -76,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock:
return _latest_release_fetch_lock
def _default_release_check_enabled() -> bool:
return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true"
def _bucket_for(line: str) -> _Bucket | None:
if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None:
return None
@ -91,9 +109,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]:
return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None))
def _stable_version(tag_name: str) -> tuple[int, int, int] | None:
match: Final = _STABLE_TAG_PATTERN.match(tag_name)
return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3)))
def _changelog_entry(link: str) -> ChangelogEntry | None:
match: Final = _CHANGELOG_PAGE_PATTERN.match(link)
if match is None:
return None
return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link)
def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None:
"""The highest stable release with a page in the changelog feed. Release candidate pages don't count."""
links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link"))
entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None)
return max(entries, key=lambda entry: entry.version, default=None)
def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
@ -102,7 +129,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch))
def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo:
def summarize_release_line(
entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]
) -> LatestReleaseInfo:
"""A patch release's notes only list its backports, so count the whole minor line it belongs to."""
counts: Final = sum(
(Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)),
@ -113,7 +142,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease
new_features=counts.get("new_features", 0),
bug_fixes=counts.get("bug_fixes", 0),
other_updates=counts.get("other_updates", 0),
release_url=latest.html_url,
release_url=entry.url,
)
@ -137,25 +166,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe
return release
async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable:
response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
if response.status_code != 200:
return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}")
try:
entry: Final = latest_changelog_entry(response.content)
except ET.ParseError as e:
return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}")
return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry
async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable:
try:
entry: Final = await _fetch_changelog_entry(client)
if isinstance(entry, LatestReleaseUnavailable):
return entry
latest: Final = parse_github_release(
await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
)
if isinstance(latest, LatestReleaseUnavailable):
return latest
version: Final = _stable_version(latest.tag_name)
if version is None:
return summarize_release_line(latest, ())
earlier: Final = await asyncio.gather(
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version))
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version))
)
except httpx.HTTPError as e:
return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}")
unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None)
if unavailable is not None:
return unavailable
return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)))
return summarize_release_line(
entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))
)
async def get_latest_release_info(
@ -188,12 +230,16 @@ async def latest_release_info(
client: Annotated[_AsyncGetClient, Depends(_default_client)],
cache: Annotated[InMemoryCache, Depends(_default_cache)],
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)],
) -> LatestReleaseInfo | None:
"""
Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
split into new features, bug fixes and other updates.
Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request.
"""
if not release_check_enabled:
return None
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
if isinstance(result, LatestReleaseUnavailable):
verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason)

View file

@ -6032,13 +6032,12 @@ class PrismaClient:
elif table_name == "user":
db_data = self.jsonify_object(data=data)
try:
new_user_row: Final = await UserRepository(self).table.upsert(
where={"user_id": data["user_id"]},
data={
"create": {**db_data},
"update": {}, # don't do anything if it already exists
},
)
new_user_row: Final = await UserRepository(self).upsert_user(db_data)
if new_user_row is None:
raise HTTPException(
status_code=403,
detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.",
)
except Exception as e:
if (
"Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`"

View file

@ -2,8 +2,11 @@
User repository for database operations on LiteLLM_UserTable.
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Generator, Mapping, Sequence
from contextlib import AbstractAsyncContextManager, contextmanager
from contextvars import ContextVar
from itertools import chain
from typing import TYPE_CHECKING, Final, Protocol
@ -35,6 +38,16 @@ ORDER BY p.user_id
"""
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None)
@contextmanager
def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]:
token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None)
try:
yield
finally:
_FREE_SSO_CREATION.reset(token)
class _UserDb(Protocol):
@ -42,6 +55,12 @@ class _UserDb(Protocol):
def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ...
class _UserTransaction(_UserDb, Protocol):
async def execute_raw(self, query: str) -> int: ...
async def query_raw(self, query: str) -> object: ...
class _PrismaClientView(Protocol):
@property
def db(self) -> _UserDb: ...
@ -49,6 +68,16 @@ class _PrismaClientView(Protocol):
@property
def writer_db(self) -> _UserDb: ...
def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ...
async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int:
from prisma import Json
total: Final = await table.count()
deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Repository for user database operations."""
@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
null, or true all count, so seats are counted as total users minus
the deactivated ones.
"""
from prisma import Json # pyright: ignore[reportUnknownVariableType]
return await _count_billable_users(self.table)
total: Final = await self.count()
deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None":
"""Return the user row, or None when a new free SSO user would exceed capacity."""
scope: Final = _FREE_SSO_CREATION.get()
if (
scope is None
or (scope[0] is not None and scope[0] != data["user_id"])
or scope[1] is not asyncio.current_task()
):
return await self.table.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
client: Final[_PrismaClientView] = self.prisma_client
async with client.tx() as transaction:
await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)")
existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]})
if existing is not None:
return existing
if await _count_billable_users(transaction.litellm_usertable) >= 5:
return None
return await transaction.litellm_usertable.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
async def create_user(
self,

View file

@ -3663,18 +3663,20 @@ class LiteLLMCompletionResponsesConfig:
if count is not None
}
)
response_usage.input_tokens_details = InputTokensDetails(
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
text_tokens=prompt_details.text_tokens,
audio_tokens=prompt_details.audio_tokens,
image_tokens=prompt_details.image_tokens,
video_tokens=prompt_details.video_tokens,
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
**cache_type_extra,
**grounding_request_counts,
response_usage.input_tokens_details = InputTokensDetails.model_validate(
{
"cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
"text_tokens": prompt_details.text_tokens,
"audio_tokens": prompt_details.audio_tokens,
"image_tokens": prompt_details.image_tokens,
"video_tokens": prompt_details.video_tokens,
"cached_tokens_details": (
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
**cache_type_extra,
**grounding_request_counts,
}
)
# Translate completion_tokens_details to output_tokens_details

View file

@ -6,10 +6,10 @@ from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
import httpx
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from typing_extensions import (
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
assert_never,
@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.litellm_completion_transformation.compaction_items import (
with_compaction_items_as_summaries,
@ -85,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.types.responses.main import *
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
CallTypes,
CostPerToken,
CustomPricingLiteLLMParams,
all_litellm_params,
)
from litellm.utils import (
ProviderConfigManager,
client,
@ -544,6 +552,47 @@ async def aresponses_api_with_mcp(
return response
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
return MappingProxyType(
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
)
class _DeploymentModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: object = None
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
)
def _is_router_deployment(deployment_model_info: object) -> bool:
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
return None
return _COST_PER_TOKEN.validate_python(
litellm_params.model_dump(
include={
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
},
exclude_none=True,
)
)
def _bridges_to_chat_completions(
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
) -> bool:
@ -1466,6 +1515,12 @@ def responses(
# get llm provider logic
litellm_params: Final = GenericLiteLLMParams(**kwargs)
affinity_api_base: Final = (
get_openai_credentials(api_base=litellm_params.api_base).api_base
if custom_llm_provider == "openai"
else litellm_params.api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
try:
effective_extra_headers: Final = (
add_provider_affinity_header(
@ -1479,8 +1534,9 @@ def responses(
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
if litellm_params.provider_affinity_header is not None
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
else extra_headers
)
except ValueError as affinity_error:
@ -1664,6 +1720,7 @@ def responses(
optional_params=dict(responses_api_request_params),
litellm_params={
**responses_api_request_params,
**_per_call_custom_pricing(litellm_params),
"aresponses": _is_async,
"litellm_call_id": litellm_call_id,
"model_info": kwargs.get("model_info"),
@ -1674,6 +1731,9 @@ def responses(
custom_llm_provider=custom_llm_provider,
)
if not _is_router_deployment(deployment_model_info):
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
input = with_compaction_items_as_summaries(
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))

View file

@ -13,6 +13,7 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
from litellm.types.llms.openai import (
AllMessageValues,
InputTokensDetails,
OutputTokensDetails,
ResponseAPIUsage,
ResponseInputParam,
@ -23,6 +24,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.responses.main import DecodedResponseId
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
SpecialEnums,
@ -41,6 +43,17 @@ def _output_token_detail(details: object, field: str) -> int | None:
return value if isinstance(value, int) else None
def _cache_write_lifetimes(cache_write_tokens_details: object) -> CacheCreationTokenDetails | None:
"""OpenAI Responses splits `cache_write_tokens` into `ttl_30m_tokens` and `ttl_12h_tokens`."""
ttl_30m_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_30m_tokens")
ttl_12h_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_12h_tokens")
if ttl_30m_tokens is None and ttl_12h_tokens is None:
return None
return CacheCreationTokenDetails(
ephemeral_30m_input_tokens=ttl_30m_tokens, ephemeral_12h_input_tokens=ttl_12h_tokens
)
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
return isinstance(value, list)
@ -1247,7 +1260,11 @@ class ResponseAPILoggingUtils:
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
if response_api_usage.input_tokens_details:
if isinstance(response_api_usage.input_tokens_details, dict):
prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details)
input_details: Final = InputTokensDetails.model_validate(response_api_usage.input_tokens_details)
prompt_tokens_details = PromptTokensDetailsWrapper(
**input_details.model_dump(exclude={"cache_write_tokens_details"}),
cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details),
)
else:
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
@ -1259,6 +1276,9 @@ class ResponseAPILoggingUtils:
),
video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
cache_creation_token_details=_cache_write_lifetimes(
getattr(response_api_usage.input_tokens_details, "cache_write_tokens_details", None)
),
cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None),
web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None),
google_maps_grounding_requests=getattr(

View file

@ -14186,6 +14186,43 @@ class Router:
deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
)
@staticmethod
def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool:
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
declared: Final = model_info.get("supports_vision")
if declared is not None:
return declared is False
raw_params: Final = plain_deployment.get("litellm_params") or {}
params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts
if not params:
return False
base_model: Final = model_info.get("base_model") or params.get("base_model")
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return litellm.utils.is_vision_explicitly_disabled(
base_model if isinstance(base_model, str) else model, custom_llm_provider
)
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
verbose_router_logger.debug(
"litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
)
return False
@staticmethod
def _request_carries_image(
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
request_kwargs: Mapping[str, object],
) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
structured: Final = resolve_structured_messages(
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
)
return bool(structured) and request_contains_image_content(structured)
@staticmethod
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
@ -14212,6 +14249,7 @@ class Router:
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- request carries an image and the deployment declares supports_vision false, unless every candidate does
- [TODO] function call and model doesn't support function calling
"""
@ -14221,7 +14259,7 @@ class Router:
# We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
_returned_deployments = list(healthy_deployments)
invalid_model_indices: Final = set() # Use set for O(1) membership checks
invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks
_context_window_error = False
_potential_error_str = ""
@ -14371,9 +14409,20 @@ class Router:
model=model,
llm_provider="",
)
if len(excluded_indices) > 0:
remaining: Final = tuple(
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
)
text_only_indices: Final[frozenset[int]] = (
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
if remaining and self._request_carries_image(messages, request_settings)
else frozenset()
)
dropped_indices: Final[frozenset[int]] = excluded_indices | (
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
)
if len(dropped_indices) > 0:
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
return _returned_deployments

View file

@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
}
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.PROVIDER_AFFINITY,
Feature.SAFEGUARDS_CLASSIFIER,
Feature.PRE_REQUEST_HOOK,
Feature.WEBSEARCH_INTERCEPTION,

View file

@ -5,10 +5,15 @@ from __future__ import annotations
from typing import Final
import litellm
from litellm.rust_bridge.detectors import overridden, present, streaming
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
def provider_affinity(request: NativeCall) -> bool:
return is_opencode_api_base(api_base(request))
def anthropic_workload_identity(request: NativeCall) -> bool:
"""Python mints an Anthropic federation token when no static credential is configured."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo

View file

@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay
from pydantic import TypeAdapter
import litellm
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool:
def provider_affinity(request: NativeCall) -> bool:
"""Python validates and fills the affinity header from the session ID."""
return present(request, "provider_affinity_header")
return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request))
def text_format(request: NativeCall) -> bool:

View file

View file

@ -0,0 +1,88 @@
from typing import Final
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
class TextCompletionStreamWrapper:
def __init__(
self,
completion_stream,
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
def __iter__(self):
return self
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
response.usage = chunk.get("usage", None)
else:
response.usage = None
return response
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
try:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopIteration
except Exception as e:
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider or "",
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
async def __anext__(self):
try:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopAsyncIteration

View file

@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[
DecisionQuestionMap: TypeAlias = Annotated[
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
Field(min_length=1, max_length=MAX_DECISION_QUESTIONS),
Field(min_length=1),
]

View file

@ -9,16 +9,38 @@ https://ai.google.dev/static/api/interactions.openapi.json
### How to Regenerate
When the API spec changes, regenerate the types with:
The compliance tests in `tests/unit/interactions/test_openapi_compliance.py` read the snapshot at
`tests/unit/interactions/interactions.openapi.json`, so a PR never depends on Google's live spec. The scheduled
`interactions-spec-canary` workflow runs those tests against the live spec once a day (`--interactions-spec-url`),
files the issue "Interactions spec canary is red" on a red run and closes it once a run is green again. A red run
means the spec moved in a way these types no longer cover
When the red run names one new status, the small fix is a hand edit adding that member to the enum the failing
schema's model carries, the way `queued` was added in #34135. When the spec moved further than that, refresh the
snapshot and regenerate the types from it:
```bash
uv tool run --from datamodel-code-generator datamodel-codegen \
--url "https://ai.google.dev/static/api/interactions.openapi.json" \
curl -fsSL https://ai.google.dev/static/api/interactions.openapi.json \
-o tests/unit/interactions/interactions.openapi.json
uv tool run --from datamodel-code-generator==0.66.0 datamodel-codegen \
--input tests/unit/interactions/interactions.openapi.json \
--input-file-type openapi \
--output litellm/types/interactions/generated.py \
--output-model-type pydantic_v2.BaseModel \
--target-python-version 3.9
--target-python-version 3.10 \
--base-class litellm.types.llms.base.LiteLLMBaseModel
```
The pin is the same `datamodel-code-generator` version `scripts/generate_trace_types.py` uses. The generator names
inline enums by order of appearance (`Status`, `Status1`, `Status2`, ...), so a regeneration can rename them. The
compliance tests resolve each status enum through its model's `status` field, but
`tests/unit/interactions/test_background_cost_polling.py` and
`tests/unit/proxy/hooks/test_proxy_track_cost_callback.py` import `Status1` by name and need the new name, and
`litellm/types/interactions/__init__.py` exports the interaction status enum as `InteractionStatus`, which has to
point at the enum the regenerated `Interaction.status` carries (a compliance test checks their members match). The
spec has grown a lot since the 2025-12-16 generation (agents, triggers, credentials, environments), so a
regeneration today is a review-sized change rather than a refresh
Then add the LiteLLM-specific types at the bottom of the generated file:
- `InteractionsAPIResponse`
- `InteractionsAPIStreamingResponse`

View file

@ -683,12 +683,12 @@ class ChatCompletionCachedContent(TypedDict):
class PromptCacheBreakpoint(TypedDict):
mode: ReadOnly[Literal["explicit"]]
ttl: NotRequired[ReadOnly[Literal["30m"]]]
ttl: NotRequired[ReadOnly[Literal["30m", "12h"]]]
class PromptCacheOptions(TypedDict, total=False):
mode: ReadOnly[Literal["implicit", "explicit"]]
ttl: ReadOnly[Literal["30m"]]
ttl: ReadOnly[Literal["30m", "12h"]]
class ChatCompletionThinkingBlock(TypedDict, total=False):
@ -1382,10 +1382,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
model_config = {"extra": "allow"}
class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject):
ttl_30m_tokens: int
ttl_12h_tokens: int
model_config = ConfigDict(extra="allow", frozen=True)
def _is_none(value: object) -> bool:
return value is None
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none)
cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none)
image_tokens: int | None = None
text_tokens: int | None = None
video_tokens: int | None = None

View file

@ -322,11 +322,19 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_above_272k_tokens_flex: float | None
cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_creation_input_token_cost_above_1hr: float | None
cache_creation_input_token_cost_above_12hr: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_flex: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_priority: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None]
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_audio_token_cost_flex: ReadOnly[float | None]
cache_read_input_audio_token_cost_priority: ReadOnly[float | None]
cache_read_input_image_token_cost: ReadOnly[float | None]
explicit_cache_read_input_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
@ -342,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_read_input_token_cost_above_512k_tokens: float | None
cache_read_input_token_cost_batches: ReadOnly[float | None]
cache_read_input_audio_token_cost_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
@ -355,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates
input_cost_per_character: float | None # only for vertex ai models
input_cost_per_audio_token: float | None
input_cost_per_audio_token_flex: ReadOnly[float | None]
input_cost_per_audio_token_priority: ReadOnly[float | None]
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
input_cost_per_token_above_100k_tokens: ReadOnly[float | None]
@ -1804,7 +1815,9 @@ class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for ol
class CacheCreationTokenDetails(LiteLLMBaseModel):
ephemeral_5m_input_tokens: int | None = None
ephemeral_30m_input_tokens: int | None = None
ephemeral_1h_input_tokens: int | None = None
ephemeral_12h_input_tokens: int | None = None
class PromptTokensDetailsWrapper(
@ -3954,6 +3967,12 @@ class MirroredPricingParams(LiteLLMBaseModel):
explicit_cache_read_input_token_cost: float | None = None
cache_creation_input_token_cost: float | None = None
cache_creation_input_token_cost_above_1hr: float | None = None
cache_creation_input_token_cost_above_12hr: float | None = None
cache_creation_input_token_cost_above_12hr_flex: float | None = None
cache_creation_input_token_cost_above_12hr_priority: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
tiered_pricing: list[dict[str, Any]] | None = None
@ -4006,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_read_input_token_cost_above_100k_tokens_batches: float | None = None
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
cache_read_input_audio_token_cost_batches: float | None = None
cache_creation_input_token_cost_batches: float | None = None
cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None
cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None
cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None
cache_read_input_audio_token_cost: float | None = None
cache_read_input_audio_token_cost_flex: float | None = None
cache_read_input_audio_token_cost_priority: float | None = None
cache_read_input_image_token_cost: float | None = None
input_cost_per_character_above_128k_tokens: float | None = None
input_cost_per_audio_token: float | None = None
input_cost_per_audio_token_flex: float | None = None
input_cost_per_audio_token_priority: float | None = None
input_cost_per_token_cache_hit: float | None = None
input_cost_per_token_above_128k_tokens: float | None = None
input_cost_per_token_above_100k_tokens: float | None = None

View file

@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
)
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
@ -218,6 +219,7 @@ def _get_cached_audio_utils():
return _audio_utils_module
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper
from litellm.types.llms.openai import (
AllMessageValues,
AllPromptValues,
@ -262,13 +264,17 @@ from litellm.types.utils import (
SearchProviders,
SelectTokenizerResponse,
StreamingChoices,
TextChoices,
TextCompletionResponse,
TranscriptionResponse,
Usage,
credential_api_key_secret_reference,
is_litellm_owned_kwarg,
)
from litellm.types.utils import (
TextChoices as _TextChoices,
)
from litellm.types.utils import (
TextCompletionResponse as _TextCompletionResponse,
)
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError):
claude_json_str = json.dumps(json_data)
import importlib.metadata
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
from typing_extensions import assert_never
TextChoices: TypeAlias = _TextChoices
TextCompletionResponse: TypeAlias = _TextCompletionResponse
TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper
from litellm import utils as litellm_utils
# These are lazy loaded via __getattr__
@ -340,7 +350,8 @@ if TYPE_CHECKING:
# Type stubs for lazy-loaded functions to help mypy understand their types
# These imports allow mypy to understand the types when these are accessed via __getattr__
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
exception_type: Callable[..., Exception]
from litellm.litellm_core_utils.get_litellm_params import (
get_base_model_from_litellm_call_metadata,
get_litellm_params,
@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None:
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None:
kind: Final = huggingface_tokenizer_kind(model)
if kind is None:
if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()):
return None
backend: Final = _huggingface_tokenizer_backend()
key: Final = _huggingface_tokenizer_failure_key(kind, backend)
@ -6762,6 +6773,13 @@ def get_model_info_helper(
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
cache_read_input_audio_token_cost_priority=_model_info.get(
"cache_read_input_audio_token_cost_priority", None
),
cache_read_input_audio_token_cost_batches=_model_info.get(
"cache_read_input_audio_token_cost_batches", None
),
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
@ -6806,6 +6824,24 @@ def get_model_info_helper(
cache_creation_input_token_cost_above_1hr=_model_info.get(
"cache_creation_input_token_cost_above_1hr", None
),
cache_creation_input_token_cost_above_12hr=_model_info.get(
"cache_creation_input_token_cost_above_12hr", None
),
cache_creation_input_token_cost_above_12hr_flex=_model_info.get(
"cache_creation_input_token_cost_above_12hr_flex", None
),
cache_creation_input_token_cost_above_12hr_priority=_model_info.get(
"cache_creation_input_token_cost_above_12hr_priority", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None
),
off_peak_pricing=_model_info.get("off_peak_pricing", None),
input_cost_per_character=_model_info.get("input_cost_per_character", None),
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
@ -6829,6 +6865,8 @@ def get_model_info_helper(
input_cost_per_second=_model_info.get("input_cost_per_second", None),
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
@ -7945,88 +7983,6 @@ def register_prompt_template(
return litellm.custom_prompt_dict
class TextCompletionStreamWrapper:
def __init__(
self,
completion_stream,
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
def __iter__(self):
return self
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
response["usage"] = chunk.get("usage", None)
return response
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
try:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopIteration
except Exception as e:
exception_type: Final = litellm_utils.exception_type
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider or "",
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
async def __anext__(self):
try:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopAsyncIteration
def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream:
return ModelResponseStream(
id=model_response.id,

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -61,7 +61,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
@ -118,7 +118,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 nodejs libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -31041,11 +31137,15 @@
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31370,11 +31470,15 @@
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
"cache_read_input_audio_token_cost_batches": 3e-08,
"cache_read_input_audio_token_cost_flex": 3e-08,
"cache_read_input_audio_token_cost_priority": 5.4e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 1e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_priority": 1.8e-08,
"input_cost_per_audio_token": 3e-07,
"input_cost_per_audio_token_flex": 1.5e-07,
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31664,12 +31768,16 @@
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_audio_token_cost_batches": 2.5e-08,
"cache_read_input_audio_token_cost_flex": 2.5e-08,
"cache_read_input_audio_token_cost_priority": 9e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"deprecation_date": "2027-05-07",
"input_cost_per_audio_token": 5e-07,
"input_cost_per_audio_token_flex": 2.5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
@ -31790,10 +31898,14 @@
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 5e-08,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -36699,6 +36811,12 @@
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
@ -36784,6 +36902,12 @@
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -36861,17 +36985,23 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-08,
"cache_creation_input_token_cost_priority": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_decisions": 1e-07,
"cache_read_input_token_cost_decisions": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
"cache_read_input_token_cost_batches": 5e-09,
@ -36947,6 +37077,12 @@
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37021,6 +37157,12 @@
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37097,6 +37239,12 @@
},
"gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -37172,6 +37320,12 @@
},
"gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_12hr": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
@ -42337,6 +42491,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -45623,6 +45905,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45648,6 +45931,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -48499,6 +48783,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -49352,6 +49647,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49371,6 +49667,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -52906,12 +53203,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53225,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -59917,6 +60218,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -59942,6 +60244,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -59967,6 +60270,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -59990,6 +60294,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -67841,6 +68146,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -67860,6 +68166,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68138,6 +68445,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68157,6 +68465,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68545,6 +68854,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68564,6 +68874,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68704,6 +69015,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68723,6 +69035,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -73330,6 +73643,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
@ -74677,6 +74991,12 @@
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,

View file

@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v"
ignoreUntil = 2026-11-01
reason = "diskcache has no fixed release published; remove this entry once one exists"
[[IgnoredVulns]]
id = "GHSA-v4x8-3wr5-3wq8"
ignoreUntil = 2026-11-20
reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships"
[[IgnoredVulns]]
id = "GHSA-vfj7-8cjw-p6xm"
ignoreUntil = 2026-11-03

View file

@ -30,6 +30,7 @@ dependencies = [
"pydantic>=2.12.0,<3.0.0; python_version >= '3.14'",
"pydantic-settings>=2.14.1,<3.0",
"jsonschema>=4.0.0,<5.0",
"certifi>=2026.10.19",
"typing-extensions>=4.13.0,<5.0",
]

View file

@ -35,7 +35,8 @@ dependencies = [
"pydantic>=2.12.0,<3.0.0; python_version >= '3.14'",
"pydantic-settings>=2.14.1,<3.0",
"jsonschema>=4.0.0,<5.0",
"boto3>=1.43.1,<2.0",
"boto3>=1.43.117,<2.0",
"certifi>=2026.10.19",
"typing-extensions>=4.13.0,<5.0",
]
@ -72,7 +73,7 @@ proxy = [
"cryptography>=49.0.0,<51.0",
"pynacl>=1.6.2,<2.0",
"websockets>=15.0.1,<16.0",
"boto3>=1.43.1,<2.0",
"boto3>=1.43.117,<2.0",
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=2.3.0,<3",
@ -136,10 +137,10 @@ semantic-router = [
"semantic-router>=0.1.15,<1.0; python_version < '3.14'",
"aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'",
]
mlflow = ["mlflow>=3.11.1,<4.0"]
mlflow = ["mlflow>=3.17.2,<4.0"]
grpc = [
# Newest non-yanked release older than the 30-day cutoff.
"grpcio==1.78.0",
# First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff
"grpcio==1.84.1",
]
stt-vertex-chirp = [
# Google Cloud Speech-to-Text v2 streaming (gRPC) for Chirp models on
@ -173,19 +174,19 @@ proxy-runtime = [
"google-cloud-speech>=2.40.0,<3.0",
"google-genai>=1.37.0,<2.0",
"anthropic[vertex]>=0.84.0,<1.0",
"grpcio==1.78.0",
"grpcio==1.84.1",
"prometheus-client>=0.20.0,<1.0",
"langfuse>=4.7,<5.0",
"opentelemetry-api==1.33.1",
"opentelemetry-sdk==1.33.1",
"opentelemetry-exporter-otlp==1.33.1",
"opentelemetry-instrumentation-fastapi==0.54b1",
"opentelemetry-api==1.45.1",
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"ddtrace>=4.8.2,<5.0",
"sentry-sdk>=2.21.0,<3.0",
"mangum>=0.17.0,<1.0",
"azure-ai-contentsafety>=1.0.0,<2.0",
"azure-storage-file-datalake>=12.20.0,<13.0",
"pypdf>=6.16.1,<7.0",
"pypdf>=6.21.2,<7.0",
"llm-sandbox>=0.3.39,<1.0",
"detect-secrets>=1.5.0,<2.0",
]
@ -222,10 +223,10 @@ dev = [
"types-PyYAML==6.0.12.20250915",
"botocore-stubs==1.43.14",
"types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30",
"opentelemetry-api==1.33.1",
"opentelemetry-sdk==1.33.1",
"opentelemetry-exporter-otlp==1.33.1",
"opentelemetry-instrumentation-fastapi==0.54b1",
"opentelemetry-api==1.45.1",
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"langfuse>=4.7,<5.0",
"fastapi-offline==1.7.6",
"fakeredis==2.34.1",
@ -254,10 +255,10 @@ proxy-dev = [
"hypercorn==0.17.3",
"prometheus-client==0.20.0",
"sentry-sdk==2.21.0",
"opentelemetry-api==1.33.1",
"opentelemetry-sdk==1.33.1",
"opentelemetry-exporter-otlp==1.33.1",
"opentelemetry-instrumentation-fastapi==0.54b1",
"opentelemetry-api==1.45.1",
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"azure-identity==1.25.2",
"a2a-sdk==1.1.0",
"backports-zstd==1.8.0; python_version < '3.14'",
@ -279,7 +280,7 @@ ci = [
"langchain==1.3.9",
"lunary==1.4.36; python_version == '3.10'",
"lunary==1.4.37; python_version >= '3.11'",
"logfire==4.6.0",
"logfire==5.2.0",
"traceloop-sdk==0.34.0",
"detect-secrets==1.5.0",
"PyGithub==2.8.1",
@ -358,7 +359,7 @@ constraint-dependencies = [
"packaging>=24.0",
"soupsieve>=2.8.4",
"httplib2>=0.32.0",
"setuptools>=83.0.0",
"setuptools>=84.1.0",
]
override-dependencies = [
# a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0.
@ -370,6 +371,23 @@ default-groups = ["dev"]
required-version = ">=0.10.9"
exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window
certifi = "2026-10-20T03:00:00Z"
# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window
grpcio = "2026-10-22T15:00:00Z"
# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window
boto3 = "2026-10-21T19:31:00Z"
botocore = "2026-10-21T19:31:00Z"
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window
mlflow = "2026-10-21T03:00:00Z"
mlflow-skinny = "2026-10-21T03:00:00Z"
mlflow-tracing = "2026-10-21T03:00:00Z"
# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window
pypdf = "2026-10-21T17:00:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }
litellm-enterprise = { workspace = true }

View file

@ -176,6 +176,7 @@ langgraph: >=1.0.10 # MIT License
langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE
hypothesis: >=6.165.10 # MPL 2.0 license
pytest-rerunfailures: >=15.1 # MPL 2.0 license
certifi: >=2026.10.19 # MPL 2.0 license - https://github.com/certifi/python-certifi/blob/master/LICENSE
pytest-recording: >=0.13.4 # MIT license
expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE
typing-extensions: >=4.13.0 # PSF-2.0 license - https://github.com/python/typing_extensions/blob/main/LICENSE

View file

@ -1,125 +0,0 @@
"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check
the daily activity routes answer 400 with, instead of a generic fetch error."""
from __future__ import annotations
import json
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Final
import pytest
from integration._support.client import JSON_OBJECT, Gateway, object_value
from integration._support.upstream import delete_scenario, register_scenario
from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway
from pydantic import JsonValue
pytestmark: Final = pytest.mark.timeout(180)
_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data")
@dataclass(frozen=True, slots=True)
class _UsageChat:
gateway: Gateway
identity: str
@pytest.fixture(scope="module")
def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]:
"""A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream."""
identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}"
handle: Final = register_scenario(identity, json_response({}))
try:
with owned_gateway(
tmp_path_factory.mktemp("audit-usage-chat"),
{"model_list": [], "router_settings": {"num_retries": 0}},
environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity},
) as owned:
yield _UsageChat(owned, identity)
finally:
delete_scenario(handle)
def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]:
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
return {
"id": "chatcmpl-usage",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}}
for tool in _TOOLS
],
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]:
"""Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up
model call carried, keyed by tool, and the tool statuses the SSE stream reported."""
register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date)))
Upstream(usage_chat.gateway.upstream_url).reset()
with usage_chat.gateway.client.stream(
"POST",
"/usage/ai/chat",
json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]},
headers={"Authorization": f"Bearer {usage_chat.gateway.key}"},
timeout=60,
) as response:
assert response.status_code == 200
events: Final = tuple(
object_value(JSON_OBJECT.validate_json(line.removeprefix("data: ")))
for line in response.iter_lines()
if line.startswith("data: ")
)
calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity)
assert len(calls) == 2, calls
messages: Final = object_value(calls[1]["body"]).get("messages")
assert isinstance(messages, list), calls[1]
tool_messages: Final = {
str(message["tool_call_id"]).removeprefix("call-"): str(message["content"])
for message in messages
if isinstance(message, dict) and message.get("role") == "tool"
}
statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call")
return tool_messages, statuses
_REVERSED: Final = "Invalid date range: end_date must be on or after start_date"
_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates"
_NO_TEAM_DATA: Final = "No Team usage data found for the given date range."
_NO_TAG_DATA: Final = "No Tag usage data found for the given date range."
@pytest.mark.parametrize(
("start_date", "end_date", "team_message", "tag_message"),
(
pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"),
pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"),
pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"),
pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"),
),
)
def test_usage_chat_tools_answer_with_the_query_layers_date_check(
usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str
) -> None:
"""The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query
takes these dates without that check, still answers with its usage summary."""
tool_messages, statuses = _tool_results(usage_chat, start_date, end_date)
assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages
assert tool_messages["get_team_usage_data"] == team_message, tool_messages
assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages
assert statuses == ("running", "complete") * len(_TOOLS), statuses

View file

@ -6985,6 +6985,106 @@ GPT_5_6_RESPONSES_CACHE_READ: Final = CostTrackingTestCase(
),
)
GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase(
name="gpt-5.6-responses_cache_write_12h",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
endpoint="/v1/responses",
request={
"model": "$MODEL",
"input": "summarize this text",
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
},
response=JsonResponse(
content_type="application/json",
body={
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
}
],
"usage": {
"input_tokens": 12928,
"output_tokens": 380,
"total_tokens": 13308,
"input_tokens_details": {
"cached_tokens": 0,
"cache_write_tokens": 5000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
},
"output_tokens_details": {"reasoning_tokens": 0},
},
},
),
expected=ExactExpected(
spend=0.0344865,
input_cost=0.0291865,
output_cost=0.00532,
prompt_tokens=12928,
completion_tokens=380,
cache_creation_cost=0.0153125,
),
)
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase(
name="gpt-6-luna-responses_cache_write_12h_no_12h_rate",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-6-luna",
endpoint="/v1/responses",
request={
"model": "$MODEL",
"input": "summarize this text",
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
},
response=JsonResponse(
content_type="application/json",
body={
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-6-luna",
"output": [
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
}
],
"usage": {
"input_tokens": 12928,
"output_tokens": 380,
"total_tokens": 13308,
"input_tokens_details": {
"cached_tokens": 0,
"cache_write_tokens": 5000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
},
"output_tokens_details": {"reasoning_tokens": 0},
},
},
),
expected=ExactExpected(
spend=0.0016078,
input_cost=0.0014178,
output_cost=0.00019,
prompt_tokens=12928,
completion_tokens=380,
cache_creation_cost=0.000625,
),
)
GPT_5_6_RESPONSES_REASONING: Final = CostTrackingTestCase(
name="gpt-5.6-responses_reasoning",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -11913,6 +12013,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
OMNI_MODERATIONS_NEXT_SINGLE,
OMNI_MODERATIONS_NEXT_LIST,
GPT_5_6_RESPONSES_CACHE_READ,
GPT_5_6_RESPONSES_CACHE_WRITE_12H,
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
GPT_5_6_RESPONSES_REASONING,
GPT_5_6_RESPONSES_STREAM,
GPT_5_6_RESPONSES_STREAM_CACHE_READ,

View file

@ -174,6 +174,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
},
},
"gpt-5.6": {
"cache_creation_input_token_cost": 2.1875e-06,
"cache_creation_input_token_cost_above_12hr": 4.375e-06,
"cache_read_input_token_cost": 1.75e-07,
"input_cost_per_audio_token": 4e-05,
"input_cost_per_token": 1.75e-06,

View file

@ -85,6 +85,12 @@ class CostMapEntry(BaseModel):
cache_read_input_token_cost: float | None = None
cache_creation_input_token_cost: float | None = None
cache_creation_input_token_cost_above_1hr: float | None = None
cache_creation_input_token_cost_above_12hr: float | None = None
cache_creation_input_token_cost_above_12hr_flex: float | None = None
cache_creation_input_token_cost_above_12hr_priority: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None
cache_read_input_token_cost_above_200k_tokens: float | None = None
cache_creation_input_token_cost_above_200k_tokens: float | None = None

View file

@ -0,0 +1,94 @@
from datetime import datetime, timezone
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
from tests.integration._support.client import Gateway, Scenario
@pytest.mark.parametrize("descending", (False, True))
def test_budget_reset_sort_orders_durations_before_pagination(
descending: bool,
gateway: Gateway,
scenario: Scenario,
) -> None:
prefix: Final = f"sort-{uuid4().hex}"
durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "5d", "𝟝d", "5d", None)
ids: Final = tuple(
scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10)
for index, duration in enumerate(durations)
)
now: Final = datetime.now(timezone.utc)
expected: Final = tuple(
budget_id
for _, budget_id in sorted(
zip(durations, ids, strict=True),
key=lambda item: (
item[0] is None,
duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0,
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": "-budget_duration" if descending else "budget_duration",
"page_size": "3",
"page": str(page),
},
)
for page in range(1, -(-len(ids) // 3) + 1)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
filtered: Final = gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": "budget_duration",
"filter[budget_duration][in]": "7d,30d",
},
)
assert filtered.status_code == 200, filtered.text
assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"]
@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit"))
@pytest.mark.parametrize("descending", (False, True))
def test_budget_limits_sort_unlimited_as_infinity(
field: str, descending: bool, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-unlimited-{uuid4().hex}"
values: Final = (20, None, 0, 3)
ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values))
expected: Final = tuple(
identity
for value, identity in sorted(
zip(values, ids, strict=True),
key=lambda item: float("inf") if item[0] is None else item[0],
reverse=descending,
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": ("-" if descending else "") + field,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected

View file

@ -1,11 +1,16 @@
from typing import FrozenSet
from datetime import datetime, timedelta, timezone
from itertools import chain
from typing import Final, FrozenSet
import pytest
from litellm.proxy.utils import hash_token
from tests.integration._support.actors import TEAM_ALPHA, Actor
from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources
from tests.integration._support.actors import create_scratch_key
from tests.integration._support.client import Gateway, Scenario
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
pytestmark = pytest.mark.asyncio(loop_scope="session")
@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor
},
)
assert resp.status_code == 403, resp.text
@pytest.mark.parametrize(
"field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit")
)
@pytest.mark.parametrize("direction", ("asc", "desc"))
@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted"))
async def test_key_list_sorts_across_pages_with_nulls_last(
field: str,
direction: str,
deleted: bool,
gateway: Gateway,
scenario: Scenario,
db: ManagementDatabase,
resources: ScratchResources,
) -> None:
team_admin: Final = scenario.user(user_role="internal_user")
owner: Final = scenario.user(user_role="internal_user")
team_id: Final = scenario.team(
members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}]
)
team_admin_key: Final = scenario.key(user_id=team_admin)
table: Final = (
DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table
)
values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None)
tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values)))
rows: Final = [
{
"token": token,
"key_alias": resources.prefix,
"team_id": team_id,
"user_id": owner,
field: (
datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value)
if field in ("expires", "last_active", "budget_reset_at") and value is not None
else value
),
}
for token, value in zip(tokens, values, strict=True)
]
await table.create_many(rows)
try:
valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None)
expected: Final = [
token
for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1]))
] + [token for value, token in zip(values, tokens, strict=True) if value is None]
for caller in (gateway.key, team_admin_key):
pages: Final = [
gateway.client.get(
"/key/list",
headers={"Authorization": f"Bearer {caller}"},
params={
"key_alias": resources.prefix,
"team_id": team_id,
"include_team_keys": "true",
"sort_by": field,
"sort_order": direction,
"size": "2",
"page": str(page),
**({"status": "deleted"} if deleted else {}),
},
)
for page in (1, 2, 3)
]
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total_count"] == len(tokens) for page in pages)
assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected
finally:
await table.delete_many(where={"token": {"in": tokens}})

View file

@ -3,7 +3,9 @@ import contextlib
import json
import uuid
from collections.abc import Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Final
from urllib.parse import parse_qs, urlencode, urlparse
@ -15,7 +17,7 @@ from pydantic import JsonValue
from tests.integration._support.client import Gateway, gateway_from_environment
from tests.integration._support.database import read_rows
from tests.integration._support.process import owned_proxy
from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy
from tests.integration._support.wire import Reply, Request, wire_server
pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role(
"metadata": {"auth_provider": IDP_PROVIDER},
}
]
def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value)
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
with _sso_gateway(tmp_path, {}) as gateway:
for index in range(4):
signed_in: Final = _sign_in(gateway, f"free-sso-{index}")
assert signed_in.status_code == 303, signed_in.text
contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8))
with ThreadPoolExecutor(max_workers=len(contenders)) as pool:
responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders))
assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403]
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}]
for subject, response in zip(contenders, responses):
if response.status_code == 403:
assert "requires a LiteLLM Enterprise license" in response.text
assert not response.cookies
assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == []
existing: Final = _sign_in(gateway, "free-sso-0")
assert existing.status_code == 303, existing.text
admin_created: Final = gateway.request(
"POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False}
)
assert admin_created.status_code == 200, admin_created.text
above_capacity: Final = _sign_in(gateway, "free-sso-0")
refused: Final = _sign_in(gateway, "free-sso-extra")
assert (above_capacity.status_code, refused.status_code) == (303, 403)
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}]

View file

@ -0,0 +1,244 @@
import json
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from tests.integration._support.client import Gateway, Scenario
from tests.integration._support.database import write_rows
@pytest.mark.parametrize(
"field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count")
)
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_team_list_sorts_across_pages_and_respects_membership(
field: str,
direction: str,
gateway: Gateway,
scenario: Scenario,
) -> None:
prefix: Final = f"sort-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
caller: Final = scenario.key(user_id=owner)
values: Final = (20, 3, 3, 100, 0)
teams: Final = tuple(
scenario.team(
team_id=f"{prefix}-{index}",
team_alias=f"{prefix}-{index}",
members_with_roles=[{"user_id": owner, "role": "admin"}],
)
for index in range(len(values))
)
outsider: Final = scenario.team(team_alias=f"{prefix}-outside")
for team_id, value in zip(teams, values, strict=True):
write_rows(
'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, '
"tpm_limit = %s::bigint, rpm_limit = %s::bigint, "
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', "
"members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) "
"WHERE team_id = %s",
(
str(value),
str(value),
str(value),
str(value),
str(value),
json.dumps(
[{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)]
),
str(value),
team_id,
),
)
write_rows(
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
(team_id, team_id, str(value)),
)
write_rows(
'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s',
(teams[-1],),
)
try:
expected: Final = tuple(
team_id
for value, team_id in sorted(
zip(values, teams, strict=True),
key=lambda item: (
(item[0] != 0 if direction == "desc" else item[0] == 0)
if field in ("max_budget", "tpm_limit", "rpm_limit")
else False,
item[0] * (1 if direction == "asc" else -1),
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"search": prefix,
"user_id": owner,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2, 3)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages))
assert returned == expected
assert outsider not in returned
assert all(page.json()["total"] == len(teams) for page in pages)
finally:
for team_id in teams:
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_team_list_sorts_organization_names_and_missing_values(
direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-{uuid4().hex}"
budget: Final = scenario.budget(max_budget=10)
organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", ""))
teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4))
try:
for (organization_id, alias), team_id in zip(organizations, teams):
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(organization_id, alias, budget),
)
write_rows(
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s',
(organization_id, team_id),
)
ascending: Final = (teams[0], teams[2], teams[1])
ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending))
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
params={
"search": prefix,
"sort_by": "organization_alias",
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1])
finally:
for team_id in teams:
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,))
for organization_id, _ in organizations:
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,))
@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_computed_team_sorts_preserve_org_and_membership_union(
field: str, direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-scope-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
budget: Final = scenario.budget(max_budget=10)
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
teams: Final = tuple(
scenario.team(
team_id=f"{prefix}-{i}",
team_alias=f"{prefix}-{i}",
members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [],
)
for i in range(4)
)
try:
for org, alias in zip(orgs, ("alpha", "zulu"), strict=True):
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(org, alias, budget),
)
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'org_admin')",
(owner, orgs[0]),
)
for index, team in enumerate(teams):
write_rows(
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, '
"models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s",
(
orgs[0] if index < 2 else orgs[1],
json.dumps(
[
{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"}
for i in range(index + 1)
]
),
str(index + 1),
team,
),
)
write_rows(
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
(team, team, str(index + 1)),
)
caller: Final = scenario.key(user_id=owner)
values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3)
expected: Final = tuple(
team
for value, team in sorted(
zip(values, teams[:3], strict=True),
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"user_id": owner,
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == 3 for page in pages)
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected
denied: Final = gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"organization_id": orgs[1],
"sort_by": field,
},
)
assert denied.status_code == 403, denied.text
finally:
for team in teams:
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,))
for org in orgs:
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))

View file

@ -0,0 +1,179 @@
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from tests.integration._support.client import Gateway, Scenario
from tests.integration._support.database import write_rows
@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None:
prefix: Final = f"sort-{uuid4().hex}"
ids: Final = tuple(
scenario.user(
user_id=f"{prefix}-{index}",
user_role="internal_user",
user_alias=f"Alias {value if index != 2 else 4:03}",
user_email=f"{prefix}-{index}@example.com",
max_budget=value,
metadata={"scim_active": value != 3},
)
for index, value in enumerate((20, 3, 3, 100, 0))
)
for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True):
write_rows(
'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, '
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s",
(f"{prefix}-{value:03}", str(value), user_id),
) if value != 3 or user_id == ids[1] else write_rows(
"UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s",
(f"{prefix}-004", user_id),
)
for index in range(value if value < 4 else 1):
scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}")
write_rows(
'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s',
(ids[-1],),
)
scenario.key(user_id=ids[-1])
write_rows(
'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1])
)
values: Final = (
(0, 1, 1, 0, 0)
if field == "status"
else (1, 3, 3, 1, 0)
if field == "key_count"
else (20, 3, 4, 100, None)
if field == "sso_user_id"
else (20, 3, 4, 100, 0)
if field == "user_alias"
else (20, 3, 3, 100, None)
)
expected: Final = tuple(
user_id
for _, user_id in sorted(
((value, user_id) for value, user_id in zip(values, ids, strict=True)),
key=lambda item: (
(item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None),
(item[0] or 0) * (1 if direction == "asc" else -1),
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/user/list",
params={
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2, 3)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == len(ids) for page in pages)
assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected
@pytest.mark.parametrize("field", ("status", "key_count"))
def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None:
team: Final = scenario.team()
member: Final = scenario.member(team)
other: Final = scenario.user(user_role="internal_user")
for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}):
baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"})
response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field})
assert response["total"] == baseline["total"]
assert sorted(user["user_id"] for user in response["users"]) == sorted(
user["user_id"] for user in baseline["users"]
)
ordinary_key: Final = scenario.key(user_id=other)
denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field})
assert denied.status_code == 403, denied.text
@pytest.mark.parametrize("field", ("status", "key_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_computed_user_sorts_preserve_org_admin_scope(
field: str, direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-users-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
members: Final = tuple(
scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1})
for index in range(4)
)
budget: Final = scenario.budget(max_budget=10)
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
try:
for org in orgs:
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(org, org, budget),
)
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'org_admin')",
(owner, orgs[0]),
)
for index, member in enumerate(members):
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'internal_user')",
(member, orgs[0] if index < 3 else orgs[1]),
)
for _ in range(index):
scenario.key(user_id=member)
caller: Final = scenario.key(user_id=owner)
values: Final = (0, 1, 0) if field == "status" else (0, 1, 2)
expected: Final = tuple(
member
for value, member in sorted(
zip(values, members[:3], strict=True),
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/user/list",
key=caller,
params={
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == 3 for page in pages)
returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages))
assert returned == expected
denied: Final = gateway.request(
"GET",
"/user/list",
key=caller,
params={
"organization_ids": orgs[1],
"sort_by": field,
},
)
assert denied.status_code == 403, denied.text
finally:
for org in orgs:
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))

View file

@ -0,0 +1,180 @@
from __future__ import annotations
import json
import uuid
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import Gateway, object_value
from integration._support.forward_proxy import refusing_forward_proxy
from integration._support.process import owned_proxy
from pydantic import JsonValue
_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm"
_CONTROL_HOST: Final = "provider.offline-mode.example"
_REMOTE_URLS: Final = {
"LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json",
"LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json",
"LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json",
"LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml",
}
_SELF_INITIATED_HOSTS: Final = frozenset(
{
"cost-map.offline-mode.example:443",
"beta-headers.offline-mode.example:443",
"autorouter.offline-mode.example:443",
"blog.offline-mode.example:443",
"raw.githubusercontent.com:443",
"docs.litellm.ai:443",
"api.pwnedpasswords.com:443",
"huggingface.co:443",
}
)
_PER_FEATURE_FLAGS: Final = (
"LITELLM_OFFLINE_MODE",
"LITELLM_LOCAL_MODEL_COST_MAP",
"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS",
"LITELLM_LOCAL_AUTOROUTER_PRESETS",
"LITELLM_LOCAL_POLICY_TEMPLATES",
"LITELLM_LOCAL_BLOG_POSTS",
"LITELLM_DISABLE_LATEST_RELEASE_CHECK",
"LITELLM_LICENSE",
)
def _bundled(name: str) -> JsonValue:
return json.loads((_PACKAGE / name).read_text(encoding="utf-8"))
def _write_config(directory: Path) -> Path:
config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": "offline-mode-control",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_base": f"https://{_CONTROL_HOST}/v1",
"api_key": "synthetic-provider-key",
},
},
{
"model_name": "offline-mode-kimi",
"litellm_params": {
"model": "bedrock/us.moonshotai.kimi-k3",
"aws_region_name": "us-east-1",
"aws_access_key_id": "synthetic-access-key",
"aws_secret_access_key": "synthetic-secret-key",
},
},
],
"router_settings": {"num_retries": 0},
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
}
)
)
return config
def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue:
response: Final = proxy.request(method, path, body)
assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}"
return json.loads(response.content)
@pytest.mark.timeout(300)
@pytest.mark.parametrize(
("offline_mode", "expected_hosts", "reload_status"),
[
pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"),
pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"),
],
)
def test_offline_mode_stops_every_self_initiated_request(
gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int
) -> None:
with refusing_forward_proxy() as egress:
environment: Final = {
"HTTPS_PROXY": egress.url,
"HTTP_PROXY": egress.url,
"ALL_PROXY": egress.url,
"NO_PROXY": "127.0.0.1,localhost",
"AIOHTTP_TRUST_ENV": "True",
**_REMOTE_URLS,
**({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}),
}
with owned_proxy(
gateway,
tmp_path,
environment,
config=_write_config(tmp_path),
remove_environment=_PER_FEATURE_FLAGS,
workers=2,
) as isolated:
source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source"))
reloaded: Final = isolated.request("POST", "/reload/model_cost_map")
beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers"))
release: Final = _json(isolated, "GET", "/get/latest_release_info")
presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets"))
templates: Final = _json(isolated, "GET", "/policy/templates")
blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts"))
user: Final = object_value(
_json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"})
)
password_set: Final = object_value(
_json(
isolated,
"POST",
"/user/update",
{"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"},
)
)
tokens: Final = object_value(
_json(
isolated,
"POST",
"/utils/token_counter",
{"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]},
)
)
kimi_tokens: Final = object_value(
_json(
isolated,
"POST",
"/utils/token_counter",
{"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]},
)
)
control: Final = isolated.request(
"POST",
"/v1/chat/completions",
{"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]},
)
reached: Final = frozenset(egress.targets())
assert reached == expected_hosts
assert control.status_code != 200, control.text
assert source["source"] == "local", source
assert source["is_env_forced"] is (offline_mode is not None), source
assert reloaded.status_code == reload_status, reloaded.text
assert beta_headers["status"] == "success", beta_headers
assert release is None
assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json")))
assert templates == _bundled("policy_templates_backup.json")
assert [object_value(post)["url"] for post in _list(blog["posts"])] == [
object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"])
]
assert password_set["user_id"] == user["user_id"], password_set
assert tokens["tokenizer_type"] == "openai_tokenizer", tokens
assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens
def _list(value: JsonValue) -> list[JsonValue]:
assert isinstance(value, list), value
return value

Some files were not shown because too many files have changed in this diff Show more