mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
chore: merge upstream main into prometheus budget metrics
Signed-off-by: amarrtech <272048731+amarrtech@users.noreply.github.com>
This commit is contained in:
commit
c2b842d8e2
345 changed files with 31781 additions and 6585 deletions
4
.github/CODEOWNERS
vendored
4
.github/CODEOWNERS
vendored
|
|
@ -1,4 +1,4 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
|
|
|
|||
91
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
91
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
name: Interactions spec canary
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 7 * * *"
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/interactions-spec-canary.yml
|
||||
- tests/unit/interactions/**
|
||||
- litellm/types/interactions/**
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency: interactions-spec-canary
|
||||
|
||||
jobs:
|
||||
interactions-spec-canary:
|
||||
name: interactions-spec-canary
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
compliance: ${{ steps.compliance.outcome }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
|
||||
|
||||
- name: Run the compliance tests against Google's live spec
|
||||
id: compliance
|
||||
run: |
|
||||
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
||||
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
||||
|
||||
tracking-issue:
|
||||
name: tracking-issue
|
||||
needs: interactions-spec-canary
|
||||
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: File the tracking issue unless the compliance step passed, close it when it did
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
with:
|
||||
script: |
|
||||
const title = "Interactions spec canary is red";
|
||||
const outcome = process.env.COMPLIANCE_OUTCOME;
|
||||
const runUrl = process.env.RUN_URL;
|
||||
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
|
||||
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
|
||||
const open = found.items.find((issue) => issue.title === title);
|
||||
if (outcome === "success") {
|
||||
if (open === undefined) return;
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
|
||||
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
|
||||
return;
|
||||
}
|
||||
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
|
||||
if (open !== undefined) {
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.create({
|
||||
...context.repo,
|
||||
title,
|
||||
assignees: ["mateo-berri"],
|
||||
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
|
||||
});
|
||||
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -425,6 +425,7 @@ jobs:
|
|||
tests/unit/realtime_api
|
||||
tests/unit/rerank_api
|
||||
tests/unit/search
|
||||
tests/unit/text_completions
|
||||
tests/unit/vector_stores
|
||||
tests/unit/videos
|
||||
python-version: "3.12"
|
||||
|
|
|
|||
11
Dockerfile
11
Dockerfile
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -61,6 +61,7 @@ RUN apk add --no-cache \
|
|||
python-3.13-dev \
|
||||
rust \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
|
||||
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
|
||||
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
|
||||
uv venv /opt/liteadmin --python python3.13 && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
|
||||
|
|
@ -141,7 +142,7 @@ USER root
|
|||
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||
|
|
@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
|
|||
|
||||
### Verify Docker Image Signatures
|
||||
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
|
||||
|
||||
**Verify using the pinned commit hash (recommended):**
|
||||
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
|
||||
|
||||
```bash
|
||||
cosign verify \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
|
||||
ghcr.io/berriai/litellm:<release-tag>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
|
||||
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
|
||||
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
|
||||
-----END PUBLIC KEY-----
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -59,6 +59,7 @@ RUN apk add --no-cache \
|
|||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
|||
USER root
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
|
|||
coreutils \
|
||||
curl \
|
||||
openssl \
|
||||
libssl3 \
|
||||
libsndfile \
|
||||
nodejs \
|
||||
npm && break || sleep 5; \
|
||||
|
|
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
|
|||
apk upgrade --no-cache && break || sleep 5; \
|
||||
done && \
|
||||
for i in 1 2 3; do \
|
||||
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
|
||||
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
|
||||
done
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ RUN uv venv --python python && \
|
|||
"google-cloud-aiplatform==1.133.0" \
|
||||
"google-genai==1.37.0" \
|
||||
"anthropic[vertex]==0.84.0" \
|
||||
"grpcio==1.78.0" \
|
||||
"grpcio==1.84.1" \
|
||||
"prometheus-client==0.20.0" \
|
||||
"langfuse==2.59.7" \
|
||||
"opentelemetry-api==1.28.0" \
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
|
||||
ARG PGBOUNCER_VERSION=1.26.0
|
||||
|
|
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
|
|||
8
litellm-rust/Cargo.lock
generated
8
litellm-rust/Cargo.lock
generated
|
|
@ -8202,18 +8202,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
|
||||
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
|
||||
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
|
|||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
url = "2.5.8"
|
||||
percent-encoding = "2.3"
|
||||
webpki-roots = "1"
|
||||
webpki-roots = "1.0.10"
|
||||
time = { version = "0.3.53", features = ["parsing"] }
|
||||
criterion = "0.8.2"
|
||||
fancy-regex = "0.19.2"
|
||||
|
|
|
|||
|
|
@ -235,7 +235,10 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
|
||||
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
|
||||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -272,7 +275,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@
|
|||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"thinking-binding-controls-2026-08-01": null,
|
||||
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
# Cache for the loaded configuration
|
||||
_BETA_HEADERS_CONFIG: dict | None = None
|
||||
|
|
@ -145,7 +146,7 @@ def get_beta_headers_config(url: str) -> dict:
|
|||
Dict containing the beta headers configuration
|
||||
"""
|
||||
# Check if local-only mode is enabled
|
||||
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
|
||||
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
|
||||
|
||||
|
|
|
|||
|
|
@ -2264,6 +2264,8 @@ def response_cost_calculator(
|
|||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2292,6 +2294,8 @@ def response_cost_calculator(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
|
|
@ -2875,6 +2879,7 @@ def _batch_rate(
|
|||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
|
|
@ -3008,6 +3013,7 @@ def _batch_prompt_cost(
|
|||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
details["audio_tokens"],
|
||||
|
|
@ -3022,7 +3028,9 @@ def _batch_prompt_cost(
|
|||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ cached_tokens * (cache_read_rate or 0.0)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
|
||||
+ written_tokens * (cache_creation_rate or 0.0)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
|
||||
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
|
||||
is unavailable. This MUST be called at app-creation time — once the lifespan
|
||||
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
|
||||
middleware after an application has started".
|
||||
runs, the middleware stack is frozen and middleware added later never runs, so
|
||||
a late call is skipped without registering anything.
|
||||
|
||||
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
|
||||
``ProxyTracerProvider``; the proxy publishes the real provider as the global
|
||||
|
|
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
try:
|
||||
if not is_otel_v2_enabled():
|
||||
return
|
||||
if app.middleware_stack is not None:
|
||||
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
|
||||
return
|
||||
|
||||
# Lazy: only the V2-enabled path needs the optional
|
||||
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
|
|||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
|
@ -13,9 +14,13 @@ from typing import Final, TypeAlias
|
|||
import requests
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
|
||||
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
JSON_CONTENT_TYPE: Final = "application/json"
|
||||
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
|
||||
|
||||
|
|
@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
|
|||
def __init__(
|
||||
self,
|
||||
endpoint: str | None,
|
||||
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
|
||||
headers: Mapping[str, str],
|
||||
certificate_file: str | None = None,
|
||||
session: "requests.Session | None" = None,
|
||||
*,
|
||||
max_request_size: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
|
||||
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
|
||||
super().__init__(
|
||||
endpoint=endpoint,
|
||||
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
|
||||
certificate_file=certificate_file,
|
||||
session=session,
|
||||
max_request_size=max_request_size,
|
||||
)
|
||||
|
||||
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
|
||||
return encode_spans_json(spans)
|
||||
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
||||
if self._shutdown:
|
||||
_logger.warning("Exporter already shutdown, ignoring batch")
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
with self._metrics.export_operation(len(spans)) as result:
|
||||
try:
|
||||
serialized_data: Final = encode_spans_json(spans)
|
||||
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
|
||||
_logger.error("Failed to encode span batch: %s", error)
|
||||
result.error = error
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
if 0 < self._max_request_size < len(serialized_data):
|
||||
result.error = RequestPayloadTooLargeError(
|
||||
f"Serialized span request size {len(serialized_data)} bytes "
|
||||
f"exceeds max_request_size {self._max_request_size} bytes."
|
||||
)
|
||||
_logger.warning("Dropping span batch: %s", result.error)
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
export_result: Final = self._client.export(serialized_data)
|
||||
if not export_result.success:
|
||||
result.error = export_result.error
|
||||
result.error_attrs = (
|
||||
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
|
||||
if export_result.status_code is not None
|
||||
else None
|
||||
)
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
return SpanExportResult.SUCCESS
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ def agentops_preset(
|
|||
|
||||
def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
|
||||
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
|
||||
import requests
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||
OTLPSpanExporter,
|
||||
)
|
||||
|
|
@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
|
|||
"""
|
||||
|
||||
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
|
||||
super().__init__(endpoint=endpoint)
|
||||
self._agentops_session = requests.Session()
|
||||
super().__init__(endpoint=endpoint, session=self._agentops_session)
|
||||
self._agentops_api_key = api_key
|
||||
self._auth_resolved = False
|
||||
|
||||
|
|
@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
|
|||
try:
|
||||
token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
|
||||
if token:
|
||||
# ``_session`` is the requests.Session the base exporter
|
||||
# POSTs through; updating its Authorization header is how the
|
||||
# minted JWT reaches every subsequent export.
|
||||
self._session.headers["Authorization"] = f"Bearer {token}"
|
||||
self._agentops_session.headers["Authorization"] = f"Bearer {token}"
|
||||
except Exception as e:
|
||||
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ class ExceptionCheckers:
|
|||
"`inputs` tokens + `max_new_tokens` must be",
|
||||
"exceeds the available context size", # llama.cpp/Lemonade
|
||||
"exceeds the maximum number of tokens allowed", # Gemini
|
||||
'"code": "context_length_exceeded"',
|
||||
]
|
||||
for substring in known_exception_substrings:
|
||||
if substring in _error_str_lowercase:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import httpx
|
|||
from pydantic import ConfigDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
|
||||
|
|
@ -129,7 +130,7 @@ class GetBlogPosts:
|
|||
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
|
||||
Fetches from ``url`` otherwise, falling back to local backup on failure.
|
||||
"""
|
||||
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
|
||||
return cls.load_local_blog_posts()
|
||||
|
||||
now: Final = time.time()
|
||||
|
|
|
|||
|
|
@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info(
|
|||
if dynamic_api_key is None and api_key is not None:
|
||||
dynamic_api_key = api_key
|
||||
return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
|
||||
|
||||
def get_provider_affinity_header(api_base: str | None) -> str | None:
|
||||
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
|
||||
|
||||
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.constants import (
|
|||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
|
|
@ -456,7 +457,7 @@ async def refetch_model_cost_map(
|
|||
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
|
||||
map they already have.
|
||||
"""
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
|
|
@ -740,7 +741,7 @@ def get_model_cost_map(
|
|||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
||||
# before litellm._key_management_settings is set.
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
|
|
|
|||
|
|
@ -161,6 +161,7 @@ from litellm.types.utils import (
|
|||
CachingDetails,
|
||||
CallTypes,
|
||||
CostBreakdown,
|
||||
CostPerToken,
|
||||
CustomPricingLiteLLMParams,
|
||||
DynamicPromptManagementParamLiteral,
|
||||
EmbeddingResponse,
|
||||
|
|
@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prometheusLogger, \
|
||||
slack_app
|
||||
custom_pricing: bool = False
|
||||
custom_cost_per_token: CostPerToken | None = None
|
||||
custom_cost_per_second: float | None = None
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
|
|
@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"call_type": self.call_type,
|
||||
"optional_params": self.optional_params,
|
||||
"custom_pricing": custom_pricing,
|
||||
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
|
||||
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
|
||||
"prompt": prompt,
|
||||
"standard_built_in_tools_params": self.standard_built_in_tools_params,
|
||||
"router_model_id": router_model_id,
|
||||
|
|
|
|||
|
|
@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
|
|||
reservation logic so neither has to depend on the other.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, SupportsFloat
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
_TIER_RANGE: Final = TypeAdapter(tuple[float, float])
|
||||
|
||||
|
||||
def _coerce_cost_per_token(value: float | str | None) -> float:
|
||||
def _coerce_cost_per_token(value: object) -> float:
|
||||
"""
|
||||
Coerce a per-token cost into a float.
|
||||
|
||||
Model cost values loaded from YAML config may arrive as strings (e.g.
|
||||
scientific notation like "4e-07"), which would break arithmetic.
|
||||
"""
|
||||
if value is None:
|
||||
return 0.0
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
return float(value)
|
||||
if isinstance(value, SupportsFloat):
|
||||
return float(value)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None:
|
||||
try:
|
||||
return _TIER_RANGE.validate_python(tier.get("range"))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def select_tier_for_input(
|
||||
tiered_pricing: list[dict],
|
||||
tiered_pricing: Sequence[Mapping[str, object]],
|
||||
input_tokens: int,
|
||||
) -> dict | None:
|
||||
) -> Mapping[str, object] | None:
|
||||
"""
|
||||
Select the pricing tier for a request based on its total input token count.
|
||||
|
||||
|
|
@ -44,19 +56,21 @@ def select_tier_for_input(
|
|||
if not tiered_pricing or input_tokens <= 0:
|
||||
return None
|
||||
|
||||
sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
|
||||
valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
|
||||
if not valid_tiers:
|
||||
ranged_tiers: Final = sorted(
|
||||
((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None),
|
||||
key=lambda ranged: ranged[0][0],
|
||||
)
|
||||
if not ranged_tiers:
|
||||
return None
|
||||
|
||||
matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
|
||||
matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end)
|
||||
if matching:
|
||||
return matching[0]
|
||||
return valid_tiers[-1]
|
||||
return ranged_tiers[-1][1]
|
||||
|
||||
|
||||
def tier_rate(
|
||||
tier: dict,
|
||||
tier: Mapping[str, object],
|
||||
cost_key: str,
|
||||
fallback_cost_key: str | None = None,
|
||||
) -> float:
|
||||
|
|
@ -68,4 +82,6 @@ def tier_rate(
|
|||
primary: Final = tier.get(cost_key)
|
||||
if primary is not None:
|
||||
return _coerce_cost_per_token(primary)
|
||||
if fallback_cost_key is None:
|
||||
return 0.0
|
||||
return _coerce_cost_per_token(tier.get(fallback_cost_key, 0))
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timezone, tzinfo
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypedDict, cast
|
||||
|
|
@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
|
|||
|
||||
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
|
||||
|
||||
_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost"
|
||||
|
||||
_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"5m": _BASE_CACHE_WRITE_RATE_KEY,
|
||||
"30m": _BASE_CACHE_WRITE_RATE_KEY,
|
||||
"1hr": "cache_creation_input_token_cost_above_1hr",
|
||||
"12hr": "cache_creation_input_token_cost_above_12hr",
|
||||
}
|
||||
)
|
||||
|
||||
_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY}
|
||||
)
|
||||
|
||||
|
||||
_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
|
|
@ -356,7 +372,21 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
|
|||
)
|
||||
|
||||
|
||||
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenRates:
|
||||
input_rate: float
|
||||
output_rate: float
|
||||
cache_read_rate: float
|
||||
cache_creation_rate: float
|
||||
reasoning_rate: float | None
|
||||
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
@property
|
||||
def billed_reasoning_rate(self) -> float:
|
||||
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
|
||||
|
||||
|
||||
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None:
|
||||
tiered_pricing: Final = model_info.get("tiered_pricing")
|
||||
if not isinstance(tiered_pricing, list) or not tiered_pricing:
|
||||
return None
|
||||
|
|
@ -376,7 +406,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N
|
|||
return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
|
||||
|
||||
|
||||
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None:
|
||||
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None:
|
||||
"""
|
||||
Resolve the base rates from a model's ``tiered_pricing`` table, if it has one.
|
||||
|
||||
|
|
@ -391,22 +421,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float,
|
|||
if tier is None:
|
||||
return None
|
||||
|
||||
cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
|
||||
completion_cost: Final = (
|
||||
tier_rate(tier, "output_cost_per_token")
|
||||
if "output_cost_per_token" in tier
|
||||
else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0
|
||||
)
|
||||
return (
|
||||
tier_rate(tier, "input_cost_per_token"),
|
||||
completion_cost,
|
||||
cache_creation_cost,
|
||||
tier_rate(tier, "cache_creation_input_token_cost_above_1hr")
|
||||
if "cache_creation_input_token_cost_above_1hr" in tier
|
||||
and tier["cache_creation_input_token_cost_above_1hr"] is not None
|
||||
else cache_creation_cost,
|
||||
tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
|
||||
cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
|
||||
rates: Final = TokenRates(
|
||||
input_rate=tier_rate(tier, "input_cost_per_token"),
|
||||
output_rate=completion_cost,
|
||||
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
|
||||
cache_creation_rate=cache_creation_rate,
|
||||
reasoning_rate=None,
|
||||
)
|
||||
return rates, tier
|
||||
|
||||
|
||||
def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool:
|
||||
|
|
@ -532,19 +560,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
|
|||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenRates:
|
||||
input_rate: float
|
||||
output_rate: float
|
||||
cache_read_rate: float
|
||||
cache_creation_rate: float
|
||||
reasoning_rate: float | None
|
||||
|
||||
@property
|
||||
def billed_reasoning_rate(self) -> float:
|
||||
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
|
||||
|
||||
|
||||
def resolve_explicit_cache_read_cost_rate(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
|
|
@ -649,7 +664,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
|
|||
if off_peak is None:
|
||||
return rates
|
||||
off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
|
||||
return TokenRates(
|
||||
return replace(
|
||||
rates,
|
||||
input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
|
||||
output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
|
||||
cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
|
||||
|
|
@ -661,32 +677,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
|
|||
def _apply_off_peak_to_base_costs(
|
||||
model_info: ModelInfo,
|
||||
current_time: datetime | None,
|
||||
base_costs: tuple[float, float, float, float | None, float],
|
||||
) -> tuple[float, float, float, float, float]:
|
||||
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
|
||||
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
|
||||
present one passes through untouched and an absent one resolves to the applied
|
||||
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
|
||||
base_rates: TokenRates,
|
||||
tier: Mapping[str, object],
|
||||
) -> TokenRates:
|
||||
"""Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no
|
||||
field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier
|
||||
that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to
|
||||
_resolve_billed_reasoning_rate.
|
||||
"""
|
||||
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
|
||||
rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
TokenRates(
|
||||
input_rate=prompt,
|
||||
output_rate=completion,
|
||||
cache_read_rate=cache_read,
|
||||
cache_creation_rate=cache_creation,
|
||||
reasoning_rate=None,
|
||||
rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates)
|
||||
return replace(
|
||||
rates,
|
||||
cache_creation_rates_by_ttl=MappingProxyType(
|
||||
{
|
||||
ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key)
|
||||
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
|
||||
}
|
||||
),
|
||||
)
|
||||
return (
|
||||
rates.input_rate,
|
||||
rates.output_rate,
|
||||
rates.cache_creation_rate,
|
||||
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
|
||||
rates.cache_read_rate,
|
||||
)
|
||||
|
||||
|
||||
def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str:
|
||||
return preferred_key if model_info.get(preferred_key) is not None else fallback_key
|
||||
|
||||
|
||||
def _get_token_base_cost(
|
||||
|
|
@ -696,7 +708,7 @@ def _get_token_base_cost(
|
|||
current_time: datetime | None = None,
|
||||
*,
|
||||
threshold_is_inclusive: bool = False,
|
||||
) -> tuple[float, float, float, float, float]:
|
||||
) -> TokenRates:
|
||||
"""
|
||||
Return prompt cost, completion cost, and cache costs for a given model and usage.
|
||||
|
||||
|
|
@ -709,15 +721,12 @@ def _get_token_base_cost(
|
|||
An absent cache-creation or cache-read rate always resolves to the resolved input
|
||||
rate, the way the tiered table and custom deployment pricing already do, since a
|
||||
provider that publishes no cache price bills cached tokens as ordinary input. An
|
||||
absent 1h write rate resolves to the cache-creation rate, off-peak included. An
|
||||
absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An
|
||||
explicit 0.0 stays a real price for all of them.
|
||||
|
||||
Returns:
|
||||
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
|
||||
"""
|
||||
tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage)
|
||||
if tiered_base_costs is not None:
|
||||
return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs)
|
||||
return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs)
|
||||
|
||||
# Get service tier aware cost keys
|
||||
input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
|
||||
|
|
@ -734,13 +743,13 @@ def _get_token_base_cost(
|
|||
|
||||
# For image generation models that don't have output_cost_per_token,
|
||||
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
|
||||
if completion_base_cost == 0.0 or completion_base_cost is None:
|
||||
if completion_base_cost == 0.0:
|
||||
output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None)
|
||||
if output_image_cost is not None:
|
||||
completion_base_cost = output_image_cost
|
||||
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
|
||||
cache_creation_cost_above_1hr = get_cost_per_unit(
|
||||
model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
|
||||
cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType(
|
||||
{ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()}
|
||||
)
|
||||
cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
|
||||
|
||||
|
|
@ -811,14 +820,6 @@ def _get_token_base_cost(
|
|||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_creation_1hr_tiered_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_read_tiered_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
|
||||
|
|
@ -830,10 +831,19 @@ def _get_token_base_cost(
|
|||
|
||||
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
|
||||
|
||||
cache_creation_cost_above_1hr = get_cost_per_unit(
|
||||
model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
|
||||
cache_creation_keys_by_ttl = MappingProxyType(
|
||||
{
|
||||
ttl: _first_present_key(
|
||||
model_info,
|
||||
get_service_tier_cost_key(
|
||||
f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
),
|
||||
key,
|
||||
)
|
||||
for ttl, key in cache_creation_keys_by_ttl.items()
|
||||
}
|
||||
)
|
||||
|
||||
cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
|
||||
except (IndexError, ValueError):
|
||||
continue
|
||||
|
|
@ -849,16 +859,26 @@ def _get_token_base_cost(
|
|||
resolved_cache_creation_cost: Final = (
|
||||
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
|
||||
)
|
||||
|
||||
return _apply_off_peak_to_base_costs(
|
||||
base_rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
resolved_cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
resolved_cache_read_cost,
|
||||
TokenRates(
|
||||
input_rate=prompt_base_cost,
|
||||
output_rate=completion_base_cost,
|
||||
cache_read_rate=resolved_cache_read_cost,
|
||||
cache_creation_rate=resolved_cache_creation_cost,
|
||||
reasoning_rate=None,
|
||||
),
|
||||
)
|
||||
return replace(
|
||||
base_rates,
|
||||
cache_creation_rates_by_ttl=MappingProxyType(
|
||||
{
|
||||
ttl: base_rates.cache_creation_rate
|
||||
if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None
|
||||
else rate
|
||||
for ttl, key in cache_creation_keys_by_ttl.items()
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -975,25 +995,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M
|
|||
def calculate_cache_writing_cost(
|
||||
cache_creation_tokens: int,
|
||||
cache_creation_token_details: CacheCreationTokenDetails | None,
|
||||
cache_creation_cost_above_1hr: float,
|
||||
cache_creation_cost: float,
|
||||
cache_creation_cost_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
"""
|
||||
Adjust cost of cache creation tokens based on the cache creation token details.
|
||||
Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts.
|
||||
Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate.
|
||||
"""
|
||||
total_cost: float = 0.0
|
||||
if cache_creation_token_details is not None:
|
||||
# get the number of 5m and 1h cache creation tokens
|
||||
cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens
|
||||
cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens
|
||||
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
|
||||
total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0
|
||||
total_cost += (
|
||||
cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0
|
||||
)
|
||||
else:
|
||||
total_cost += cache_creation_tokens * cache_creation_cost
|
||||
return total_cost
|
||||
if cache_creation_token_details is None:
|
||||
return cache_creation_tokens * cache_creation_cost
|
||||
tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType(
|
||||
{
|
||||
"5m": cache_creation_token_details.ephemeral_5m_input_tokens,
|
||||
"30m": cache_creation_token_details.ephemeral_30m_input_tokens,
|
||||
"1hr": cache_creation_token_details.ephemeral_1h_input_tokens,
|
||||
"12hr": cache_creation_token_details.ephemeral_12h_input_tokens,
|
||||
}
|
||||
)
|
||||
unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0)
|
||||
return unattributed_tokens * cache_creation_cost + sum(
|
||||
(tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost)
|
||||
for ttl, tokens in tokens_by_ttl.items()
|
||||
)
|
||||
|
||||
|
||||
class PromptTokensDetailsResult(TypedDict):
|
||||
|
|
@ -1178,7 +1201,7 @@ def _calculate_input_cost(
|
|||
prompt_base_cost: float,
|
||||
cache_read_cost: float,
|
||||
cache_creation_cost: float,
|
||||
cache_creation_cost_above_1hr: float,
|
||||
cache_creation_cost_by_ttl: Mapping[str, float],
|
||||
service_tier: str | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
|
|
@ -1248,8 +1271,8 @@ def _calculate_input_cost(
|
|||
prompt_cost += calculate_cache_writing_cost(
|
||||
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
|
||||
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
|
||||
)
|
||||
|
||||
### CHARACTER COST
|
||||
|
|
@ -1510,34 +1533,31 @@ def generic_cost_per_token(
|
|||
)
|
||||
|
||||
billing_time: Final = current_time if current_time is not None else current_billing_time()
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
) = _get_token_base_cost(
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billing_time,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
completion_base_cost: Final = base_costs.output_rate
|
||||
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
current_time=billing_time,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost
|
||||
effective_cache_read_cost: Final = (
|
||||
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
|
||||
)
|
||||
|
||||
prompt_cost = _calculate_input_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=resolved_model_info,
|
||||
prompt_base_cost=prompt_base_cost,
|
||||
prompt_base_cost=base_costs.input_rate,
|
||||
cache_read_cost=effective_cache_read_cost,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
|
|
@ -1640,7 +1660,7 @@ class BilledTokenRates:
|
|||
cache_read_input_token_cost: float
|
||||
cache_read_input_audio_token_cost: float
|
||||
cache_creation_input_token_cost: float
|
||||
cache_creation_input_token_cost_above_1hr: float
|
||||
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
|
||||
output_cost_per_reasoning_token: float
|
||||
|
||||
def scaled(self, multiplier: float) -> "BilledTokenRates":
|
||||
|
|
@ -1652,7 +1672,9 @@ class BilledTokenRates:
|
|||
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
|
||||
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
|
||||
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
|
||||
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
|
||||
cache_creation_input_token_cost_by_ttl=MappingProxyType(
|
||||
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
|
||||
),
|
||||
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
|
||||
)
|
||||
|
||||
|
|
@ -1702,7 +1724,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
|
|||
cache_read_input_token_cost=cache_read_rate,
|
||||
cache_read_input_audio_token_cost=cache_read_rate,
|
||||
cache_creation_input_token_cost=cache_creation_rate,
|
||||
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
|
||||
cache_creation_input_token_cost_by_ttl=MappingProxyType({}),
|
||||
output_cost_per_reasoning_token=output_rate,
|
||||
)
|
||||
|
||||
|
|
@ -1717,13 +1739,7 @@ def _cost_map_billed_rates(
|
|||
current_time: datetime | None,
|
||||
) -> BilledTokenRates:
|
||||
billing_time: Final = current_time if current_time is not None else current_billing_time()
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost_rate,
|
||||
cache_creation_cost_above_1hr_rate,
|
||||
cache_read_cost_rate,
|
||||
) = _get_token_base_cost(
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
|
|
@ -1737,14 +1753,14 @@ def _cost_map_billed_rates(
|
|||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost_rate: Final = (
|
||||
cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
|
||||
base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
|
||||
)
|
||||
|
||||
reasoning_rate: Final = _resolve_billed_reasoning_rate(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
completion_base_cost=base_costs.output_rate,
|
||||
current_time=billing_time,
|
||||
)
|
||||
audio_cache_read_rate: Final = get_cost_per_unit(
|
||||
|
|
@ -1759,14 +1775,14 @@ def _cost_map_billed_rates(
|
|||
* get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
|
||||
)
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=prompt_base_cost,
|
||||
output_cost_per_token=completion_base_cost,
|
||||
input_cost_per_token=base_costs.input_rate,
|
||||
output_cost_per_token=base_costs.output_rate,
|
||||
cache_read_input_token_cost=effective_cache_read_cost_rate,
|
||||
cache_read_input_audio_token_cost=(
|
||||
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
|
||||
),
|
||||
cache_creation_input_token_cost=cache_creation_cost_rate,
|
||||
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
|
||||
cache_creation_input_token_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
output_cost_per_reasoning_token=reasoning_rate,
|
||||
).scaled(multiplier)
|
||||
|
||||
|
|
@ -1854,8 +1870,8 @@ def get_token_type_cost_breakdown(
|
|||
else calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
|
||||
cache_creation_cost=rates.cache_creation_input_token_cost,
|
||||
cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl,
|
||||
)
|
||||
)
|
||||
return TokenTypeCostBreakdown(
|
||||
|
|
@ -1886,7 +1902,7 @@ def calculate_prompt_caching_savings(
|
|||
biller saw them rather than at the later spend write.
|
||||
"""
|
||||
model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider)
|
||||
prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost(
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=model_info_with_cache_read_default,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
|
|
@ -1899,9 +1915,11 @@ def calculate_prompt_caching_savings(
|
|||
current_time=billed_at,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost
|
||||
write_rate: Final = cache_creation_cost or prompt_base_cost
|
||||
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
|
||||
prompt_base_cost: Final = base_costs.input_rate
|
||||
effective_cache_read_cost: Final = (
|
||||
base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost
|
||||
)
|
||||
write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost
|
||||
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
|
||||
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
|
||||
|
|
@ -1909,7 +1927,9 @@ def calculate_prompt_caching_savings(
|
|||
cache_creation_details: Final = (
|
||||
CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0),
|
||||
ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0),
|
||||
ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0),
|
||||
ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0),
|
||||
)
|
||||
if details is not None
|
||||
else None
|
||||
|
|
@ -1918,8 +1938,10 @@ def calculate_prompt_caching_savings(
|
|||
write_premium: Final = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_details,
|
||||
cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost,
|
||||
cache_creation_cost=write_rate - prompt_base_cost,
|
||||
cache_creation_cost_by_ttl=MappingProxyType(
|
||||
{ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
|
||||
),
|
||||
)
|
||||
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
|
||||
model_info, vertex_location
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
PromptTokensDetailsWrapper,
|
||||
|
|
@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total"
|
|||
|
||||
_TEXT_INPUT_RATE: Final = "input_cost_per_token"
|
||||
_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token"
|
||||
_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost"
|
||||
_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost"
|
||||
_TEXT_OUTPUT_RATE: Final = "output_cost_per_token"
|
||||
_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token"
|
||||
_RATE_KEY_MARKERS: Final = ("cost", "pricing")
|
||||
|
|
@ -33,19 +36,38 @@ def _tokens(value: object) -> int:
|
|||
|
||||
|
||||
def used_pricing_keys(usage: Usage) -> tuple[str, ...]:
|
||||
prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details)
|
||||
prompt_details: Final = parse_prompt_tokens_details(usage)
|
||||
prompt_audio: Final = prompt_details["audio_tokens"]
|
||||
completion_audio: Final = _audio_tokens(usage.completion_tokens_details)
|
||||
prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio
|
||||
prompt_text: Final = max(
|
||||
_tokens(usage.prompt_tokens)
|
||||
- prompt_details["cache_hit_tokens"]
|
||||
- prompt_audio
|
||||
- prompt_details["image_tokens"]
|
||||
- prompt_details["video_tokens"],
|
||||
0,
|
||||
)
|
||||
cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0)
|
||||
completion_text: Final = _tokens(usage.completion_tokens) - completion_audio
|
||||
components: Final = (
|
||||
(_TEXT_INPUT_RATE, prompt_text),
|
||||
(_AUDIO_INPUT_RATE, prompt_audio),
|
||||
(_CACHE_READ_INPUT_RATE, cached_text),
|
||||
(_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]),
|
||||
(_TEXT_OUTPUT_RATE, completion_text),
|
||||
(_AUDIO_OUTPUT_RATE, completion_audio),
|
||||
)
|
||||
return tuple(key for key, count in components if count > 0)
|
||||
|
||||
|
||||
def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str:
|
||||
if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None:
|
||||
return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE
|
||||
if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None:
|
||||
return _TEXT_INPUT_RATE
|
||||
return key
|
||||
|
||||
|
||||
def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None:
|
||||
try:
|
||||
return _NESTED_PRICING.validate_python(value)
|
||||
|
|
@ -89,7 +111,8 @@ def diagnose_zero_cost(
|
|||
pricing_entry: Mapping[str, object],
|
||||
calculation_failed: bool,
|
||||
) -> StandardLoggingZeroCostDiagnostic | None:
|
||||
used_keys: Final = used_pricing_keys(usage)
|
||||
effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage))
|
||||
used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index])
|
||||
if not used_keys:
|
||||
return None
|
||||
missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None)
|
||||
|
|
|
|||
7
litellm/litellm_core_utils/offline_mode.py
Normal file
7
litellm/litellm_core_utils/offline_mode.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import os
|
||||
|
||||
|
||||
def is_offline_mode() -> bool:
|
||||
"""True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather
|
||||
than from a destination the operator configured, must check this and use its bundled copy or skip instead"""
|
||||
return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true"
|
||||
|
|
@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None:
|
|||
return getattr(value, key, None)
|
||||
|
||||
|
||||
def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None:
|
||||
def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None:
|
||||
header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None
|
||||
if header is None:
|
||||
return None
|
||||
return default_header
|
||||
if not isinstance(header, str):
|
||||
raise TypeError("provider_affinity_header must be a string")
|
||||
return validate_provider_affinity_header_name(header)
|
||||
|
|
@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
|
|||
|
||||
|
||||
def add_provider_affinity_header(
|
||||
headers: Mapping[str, object], litellm_params: object | None
|
||||
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
|
||||
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
|
||||
header_name: Final = _get_provider_affinity_header_name(litellm_params)
|
||||
header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header)
|
||||
if header_name is None or any(key.lower() == header_name.lower() for key in headers):
|
||||
return dict(headers)
|
||||
|
||||
|
|
|
|||
|
|
@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
|
|||
_messages_carry_output_config = messages_carry_output_config
|
||||
|
||||
|
||||
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
|
||||
if "stop_sequence" not in response:
|
||||
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
|
||||
return response
|
||||
|
||||
|
||||
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||
_workload_identity_eligible: ClassVar[bool] = True
|
||||
|
||||
|
|
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> AnthropicMessagesResponse:
|
||||
"""
|
||||
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
|
||||
"""
|
||||
try:
|
||||
raw_response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
return AnthropicMessagesResponse(**raw_response_json)
|
||||
response: Final = AnthropicMessagesResponse(**raw_response_json)
|
||||
return _ensure_stop_sequence(response)
|
||||
|
||||
def get_async_streaming_response_iterator(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
|
|||
else:
|
||||
mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
|
||||
|
||||
# Extract engine from model name (e.g., "aws_polly/neural" -> "neural")
|
||||
# model arrives without the "aws_polly/" prefix, e.g. "standard"
|
||||
engine: Final = self._extract_engine_from_model(model)
|
||||
mapped_params["engine"] = engine
|
||||
|
||||
|
|
@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
|
|||
"""
|
||||
Extract engine from model name.
|
||||
|
||||
speech() strips the "aws_polly/" prefix before the config is called,
|
||||
so the bare engine name is what normally arrives here.
|
||||
|
||||
Examples:
|
||||
- aws_polly/neural -> neural
|
||||
- aws_polly/standard -> standard
|
||||
- standard -> standard
|
||||
- aws_polly/long-form -> long-form
|
||||
- aws_polly -> neural (default)
|
||||
"""
|
||||
if "/" in model:
|
||||
parts: Final = model.split("/")
|
||||
if len(parts) >= 2:
|
||||
engine: Final = parts[1].lower()
|
||||
if engine in self.VALID_ENGINES:
|
||||
return engine
|
||||
engine: Final = model.rsplit("/", 1)[-1].lower()
|
||||
if engine in self.VALID_ENGINES:
|
||||
return engine
|
||||
return self.DEFAULT_ENGINE
|
||||
|
||||
def validate_environment(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from collections.abc import Mapping
|
|||
from copy import deepcopy
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from openai.types.responses import ResponseReasoningItem
|
||||
|
|
@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
|
||||
"""
|
||||
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
|
||||
responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params)
|
||||
parsed_base: Final = urlsplit(resolved_api_base or "")
|
||||
path: Final = parsed_base.path.rstrip("/")
|
||||
if path.endswith(("/openai/v1", "/openai/v1/responses")):
|
||||
responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path
|
||||
return BaseAzureLLM.get_base_azure_url(
|
||||
api_base=parsed_base._replace(path=responses_path).geturl(),
|
||||
litellm_params=responses_params,
|
||||
route="/openai/v1/responses",
|
||||
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
|
||||
)
|
||||
|
||||
return BaseAzureLLM.get_base_azure_url(
|
||||
api_base=resolved_api_base,
|
||||
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
|
||||
litellm_params=responses_params,
|
||||
route="/openai/responses",
|
||||
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
audio_or_image_in_message_content,
|
||||
convert_content_list_to_str,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
|
|
@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
Operates on a deep copy so the caller's messages keep their thinking blocks
|
||||
and provider metadata, which a fallback to another provider still needs.
|
||||
"""
|
||||
stripped_messages: Final = copy.deepcopy(messages)
|
||||
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
|
||||
for message in stripped_messages:
|
||||
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
|
||||
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from typing_extensions import Self
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
chat_completion_tools_for,
|
||||
is_flat_responses_tool,
|
||||
|
|
@ -67,12 +68,14 @@ class KimiK3TokenizerSource:
|
|||
def _hugging_face_ranks_path() -> str:
|
||||
import huggingface_hub
|
||||
|
||||
return huggingface_hub.hf_hub_download(
|
||||
repo_id=KIMI_K3_REPO,
|
||||
filename=KIMI_K3_RANKS_FILE,
|
||||
revision=KIMI_K3_REVISION,
|
||||
local_files_only=litellm.disable_hf_tokenizer_download is True,
|
||||
)
|
||||
if litellm.disable_hf_tokenizer_download is True or is_offline_mode():
|
||||
cached: Final = huggingface_hub.try_to_load_from_cache(
|
||||
repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION
|
||||
)
|
||||
if not isinstance(cached, str):
|
||||
raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache")
|
||||
return cached
|
||||
return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION)
|
||||
|
||||
|
||||
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
|
|||
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
|
||||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.realtime_errors import (
|
||||
close_after_upstream_handshake_refusal,
|
||||
realtime_error_event,
|
||||
|
|
@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import (
|
|||
)
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -2435,13 +2438,13 @@ class BaseLLMHTTPHandler:
|
|||
# Also check for extra_headers in kwargs (from config or direct calls)
|
||||
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
|
||||
# Merge all header sources: forwarded < extra_headers < provider_specific
|
||||
merged_headers: Final = {}
|
||||
if forwarded_headers:
|
||||
merged_headers.update(forwarded_headers)
|
||||
if extra_headers_from_kwargs:
|
||||
merged_headers.update(extra_headers_from_kwargs)
|
||||
if provider_specific_headers:
|
||||
merged_headers.update(provider_specific_headers)
|
||||
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{
|
||||
**(forwarded_headers or {}),
|
||||
**(extra_headers_from_kwargs or {}),
|
||||
**(provider_specific_headers or {}),
|
||||
}
|
||||
)
|
||||
request_params: Final = reduce(
|
||||
delete_nested_value,
|
||||
litellm_params.get("additional_drop_params") or (),
|
||||
|
|
@ -2452,11 +2455,19 @@ class BaseLLMHTTPHandler:
|
|||
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
|
||||
):
|
||||
await aload_beta_headers_config()
|
||||
affinity_api_base: Final = (
|
||||
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
(
|
||||
headers,
|
||||
api_base,
|
||||
) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment(
|
||||
headers=merged_headers or {},
|
||||
headers=(
|
||||
add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header)
|
||||
if default_affinity_header is not None
|
||||
else dict(merged_headers)
|
||||
),
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=request_params,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
|
|||
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
|
@ -72,7 +73,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates:
|
|||
)
|
||||
|
||||
|
||||
def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
|
||||
def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates:
|
||||
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
|
||||
# spelling out only input rates would serve every completion for free, so there the model's
|
||||
# own output rates stand in
|
||||
|
|
|
|||
10
litellm/llms/opencode/common_utils.py
Normal file
10
litellm/llms/opencode/common_utils.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import re
|
||||
|
||||
|
||||
def is_opencode_api_base(api_base: str | None) -> bool:
|
||||
return bool(
|
||||
api_base
|
||||
and re.fullmatch(
|
||||
r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base
|
||||
)
|
||||
)
|
||||
6
litellm/llms/opencode/transformation.py
Normal file
6
litellm/llms/opencode/transformation.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
|
||||
|
||||
class OpenCodeProviderAdapter:
|
||||
def get_provider_affinity_header(self, api_base: str | None) -> str | None:
|
||||
return "x-opencode-session" if is_opencode_api_base(api_base) else None
|
||||
|
|
@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_httpx_client,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif modality == "DOCUMENT":
|
||||
cached_text_tokens = (cached_text_tokens or 0) + token_count
|
||||
|
||||
## Calculate non-cached tokens by subtracting cached from total (per modality)
|
||||
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
|
||||
## See: https://github.com/BerriAI/litellm/issues/18750
|
||||
if cached_text_tokens is not None and prompt_text_tokens is not None:
|
||||
# Explicit caching: subtract cached tokens per modality from cacheTokensDetails
|
||||
prompt_text_tokens = prompt_text_tokens - cached_text_tokens
|
||||
elif (
|
||||
if (
|
||||
cached_tokens is not None
|
||||
and prompt_text_tokens is not None
|
||||
and cached_text_tokens is None
|
||||
and "cacheTokensDetails" not in usage_metadata
|
||||
):
|
||||
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
|
||||
# Subtract from text tokens since implicit caching is primarily for text content
|
||||
# See: https://github.com/BerriAI/litellm/issues/16341
|
||||
prompt_text_tokens = prompt_text_tokens - cached_tokens
|
||||
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
|
||||
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
|
||||
if cached_image_tokens is not None and prompt_image_tokens is not None:
|
||||
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
|
||||
if cached_video_tokens is not None and prompt_video_tokens is not None:
|
||||
prompt_video_tokens = prompt_video_tokens - cached_video_tokens
|
||||
|
||||
|
|
@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
billable_text_tokens: Final = (
|
||||
prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens
|
||||
)
|
||||
cached_tokens_details_fields: Final = MappingProxyType(
|
||||
{
|
||||
key: token_count
|
||||
for key, token_count in (
|
||||
("text_tokens", cached_text_tokens),
|
||||
("audio_tokens", cached_audio_tokens),
|
||||
("image_tokens", cached_image_tokens),
|
||||
)
|
||||
if token_count is not None
|
||||
}
|
||||
)
|
||||
cached_tokens_details: Final = (
|
||||
CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None
|
||||
)
|
||||
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
cached_tokens=cached_tokens,
|
||||
cached_tokens_details=cached_tokens_details,
|
||||
audio_tokens=prompt_audio_tokens,
|
||||
text_tokens=billable_text_tokens,
|
||||
image_tokens=prompt_image_tokens,
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import (
|
|||
with_control_options,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
get_provider_affinity_header,
|
||||
invalid_provider_routing_hints_message,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import (
|
||||
|
|
@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl
|
|||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
VertexAIModelRoute,
|
||||
|
|
@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import (
|
|||
resolve_chat_reasoning,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
|
||||
from litellm.types.completion import (
|
||||
CompletionDispatchContext,
|
||||
_CompletionDispatchResult,
|
||||
|
|
@ -169,7 +172,6 @@ from litellm.utils import (
|
|||
ProviderConfigManager,
|
||||
TextChoices,
|
||||
TextCompletionResponse,
|
||||
TextCompletionStreamWrapper,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
add_provider_specific_params_to_optional_params,
|
||||
|
|
@ -5990,19 +5992,28 @@ def completion(
|
|||
},
|
||||
)
|
||||
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
|
||||
if litellm_params.get("provider_affinity_header") is not None:
|
||||
affinity_api_base: Final = (
|
||||
str(client.base_url)
|
||||
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
|
||||
else get_openai_credentials(api_base=api_base).api_base
|
||||
if custom_llm_provider == "openai"
|
||||
else api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None:
|
||||
try:
|
||||
headers = add_provider_affinity_header(
|
||||
headers=headers or litellm.headers or MappingProxyType({}),
|
||||
litellm_params=MappingProxyType(
|
||||
{
|
||||
"provider_affinity_header": litellm_params["provider_affinity_header"],
|
||||
"provider_affinity_header": litellm_params.get("provider_affinity_header"),
|
||||
"litellm_session_id": kwargs.get("litellm_session_id"),
|
||||
"session_id": kwargs.get("session_id"),
|
||||
"metadata": metadata,
|
||||
"litellm_metadata": kwargs.get("litellm_metadata"),
|
||||
}
|
||||
),
|
||||
default_header=default_affinity_header,
|
||||
)
|
||||
except ValueError as affinity_error:
|
||||
raise litellm.BadRequestError(
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target:
|
|||
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
||||
{
|
||||
**detectors.COMMON,
|
||||
Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity,
|
||||
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
|
||||
Feature.RESPONSE_CACHE: detectors.response_cache,
|
||||
Feature.MODIFY_PARAMS: detectors.modify_params,
|
||||
|
|
|
|||
|
|
@ -12827,6 +12827,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 16384,
|
||||
|
|
@ -12840,6 +12841,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"azure_ai/Phi-4-mini-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12851,6 +12853,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-multimodal-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_audio_token": 4e-06,
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -12865,6 +12868,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4-mini-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12876,6 +12880,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 32768,
|
||||
|
|
@ -24892,27 +24897,118 @@
|
|||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_multilingual_v2": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4": {
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4_turbo": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3_conversational": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
|
|
@ -31041,11 +31137,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost_batches": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31370,11 +31470,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 3e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 3e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 5.4e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
"input_cost_per_audio_token_flex": 1.5e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31664,12 +31768,16 @@
|
|||
},
|
||||
"gemini/gemini-3.1-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 9e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_audio_token_flex": 2.5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"input_cost_per_token_flex": 1.25e-07,
|
||||
|
|
@ -31790,10 +31898,14 @@
|
|||
},
|
||||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 5e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -36699,6 +36811,12 @@
|
|||
},
|
||||
"gpt-6-astra": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
|
||||
|
|
@ -36784,6 +36902,12 @@
|
|||
},
|
||||
"gpt-6-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -36861,17 +36985,23 @@
|
|||
},
|
||||
"gpt-6-luna": {
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_decisions": 1e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
|
||||
"cache_creation_input_token_cost_decisions": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
|
||||
"cache_creation_input_token_cost_flex": 6.25e-08,
|
||||
"cache_creation_input_token_cost_priority": 2.5e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_decisions": 1e-07,
|
||||
"cache_read_input_token_cost_decisions": 1e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-09,
|
||||
|
|
@ -36947,6 +37077,12 @@
|
|||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37021,6 +37157,12 @@
|
|||
},
|
||||
"gpt-5.6-sol": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37097,6 +37239,12 @@
|
|||
},
|
||||
"gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -37172,6 +37320,12 @@
|
|||
},
|
||||
"gpt-5.6-luna": {
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
|
|
@ -42337,6 +42491,134 @@
|
|||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3-0": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3-0": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3-0": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3-0": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-small-2507": {
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -45623,6 +45905,7 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -45648,6 +45931,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -48499,6 +48783,17 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
|
||||
"cache_read_input_token_cost": 1.5e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://www.together.ai/pricing",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
@ -49352,6 +49647,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"us-gov.openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -49371,6 +49667,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"us-gov.openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -52906,12 +53203,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
|
||||
"input_cost_per_token": 3.5e-07,
|
||||
"input_cost_per_token_batches": 1.75e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"max_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.15e-06,
|
||||
"output_cost_per_token_batches": 5.75e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -52926,12 +53225,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 10000000,
|
||||
"max_output_tokens": 10000000,
|
||||
"max_tokens": 10000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-07,
|
||||
"output_cost_per_token_batches": 3.5e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -59917,6 +60218,7 @@
|
|||
}
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7.5e-08,
|
||||
"input_cost_per_token_flex": 7.5e-08,
|
||||
|
|
@ -59942,6 +60244,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3.5e-08,
|
||||
"input_cost_per_token_flex": 3.5e-08,
|
||||
|
|
@ -59967,6 +60270,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7e-08,
|
||||
"input_cost_per_token_flex": 7e-08,
|
||||
|
|
@ -59990,6 +60294,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
|
|
@ -67841,6 +68146,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -67860,6 +68166,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68138,6 +68445,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68157,6 +68465,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68545,6 +68854,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68564,6 +68874,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68704,6 +69015,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68723,6 +69035,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -73330,6 +73643,7 @@
|
|||
]
|
||||
},
|
||||
"vertex_ai/veo-2.0-generate-001": {
|
||||
"deprecation_date": "2026-06-30",
|
||||
"litellm_provider": "vertex_ai-video-models",
|
||||
"max_input_tokens": 1024,
|
||||
"max_tokens": 1024,
|
||||
|
|
@ -74677,6 +74991,12 @@
|
|||
},
|
||||
"gpt-6.1-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
|
|
|
|||
|
|
@ -1368,7 +1368,7 @@ class MCPRequestHandler:
|
|||
|
||||
if auth is None:
|
||||
return None
|
||||
refreshed = auth.model_copy()
|
||||
refreshed: Final = auth.model_copy(update={"requires_fresh_policy": True})
|
||||
if (auth.mcp_admitted_user_subject or auth.is_session_token) and auth.user_id:
|
||||
# A session credential (``lite login`` or a redeemed ID-JAG) has no key row, its authority is
|
||||
# sealed in the token: refresh the live user policy and keep the token's team, routes and models.
|
||||
|
|
@ -1386,7 +1386,12 @@ class MCPRequestHandler:
|
|||
refreshed.project_id = current.project_id
|
||||
refreshed.user_id = current.user_id
|
||||
refreshed.access_group_ids = current.access_group_ids
|
||||
refreshed.requires_fresh_policy = True
|
||||
else:
|
||||
return refreshed
|
||||
if refreshed.mcp_toolset_id is not None:
|
||||
from litellm.proxy._experimental.mcp_server.server import apply_toolset_scope
|
||||
|
||||
return await apply_toolset_scope(refreshed, refreshed.mcp_toolset_id)
|
||||
return refreshed
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -1797,7 +1797,7 @@ if MCP_AVAILABLE:
|
|||
return acting.model_copy(update={"mcp_toolset_id": toolset_id})
|
||||
|
||||
tool_permissions = await operations.global_mcp_server_manager.resolve_toolset_tool_permissions(
|
||||
toolset_ids=[toolset_id]
|
||||
toolset_ids=[toolset_id], requires_fresh_policy=acting.requires_fresh_policy
|
||||
)
|
||||
server_ids: Final = list(tool_permissions.keys())
|
||||
existing_op: Final = acting.object_permission
|
||||
|
|
|
|||
BIN
litellm/proxy/_experimental/out/assets/aria-agent.png
Normal file
BIN
litellm/proxy/_experimental/out/assets/aria-agent.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 685 KiB |
27
litellm/proxy/_experimental/out/assets/logos/claude.svg
Normal file
27
litellm/proxy/_experimental/out/assets/logos/claude.svg
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
<!--
|
||||
Source: https://github.com/lobehub/lobe-icons/blob/master/packages/static-svg/icons/claude-color.svg
|
||||
Retrieved: 2026-10-22
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 LobeHub
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-->
|
||||
<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z" fill="#D97757" fill-rule="nonzero"></path></svg>
|
||||
|
|
@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
|
||||
path_prefixes=("/vantage",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="usage_ai",
|
||||
module_path="litellm.proxy.management_endpoints.usage_endpoints",
|
||||
path_prefixes=("/usage/ai",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="prompts",
|
||||
module_path="litellm.proxy.prompts.prompt_endpoints",
|
||||
|
|
|
|||
|
|
@ -11407,7 +11407,7 @@
|
|||
},
|
||||
"/systemone": {
|
||||
"post": {
|
||||
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
|
||||
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
|
||||
"operationId": "systemone_systemone_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
|
|
@ -11813,7 +11813,6 @@
|
|||
]
|
||||
},
|
||||
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
|
||||
"maxProperties": 128,
|
||||
"minProperties": 1,
|
||||
"propertyNames": {
|
||||
"minLength": 1
|
||||
|
|
@ -12814,7 +12813,7 @@
|
|||
},
|
||||
"/v1/systemone": {
|
||||
"post": {
|
||||
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
|
||||
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
|
||||
"operationId": "systemone_v1_systemone_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
|
|
@ -13220,7 +13219,6 @@
|
|||
]
|
||||
},
|
||||
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
|
||||
"maxProperties": 128,
|
||||
"minProperties": 1,
|
||||
"propertyNames": {
|
||||
"minLength": 1
|
||||
|
|
@ -22860,6 +22858,25 @@
|
|||
"title": "Body_image_edit_api_openai_deployments__model__images_edits_post",
|
||||
"type": "object"
|
||||
},
|
||||
"CacheWriteTokensDetails": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"ttl_12h_tokens": {
|
||||
"title": "Ttl 12H Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"ttl_30m_tokens": {
|
||||
"title": "Ttl 30M Tokens",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"ttl_30m_tokens",
|
||||
"ttl_12h_tokens"
|
||||
],
|
||||
"title": "CacheWriteTokensDetails",
|
||||
"type": "object"
|
||||
},
|
||||
"CachedTokensDetails": {
|
||||
"properties": {
|
||||
"audio_tokens": {
|
||||
|
|
@ -24354,6 +24371,27 @@
|
|||
],
|
||||
"title": "Audio Tokens"
|
||||
},
|
||||
"cache_write_tokens": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Cache Write Tokens"
|
||||
},
|
||||
"cache_write_tokens_details": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/CacheWriteTokensDetails"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"cached_tokens": {
|
||||
"default": 0,
|
||||
"title": "Cached Tokens",
|
||||
|
|
@ -57900,163 +57938,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"usage_ai": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
"ChatMessage": {
|
||||
"properties": {
|
||||
"content": {
|
||||
"title": "Content",
|
||||
"type": "string"
|
||||
},
|
||||
"role": {
|
||||
"enum": [
|
||||
"user",
|
||||
"assistant"
|
||||
],
|
||||
"title": "Role",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"title": "ChatMessage",
|
||||
"type": "object"
|
||||
},
|
||||
"HTTPValidationError": {
|
||||
"properties": {
|
||||
"detail": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ValidationError"
|
||||
},
|
||||
"title": "Detail",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"title": "HTTPValidationError",
|
||||
"type": "object"
|
||||
},
|
||||
"UsageAIChatRequest": {
|
||||
"properties": {
|
||||
"messages": {
|
||||
"description": "Chat messages (user/assistant history)",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ChatMessage"
|
||||
},
|
||||
"title": "Messages",
|
||||
"type": "array"
|
||||
},
|
||||
"model": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Model to use for AI chat",
|
||||
"title": "Model"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"messages"
|
||||
],
|
||||
"title": "UsageAIChatRequest",
|
||||
"type": "object"
|
||||
},
|
||||
"ValidationError": {
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"title": "Context",
|
||||
"type": "object"
|
||||
},
|
||||
"input": {
|
||||
"title": "Input"
|
||||
},
|
||||
"loc": {
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "integer"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Location",
|
||||
"type": "array"
|
||||
},
|
||||
"msg": {
|
||||
"title": "Message",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"title": "Error Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"loc",
|
||||
"msg",
|
||||
"type"
|
||||
],
|
||||
"title": "ValidationError",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"paths": {
|
||||
"/usage/ai/chat": {
|
||||
"post": {
|
||||
"description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.",
|
||||
"operationId": "usage_ai_chat_usage_ai_chat_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/UsageAIChatRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Usage Ai Chat",
|
||||
"tags": [
|
||||
"usage_ai"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"vantage": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
|
|
|
|||
|
|
@ -2717,6 +2717,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
"hidden model can still be called."
|
||||
),
|
||||
)
|
||||
return_wildcard_routes: bool | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"When true, `/v1/models` lists wildcard routes like `openai/*` unless the request passes "
|
||||
"`return_wildcard_routes=false`."
|
||||
),
|
||||
)
|
||||
alerting: list | None = Field(
|
||||
None,
|
||||
description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL",
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import Final
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._version import version
|
||||
from litellm.constants import HIBP_RANGE_API_BASE
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
|
@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool
|
|||
|
||||
|
||||
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
|
||||
return general_settings.get("password_policy_check_breached_passwords", True) is not False
|
||||
return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False
|
||||
|
||||
|
||||
async def is_password_breached(
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import json
|
|||
import os
|
||||
import random
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
|
|
@ -90,6 +89,7 @@ from litellm.proxy.spend_tracking.savings import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.repositories.prisma_protocols import BatchTable
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -313,6 +313,38 @@ def _spend_tables_left_to_send(
|
|||
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
|
||||
|
||||
|
||||
def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None:
|
||||
spend_log_error(
|
||||
"Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s",
|
||||
step_name,
|
||||
payload.get("request_id"),
|
||||
payload.get("api_key"),
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
|
||||
_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata
|
||||
|
||||
|
||||
def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata:
|
||||
"""Malformed metadata costs the request its metadata-derived fields, never its daily rollup row."""
|
||||
raw: Final = payload.get("metadata")
|
||||
if not isinstance(raw, str):
|
||||
return _EMPTY_SPEND_LOG_METADATA
|
||||
try:
|
||||
parsed: Final[SpendLogsMetadata] = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
spend_log_error(
|
||||
"Spend tracking - unreadable spend log metadata, rollups for this request are written without "
|
||||
"metadata-derived fields. request_id=%s api_key=%s",
|
||||
payload.get("request_id"),
|
||||
payload.get("api_key"),
|
||||
exc=e,
|
||||
)
|
||||
return _EMPTY_SPEND_LOG_METADATA
|
||||
return parsed
|
||||
|
||||
|
||||
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
|
||||
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
|
||||
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
|
||||
|
|
@ -647,8 +679,8 @@ class DBSpendUpdateWriter:
|
|||
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
|
||||
)
|
||||
|
||||
# Single task replaces 11 create_task() calls
|
||||
asyncio.create_task(
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
self._batch_database_updates(
|
||||
response_cost=response_cost,
|
||||
user_id=user_id,
|
||||
|
|
@ -661,7 +693,8 @@ class DBSpendUpdateWriter:
|
|||
litellm_proxy_budget_name=litellm_proxy_budget_name,
|
||||
payload=payload,
|
||||
request_model_access_groups=get_request_model_access_groups(kwargs),
|
||||
)
|
||||
),
|
||||
task_name="spend rollups",
|
||||
)
|
||||
|
||||
self._enqueue_tool_registry_upsert(
|
||||
|
|
@ -1086,91 +1119,52 @@ class DBSpendUpdateWriter:
|
|||
project_id: str | None = None,
|
||||
):
|
||||
"""
|
||||
Runs all 13 spend-update helpers sequentially inside a single asyncio task.
|
||||
|
||||
Each helper is wrapped in try/except so one failure doesn't prevent the others.
|
||||
|
||||
The deepcopy runs here, off the awaited request path, so the daily spend
|
||||
helpers get a payload isolated from the spend-log queue entry and the caller.
|
||||
Runs every spend-update helper for one request. Each helper goes through
|
||||
_run_spend_update_step so one failure never skips the sibling writes. The deepcopy
|
||||
isolates the daily spend helpers' payload from the spend-log queue entry and the caller.
|
||||
"""
|
||||
payload_copy: Final = copy.deepcopy(payload)
|
||||
request_tags: Final = payload_copy.get("request_tags")
|
||||
try:
|
||||
await self._update_user_db(
|
||||
await self._run_spend_update_step(
|
||||
"_update_user_db",
|
||||
self._update_user_db(
|
||||
response_cost=response_cost,
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
litellm_proxy_budget_name=litellm_proxy_budget_name,
|
||||
end_user_id=end_user_id,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_user_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_key_db(
|
||||
response_cost=response_cost,
|
||||
hashed_token=hashed_token,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_key_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_team_db(
|
||||
response_cost=response_cost,
|
||||
team_id=team_id,
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_team_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_org_db(
|
||||
response_cost=response_cost,
|
||||
org_id=org_id,
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_org_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_project_db(
|
||||
response_cost=response_cost,
|
||||
project_id=project_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_project_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_tag_db(
|
||||
response_cost=response_cost,
|
||||
request_tags=request_tags,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_tag_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_key_db",
|
||||
self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_team_db",
|
||||
self._update_team_db(
|
||||
response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_org_db",
|
||||
self._update_org_db(
|
||||
response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_project_db",
|
||||
self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_tag_db",
|
||||
self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._update_model_access_group_db(
|
||||
response_cost=response_cost,
|
||||
request_model_access_groups=request_model_access_groups,
|
||||
|
|
@ -1178,96 +1172,64 @@ class DBSpendUpdateWriter:
|
|||
prisma_client=prisma_client,
|
||||
router=get_llm_router(),
|
||||
)
|
||||
|
||||
_agent_id_for_spend: Final = payload_copy.get("agent_id")
|
||||
try:
|
||||
await self._update_agent_db(
|
||||
response_cost=response_cost,
|
||||
agent_id=_agent_id_for_spend,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_agent_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_user_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_end_user_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_agent_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_team_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_agent_db",
|
||||
self._update_agent_db(
|
||||
response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_user_transaction",
|
||||
self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_end_user_transaction",
|
||||
self.add_spend_log_transaction_to_daily_end_user_transaction(
|
||||
payload=payload_copy, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_agent_transaction",
|
||||
self.add_spend_log_transaction_to_daily_agent_transaction(
|
||||
payload=payload_copy, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_team_transaction",
|
||||
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
try:
|
||||
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
|
||||
except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _record_request_error failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_org_transaction(
|
||||
payload=payload_copy,
|
||||
org_id=org_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_tag_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
|
||||
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_org_transaction",
|
||||
self.add_spend_log_transaction_to_daily_org_transaction(
|
||||
payload=payload_copy, org_id=org_id, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_tag_transaction",
|
||||
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
|
||||
|
||||
@staticmethod
|
||||
async def _run_spend_update_step(
|
||||
step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload
|
||||
) -> None:
|
||||
try:
|
||||
await step
|
||||
except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes
|
||||
_report_spend_update_step_failure(step_name, payload=payload, exc=e)
|
||||
|
||||
async def _update_key_db(
|
||||
self,
|
||||
response_cost: float | None,
|
||||
|
|
@ -2810,7 +2772,7 @@ class DBSpendUpdateWriter:
|
|||
date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "")
|
||||
if not date:
|
||||
return
|
||||
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
metadata: Final = _parse_spend_log_metadata(payload)
|
||||
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
|
||||
request_error_accumulator.record(
|
||||
payload=payload,
|
||||
|
|
@ -2861,7 +2823,7 @@ class DBSpendUpdateWriter:
|
|||
# spend, token and per-entity columns written here stay either way.
|
||||
request_status: Final = prisma_client.get_request_status(payload)
|
||||
verbose_proxy_logger.debug("Logged request status: %s", request_status)
|
||||
_metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
_metadata: Final = _parse_spend_log_metadata(payload)
|
||||
usage_obj: Final = _metadata.get("usage_object", {}) or {}
|
||||
if isinstance(payload["startTime"], datetime):
|
||||
start_time: Final = payload["startTime"].isoformat()
|
||||
|
|
|
|||
|
|
@ -228,7 +228,7 @@ async def systemone(
|
|||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
"""
|
||||
Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
|
||||
Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
|
||||
|
||||
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
|
||||
`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,
|
||||
|
|
|
|||
|
|
@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = (
|
|||
"/v1/messages",
|
||||
"responses",
|
||||
"files",
|
||||
"interactions",
|
||||
)
|
||||
|
||||
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
|
||||
|
|
@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
|||
LlmProviders.VERTEX_AI.value,
|
||||
)
|
||||
)
|
||||
_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
||||
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
|
||||
)
|
||||
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
|
||||
|
||||
|
||||
|
|
@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request(
|
|||
custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers
|
||||
),
|
||||
ProviderSpecificHeader(
|
||||
custom_llm_provider=LlmProviders.DATABRICKS.value,
|
||||
custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS,
|
||||
extra_headers=anthropic_api_headers,
|
||||
native_anthropic_messages_only=True,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -2093,7 +2093,9 @@ async def get_user_key_counts(
|
|||
return result
|
||||
|
||||
|
||||
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
|
||||
def _validate_sort_params(
|
||||
sort_by: str | None, sort_order: str
|
||||
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
|
||||
order_by: Final[dict[str, str]] = {}
|
||||
|
||||
if sort_by is None:
|
||||
|
|
@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
|
|||
"spend",
|
||||
"user_alias",
|
||||
"user_role",
|
||||
"max_budget",
|
||||
"updated_at",
|
||||
"sso_user_id",
|
||||
"status",
|
||||
"key_count",
|
||||
]
|
||||
if sort_by not in valid_columns:
|
||||
raise HTTPException(
|
||||
|
|
@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
|
|||
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
|
||||
)
|
||||
|
||||
if sort_by in {"status", "key_count"}:
|
||||
return None
|
||||
if sort_by in {"max_budget", "updated_at", "sso_user_id"}:
|
||||
nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last"
|
||||
return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"})
|
||||
order_by[sort_by] = sort_order.lower()
|
||||
|
||||
return order_by
|
||||
|
|
@ -2338,16 +2350,40 @@ async def get_users(
|
|||
|
||||
# Build order_by conditions
|
||||
|
||||
order_by: Final[dict[str, str] | None] = (
|
||||
order_by: Final = (
|
||||
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
|
||||
)
|
||||
|
||||
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
|
||||
where=where,
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
|
||||
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
|
||||
|
||||
computed_sort: Final = sort_by in ("status", "key_count")
|
||||
sorted_ids: Final = (
|
||||
await sorted_user_ids(
|
||||
prisma_client,
|
||||
sort_by="status" if sort_by == "status" else "key_count",
|
||||
descending=sort_order.lower() == "desc",
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
role=role,
|
||||
user_ids=user_ids,
|
||||
user_email=user_email,
|
||||
team=team,
|
||||
sso_user_ids=sso_user_ids,
|
||||
organization_ids=organization_ids,
|
||||
search=search,
|
||||
)
|
||||
if computed_sort
|
||||
else ()
|
||||
)
|
||||
page_users: Final = await UserRepository(prisma_client).table.find_many(
|
||||
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
|
||||
where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where,
|
||||
skip=0 if computed_sort else skip,
|
||||
take=page_size,
|
||||
order=order_by if order_by else {"created_at": "desc"},
|
||||
)
|
||||
positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)}
|
||||
users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users
|
||||
|
||||
# Get total count of user rows
|
||||
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)
|
||||
|
|
|
|||
|
|
@ -7397,20 +7397,26 @@ async def key_aliases(
|
|||
)
|
||||
|
||||
|
||||
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
|
||||
order_by: Final[dict[str, str]] = {}
|
||||
|
||||
def _validate_sort_params(
|
||||
sort_by: str | None, sort_order: str
|
||||
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
|
||||
if sort_by is None:
|
||||
return None
|
||||
# Validate sort_by is a valid column
|
||||
valid_columns: Final = [
|
||||
valid_columns: Final = (
|
||||
"spend",
|
||||
"max_budget",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"token",
|
||||
"key_alias",
|
||||
]
|
||||
"expires",
|
||||
"last_active",
|
||||
"budget_reset_at",
|
||||
"total_spend",
|
||||
"tpm_limit",
|
||||
"rpm_limit",
|
||||
)
|
||||
if sort_by not in valid_columns:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
|
|
@ -7424,9 +7430,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
|
|||
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
|
||||
)
|
||||
|
||||
order_by[sort_by] = sort_order.lower()
|
||||
|
||||
return order_by
|
||||
direction: Final = sort_order.lower()
|
||||
nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit")
|
||||
if sort_by not in (*nullable_sort_columns, "total_spend"):
|
||||
return {sort_by: direction}
|
||||
return [
|
||||
{sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction},
|
||||
{"token": "asc"},
|
||||
]
|
||||
|
||||
|
||||
def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]:
|
||||
|
|
@ -7668,7 +7679,7 @@ async def list_key_helper(
|
|||
|
||||
verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size)
|
||||
|
||||
order_by: Final[dict[str, str] | None] = (
|
||||
order_by: Final = (
|
||||
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final
|
||||
|
||||
|
|
@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...])
|
|||
SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields)
|
||||
|
||||
|
||||
# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt.
|
||||
_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰0𐒠𐴰𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑜰𑣠𑥐𑱐𑵐𑶠\U00011de0𑽐𖩠𖫀𖭐𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞥐🯰"
|
||||
_DECIMAL_DIGITS: Final = "".join(
|
||||
chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES)
|
||||
)
|
||||
_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES)
|
||||
|
||||
|
||||
_BUDGET_DURATION_SECONDS: Final = f"""(
|
||||
SELECT EXTRACT(EPOCH FROM (
|
||||
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +
|
||||
(parts[1] || ' ' || CASE parts[2]
|
||||
WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours'
|
||||
WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval
|
||||
- (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
|
||||
)) FROM regexp_match(
|
||||
CASE lower(trim(budget_duration))
|
||||
WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h'
|
||||
WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d'
|
||||
ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration
|
||||
ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END,
|
||||
'^([0-9]+)(mo|[smhdw])'
|
||||
) AS duration(parts)
|
||||
)"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PrismaBudgetListExecutor:
|
||||
"""The database half of the budgets list. Every caller-supplied value is bound to a
|
||||
|
|
@ -91,10 +118,16 @@ class PrismaBudgetListExecutor:
|
|||
|
||||
async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]:
|
||||
clauses, params = where_sql(plan.where)
|
||||
ordering: Final = ", ".join(
|
||||
order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST")
|
||||
if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")
|
||||
else order_by_sql((key,))
|
||||
for key in plan.order
|
||||
).replace('"budget_duration"', _BUDGET_DURATION_SECONDS)
|
||||
sql: Final = (
|
||||
f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}"
|
||||
+ (f" WHERE {clauses}" if clauses else "")
|
||||
+ f" ORDER BY {order_by_sql(plan.order)}"
|
||||
+ f" ORDER BY {ordering}"
|
||||
+ f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
|
||||
)
|
||||
return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip))
|
||||
|
|
@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope:
|
|||
return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}")
|
||||
|
||||
|
||||
# budget_duration is deliberately absent from `sortable`: the column holds strings
|
||||
# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d".
|
||||
BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
|
||||
{
|
||||
"budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))),
|
||||
|
|
@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
|
|||
|
||||
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
|
||||
resource="budgets",
|
||||
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
|
||||
sortable=frozenset(
|
||||
("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration")
|
||||
),
|
||||
searchable=frozenset(("budget_id",)),
|
||||
filters=BUDGET_FILTERS,
|
||||
default_sort=(SortKey(field="created_at", descending=True),),
|
||||
|
|
@ -154,8 +187,9 @@ async def list_budgets(
|
|||
way to page, sort or filter it.
|
||||
|
||||
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
|
||||
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
|
||||
and defaults to `-created_at`. `budget_id` is appended to every sort as the
|
||||
`rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending,
|
||||
and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months
|
||||
measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the
|
||||
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
|
||||
`page_size` defaults to 50 and is capped at 100. Filters are
|
||||
`filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]`
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.constants import (
|
|||
MAX_COMPETITOR_NAMES,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.llms.openai.chat.guardrail_translation.handler import (
|
||||
OpenAIChatCompletionsHandler,
|
||||
)
|
||||
|
|
@ -657,7 +658,7 @@ async def get_policy_templates(
|
|||
Fetches from GitHub with automatic fallback to local backup on failure.
|
||||
Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.
|
||||
"""
|
||||
use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
|
||||
use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
|
||||
"true",
|
||||
"1",
|
||||
"yes",
|
||||
|
|
|
|||
|
|
@ -6109,13 +6109,52 @@ async def list_team_v2(
|
|||
"total_pages": 0,
|
||||
}
|
||||
|
||||
# Build order_by conditions
|
||||
valid_sort_columns: Final = ["team_id", "team_alias", "created_at"]
|
||||
order_by = None
|
||||
if sort_by and sort_by in valid_sort_columns:
|
||||
if sort_order.lower() not in ["asc", "desc"]:
|
||||
sort_order = "asc"
|
||||
order_by = {sort_by: sort_order.lower()}
|
||||
from litellm.proxy.management_helpers.team_list_sorting import (
|
||||
COMPUTED_TEAM_SORTS,
|
||||
UnsupportedTeamFilter,
|
||||
sorted_team_ids,
|
||||
)
|
||||
|
||||
valid_sort_columns: Final = (
|
||||
"team_id",
|
||||
"team_alias",
|
||||
"created_at",
|
||||
"spend",
|
||||
"max_budget",
|
||||
"tpm_limit",
|
||||
"rpm_limit",
|
||||
"updated_at",
|
||||
)
|
||||
sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc"
|
||||
nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit")
|
||||
order_by: Final = (
|
||||
(
|
||||
{sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}},
|
||||
{"team_id": "asc"},
|
||||
)
|
||||
if nullable_sort
|
||||
else ({sort_by: sort_direction}, {"team_id": "asc"})
|
||||
if sort_by in ("spend", "updated_at")
|
||||
else {sort_by: sort_direction}
|
||||
if sort_by in valid_sort_columns
|
||||
else None
|
||||
)
|
||||
computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS
|
||||
sorted_ids: Final = (
|
||||
await sorted_team_ids(
|
||||
prisma_client,
|
||||
where=where_conditions,
|
||||
sort_by=sort_by or "",
|
||||
descending=sort_direction == "desc",
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
)
|
||||
if computed_sort
|
||||
else ()
|
||||
)
|
||||
|
||||
if isinstance(sorted_ids, UnsupportedTeamFilter):
|
||||
raise HTTPException(status_code=500, detail="Unsupported team sorting filter")
|
||||
|
||||
# Get teams with pagination
|
||||
if use_deleted_table:
|
||||
|
|
@ -6130,26 +6169,32 @@ async def list_team_v2(
|
|||
total_count = await _deleted_team_db(prisma_client).count(where=where_conditions)
|
||||
else:
|
||||
teams = await _team_db(prisma_client).find_many(
|
||||
where=where_conditions,
|
||||
skip=skip,
|
||||
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
|
||||
where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions,
|
||||
skip=0 if computed_sort else skip,
|
||||
take=page_size,
|
||||
order=order_by if order_by else {"created_at": "desc"}, # Default sort
|
||||
order=order_by if order_by else {"created_at": "desc"},
|
||||
include=_INCLUDE_MODEL_TABLE,
|
||||
)
|
||||
# Get total count for pagination
|
||||
total_count = await _team_db(prisma_client).count(where=where_conditions)
|
||||
|
||||
positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)}
|
||||
sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams
|
||||
|
||||
# Calculate total pages
|
||||
total_pages: Final = -(-total_count // page_size) # Ceiling division
|
||||
|
||||
# Aggregate virtual-key counts per team for the current page. The deleted
|
||||
# table does not carry keys_count, so it is skipped.
|
||||
keys_count_by_team: dict[str, int] = {}
|
||||
if not use_deleted_table:
|
||||
keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams)
|
||||
keys_count_by_team: Final = (
|
||||
await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {}
|
||||
)
|
||||
|
||||
# Convert Prisma models to response models with members_count and keys_count
|
||||
team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team)
|
||||
team_list: Final = _convert_teams_to_response_models(
|
||||
sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team
|
||||
)
|
||||
|
||||
# Resolve resources inherited from access groups (single batch query)
|
||||
if not use_deleted_table:
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ from litellm.proxy.utils import (
|
|||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import SSOConfigRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.user_repository import UserRepository, free_sso_user_creation
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
|
||||
|
|
@ -1074,25 +1074,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token(
|
|||
)
|
||||
|
||||
|
||||
async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None:
|
||||
"""Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license."""
|
||||
if premium_user is True:
|
||||
return
|
||||
if prisma_client is None:
|
||||
raise ProxyException(
|
||||
message=CommonProxyErrors.db_not_connected_error.value,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="premium_user",
|
||||
code=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
billable_users: Final = await UserRepository(prisma_client).count_billable_users()
|
||||
if billable_users and billable_users > 5:
|
||||
raise ProxyException(
|
||||
message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="premium_user",
|
||||
code=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
class _SSOUserAdmissionError(ProxyException):
|
||||
pass
|
||||
|
||||
|
||||
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
|
||||
|
|
@ -1112,8 +1095,6 @@ async def google_login(
|
|||
from litellm.proxy.proxy_server import (
|
||||
cli_sso_session_cache,
|
||||
general_settings,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
saml_sso_cache,
|
||||
user_custom_ui_sso_sign_in_handler,
|
||||
)
|
||||
|
|
@ -1128,15 +1109,6 @@ async def google_login(
|
|||
if admin_ui_is_disabled and not is_cli_sso_login:
|
||||
return admin_ui_disabled()
|
||||
|
||||
####### Check if user is a Enterprise / Premium User #######
|
||||
if (
|
||||
microsoft_client_id is not None
|
||||
or google_client_id is not None
|
||||
or generic_client_id is not None
|
||||
or SAMLAuthHandler.is_saml_configured()
|
||||
):
|
||||
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
|
||||
|
||||
####### Detect DB + MASTER KEY in .env #######
|
||||
missing_env_vars: Final = show_missing_vars_in_env()
|
||||
if missing_env_vars is not None:
|
||||
|
|
@ -1963,6 +1935,8 @@ async def get_user_info_from_db(
|
|||
)
|
||||
|
||||
return user_info
|
||||
except _SSOUserAdmissionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
|
||||
verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e)
|
||||
|
|
@ -2328,7 +2302,6 @@ async def saml_callback(request: Request):
|
|||
general_settings,
|
||||
jwt_handler,
|
||||
master_key,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
saml_sso_cache,
|
||||
)
|
||||
|
|
@ -2352,8 +2325,6 @@ async def saml_callback(request: Request):
|
|||
|
||||
user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data)
|
||||
|
||||
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
|
||||
|
||||
ui_access_mode: Final = general_settings.get("ui_access_mode", None)
|
||||
relay_state: Final = post_data.get("RelayState")
|
||||
cp_return_to: Final[str | None] = (
|
||||
|
|
@ -2821,12 +2792,20 @@ async def insert_sso_user(
|
|||
if result_openid and hasattr(result_openid, "provider"):
|
||||
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
|
||||
|
||||
response: Final = await new_user(
|
||||
data=new_user_request,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
from litellm.proxy.proxy_server import premium_user
|
||||
|
||||
return response
|
||||
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
|
||||
try:
|
||||
return await new_user(
|
||||
data=new_user_request,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
except ProxyException as error:
|
||||
if premium_user is True:
|
||||
raise
|
||||
raise _SSOUserAdmissionError(
|
||||
message=error.message, type=error.type, param=error.param, code=error.code
|
||||
) from error
|
||||
|
||||
|
||||
DefaultTeamJoin: TypeAlias = Callable[
|
||||
|
|
@ -3567,6 +3546,8 @@ class SSOAuthenticationHandler:
|
|||
prisma_client=prisma_client,
|
||||
)
|
||||
return user_info
|
||||
except _SSOUserAdmissionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
if (
|
||||
user_info is None
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
"""
|
||||
Usage endpoints package.
|
||||
|
||||
Re-exports the router from endpoints module.
|
||||
"""
|
||||
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401
|
||||
router,
|
||||
)
|
||||
|
|
@ -1,627 +0,0 @@
|
|||
"""
|
||||
AI Usage Chat - uses LLM tool calling to answer questions about
|
||||
usage/spend data by querying the aggregated daily activity endpoints.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
|
||||
from datetime import date
|
||||
from typing import Final, Literal, NamedTuple, Protocol, cast, overload
|
||||
|
||||
from fastapi import HTTPException
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
)
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
USAGE_AI_TEMPERATURE: Final = 0.2
|
||||
|
||||
TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend"
|
||||
TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend"
|
||||
TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend"
|
||||
|
||||
ENTITY_FIELD_USER: Final = "user_id"
|
||||
ENTITY_FIELD_TEAM: Final = "team_id"
|
||||
ENTITY_FIELD_TAG: Final = "tag"
|
||||
|
||||
PAGINATED_PAGE_SIZE: Final = 200
|
||||
MAX_CHAT_MESSAGES: Final = 20
|
||||
TOP_N_MODELS: Final = 15
|
||||
TOP_N_PROVIDERS: Final = 10
|
||||
TOP_N_KEYS: Final = 10
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SSEStatusEvent(TypedDict):
|
||||
type: Literal["status"]
|
||||
message: str
|
||||
|
||||
|
||||
class SSEToolCallEvent(TypedDict, total=False):
|
||||
type: Literal["tool_call"]
|
||||
tool_name: str
|
||||
tool_label: str
|
||||
arguments: dict[str, str]
|
||||
status: Literal["running", "complete", "error"]
|
||||
error: str
|
||||
|
||||
|
||||
class SSEChunkEvent(TypedDict):
|
||||
type: Literal["chunk"]
|
||||
content: str
|
||||
|
||||
|
||||
class SSEDoneEvent(TypedDict):
|
||||
type: Literal["done"]
|
||||
|
||||
|
||||
class SSEErrorEvent(TypedDict):
|
||||
type: Literal["error"]
|
||||
message: str
|
||||
|
||||
|
||||
SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent
|
||||
|
||||
|
||||
class _EntityEntry(TypedDict, total=False):
|
||||
metrics: ReadOnly[Mapping[str, float]]
|
||||
metadata: ReadOnly[Mapping[str, str]]
|
||||
|
||||
|
||||
class _DayDump(TypedDict, total=False):
|
||||
breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]]
|
||||
|
||||
|
||||
class _EntityTotal(NamedTuple):
|
||||
"""Running per-entity totals accumulated while summarising a usage dump."""
|
||||
|
||||
alias: str
|
||||
spend: float
|
||||
requests: float
|
||||
tokens: float
|
||||
|
||||
|
||||
class _UsageDump(Protocol):
|
||||
@overload
|
||||
def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ...
|
||||
@overload
|
||||
def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ...
|
||||
|
||||
|
||||
class _ToolFunctionDef(TypedDict):
|
||||
name: ReadOnly[str]
|
||||
description: ReadOnly[str]
|
||||
parameters: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
class _ToolDef(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
function: ReadOnly[_ToolFunctionDef]
|
||||
|
||||
|
||||
class ToolHandler(TypedDict):
|
||||
fetch: Callable[..., Awaitable[_UsageDump]]
|
||||
summarise: Callable[[_UsageDump], str]
|
||||
label: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool definitions (OpenAI function-calling schema)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DATE_PARAMS: Final = {
|
||||
"start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"},
|
||||
"end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"},
|
||||
}
|
||||
|
||||
_TOOL_USAGE: Final[_ToolDef] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_usage_data",
|
||||
"description": (
|
||||
"Fetch aggregated global usage/spend data. Returns daily spend, "
|
||||
"token counts, request counts, and breakdowns by model, provider, "
|
||||
"and API key. Use for overall spend, top models, top providers."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
**_DATE_PARAMS,
|
||||
"user_id": {
|
||||
"type": "string",
|
||||
"description": "Optional user ID filter. Omit for global view.",
|
||||
},
|
||||
},
|
||||
"required": ["start_date", "end_date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_TOOL_TEAM: Final[_ToolDef] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_team_usage_data",
|
||||
"description": (
|
||||
"Fetch usage/spend data broken down by team. Use for questions "
|
||||
"like 'which team spends the most' or 'show me team X usage'."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
**_DATE_PARAMS,
|
||||
"team_ids": {
|
||||
"type": "string",
|
||||
"description": "Optional comma-separated team IDs. Omit for all teams.",
|
||||
},
|
||||
},
|
||||
"required": ["start_date", "end_date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_TOOL_TAG: Final[_ToolDef] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_tag_usage_data",
|
||||
"description": (
|
||||
"Fetch usage/spend data broken down by tag. Tags are labels "
|
||||
"attached to requests (features, environments, credentials)."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
**_DATE_PARAMS,
|
||||
"tags": {
|
||||
"type": "string",
|
||||
"description": "Optional comma-separated tag names. Omit for all tags.",
|
||||
},
|
||||
},
|
||||
"required": ["start_date", "end_date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
TOOLS_BASE: Final = [_TOOL_USAGE]
|
||||
TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG]
|
||||
|
||||
|
||||
def get_tools_for_role(is_admin: bool) -> list[_ToolDef]:
|
||||
"""Return the tool list appropriate for the user's role."""
|
||||
return TOOLS_ADMIN if is_admin else TOOLS_BASE
|
||||
|
||||
|
||||
_SYSTEM_PROMPT_BASE: Final = (
|
||||
"You are an AI assistant embedded in the LiteLLM Usage dashboard. "
|
||||
"You help users understand their LLM API spend and usage data.\n\n"
|
||||
"ALWAYS call the appropriate tool(s) first to fetch data before answering. "
|
||||
"You may call multiple tools if the question spans different dimensions.\n\n"
|
||||
"Guidelines:\n"
|
||||
"- Be concise and specific. Use exact numbers from the data.\n"
|
||||
"- Format costs as dollar amounts (e.g. $12.34).\n"
|
||||
"- When comparing entities, show a ranked list.\n"
|
||||
"- If data is empty or no results found, say so clearly.\n"
|
||||
"- Do not hallucinate data — only use what the tools return.\n"
|
||||
"- Today's date will be provided below. Use it to interpret relative dates "
|
||||
"like 'this week', 'this month', 'last 7 days', etc."
|
||||
)
|
||||
|
||||
_TOOL_DESCRIPTIONS_ADMIN: Final = (
|
||||
"You have access to these tools:\n"
|
||||
"- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n"
|
||||
"- `get_team_usage_data`: Team-level usage breakdown\n"
|
||||
"- `get_tag_usage_data`: Tag-level usage breakdown\n\n"
|
||||
)
|
||||
|
||||
_TOOL_DESCRIPTIONS_BASE: Final = (
|
||||
"You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _build_system_prompt(is_admin: bool) -> str:
|
||||
"""Build role-appropriate system prompt with today's date."""
|
||||
tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE
|
||||
return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}"
|
||||
|
||||
|
||||
# keep a public reference for test assertions
|
||||
SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data fetchers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_csv_ids(raw: str | None) -> list[str] | None:
|
||||
if not raw:
|
||||
return None
|
||||
return [t.strip() for t in raw.split(",") if t.strip()]
|
||||
|
||||
|
||||
async def _query_activity(
|
||||
table_name: str,
|
||||
entity_id_field: str,
|
||||
entity_id: str | list[str] | None,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
*,
|
||||
use_aggregated: bool = False,
|
||||
) -> SpendAnalyticsPaginatedResponse:
|
||||
"""Shared helper that calls the daily activity query layer."""
|
||||
from litellm.proxy.management_endpoints.common_daily_activity import (
|
||||
daily_activity_repository,
|
||||
daily_activity_scope,
|
||||
get_daily_activity,
|
||||
get_daily_activity_aggregated,
|
||||
)
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if use_aggregated:
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": CommonProxyErrors.db_not_connected_error.value},
|
||||
)
|
||||
repository: Final = daily_activity_repository(prisma_client)
|
||||
scope: Final = daily_activity_scope(
|
||||
table_name,
|
||||
entity_id_field,
|
||||
entity_id,
|
||||
None,
|
||||
None,
|
||||
start_date,
|
||||
end_date,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
return await get_daily_activity_aggregated(
|
||||
repository,
|
||||
scope,
|
||||
)
|
||||
return await get_daily_activity(
|
||||
prisma_client=prisma_client,
|
||||
table_name=table_name,
|
||||
entity_id_field=entity_id_field,
|
||||
entity_id=entity_id,
|
||||
entity_metadata_field=None,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
model=None,
|
||||
api_key=None,
|
||||
page=1,
|
||||
page_size=PAGINATED_PAGE_SIZE,
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump:
|
||||
resp: Final = await _query_activity(
|
||||
TABLE_DAILY_USER_SPEND,
|
||||
ENTITY_FIELD_USER,
|
||||
user_id,
|
||||
start_date,
|
||||
end_date,
|
||||
use_aggregated=True,
|
||||
)
|
||||
return resp.model_dump(mode="json")
|
||||
|
||||
|
||||
async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump:
|
||||
resp: Final = await _query_activity(
|
||||
TABLE_DAILY_TEAM_SPEND,
|
||||
ENTITY_FIELD_TEAM,
|
||||
_parse_csv_ids(team_ids),
|
||||
start_date,
|
||||
end_date,
|
||||
)
|
||||
return resp.model_dump(mode="json")
|
||||
|
||||
|
||||
async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump:
|
||||
resp: Final = await _query_activity(
|
||||
TABLE_DAILY_TAG_SPEND,
|
||||
ENTITY_FIELD_TAG,
|
||||
_parse_csv_ids(tags),
|
||||
start_date,
|
||||
end_date,
|
||||
)
|
||||
return resp.model_dump(mode="json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summarisers — convert raw JSON to concise text the LLM can reason over
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _accumulate_breakdown(
|
||||
results: Sequence[_DayDump], dimension: str, fields: Sequence[str]
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""Aggregate a single breakdown dimension across days."""
|
||||
totals: Final[dict[str, dict[str, float]]] = {}
|
||||
for day in results:
|
||||
for key, entry in day.get("breakdown", {}).get(dimension, {}).items():
|
||||
if key not in totals:
|
||||
totals[key] = {f: 0.0 for f in fields}
|
||||
m = entry.get("metrics", {})
|
||||
for f in fields:
|
||||
totals[key][f] += m.get(f, 0)
|
||||
return totals
|
||||
|
||||
|
||||
def _ranked_lines(
|
||||
totals: dict[str, dict[str, float]],
|
||||
fmt: Callable[[str, dict[str, float]], str],
|
||||
limit: int,
|
||||
) -> list[str]:
|
||||
"""Sort by spend descending, format each entry, and truncate."""
|
||||
return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]]
|
||||
|
||||
|
||||
def _summarise_usage_data(data: _UsageDump) -> str:
|
||||
meta: Final = data.get("metadata", {})
|
||||
results: Final = data.get("results", [])
|
||||
|
||||
header: Final = (
|
||||
f"Total Spend: ${meta.get('total_spend', 0):.4f}\n"
|
||||
f"Total Requests: {meta.get('total_api_requests', 0)}\n"
|
||||
f"Successful: {meta.get('total_successful_requests', 0)} | "
|
||||
f"Failed: {meta.get('total_failed_requests', 0)}\n"
|
||||
f"Total Tokens: {meta.get('total_tokens', 0)}"
|
||||
)
|
||||
|
||||
models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"])
|
||||
providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"])
|
||||
|
||||
model_lines: Final = _ranked_lines(
|
||||
models,
|
||||
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)",
|
||||
TOP_N_MODELS,
|
||||
)
|
||||
provider_lines: Final = _ranked_lines(
|
||||
providers,
|
||||
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)",
|
||||
TOP_N_PROVIDERS,
|
||||
)
|
||||
|
||||
sections = [header, ""]
|
||||
sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""]
|
||||
sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"])
|
||||
return "\n".join(sections)
|
||||
|
||||
|
||||
def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str:
|
||||
"""Summarise team/tag entity usage data."""
|
||||
results: Final = data.get("results", [])
|
||||
if not results:
|
||||
return f"No {entity_label} usage data found for the given date range."
|
||||
|
||||
totals: Final[dict[str, _EntityTotal]] = {}
|
||||
for day in results:
|
||||
for eid, entry in day.get("breakdown", {}).get("entities", {}).items():
|
||||
previous = totals.get(eid)
|
||||
m = entry.get("metrics", {})
|
||||
totals[eid] = _EntityTotal(
|
||||
alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid),
|
||||
spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0),
|
||||
requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0),
|
||||
tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0),
|
||||
)
|
||||
|
||||
lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""]
|
||||
for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend):
|
||||
label = d.alias if d.alias != eid else eid
|
||||
lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool dispatch registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TOOL_HANDLERS: Final[dict[str, ToolHandler]] = {
|
||||
"get_usage_data": ToolHandler(
|
||||
fetch=_fetch_usage_data,
|
||||
summarise=_summarise_usage_data,
|
||||
label="global usage data",
|
||||
),
|
||||
"get_team_usage_data": ToolHandler(
|
||||
fetch=_fetch_team_usage_data,
|
||||
summarise=lambda data: _summarise_entity_data(data, "Team"),
|
||||
label="team usage data",
|
||||
),
|
||||
"get_tag_usage_data": ToolHandler(
|
||||
fetch=_fetch_tag_usage_data,
|
||||
summarise=lambda data: _summarise_entity_data(data, "Tag"),
|
||||
label="tag usage data",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SSE streaming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _sse(event: SSEEvent) -> str:
|
||||
return f"data: {json.dumps(event)}\n\n"
|
||||
|
||||
|
||||
def _resolve_fetch_kwargs(
|
||||
fn_name: str,
|
||||
fn_args: Mapping[str, str],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
) -> dict[str, str]:
|
||||
"""Build keyword arguments for a tool's fetch function."""
|
||||
start_date: Final = fn_args.get("start_date", "")
|
||||
end_date: Final = fn_args.get("end_date", "")
|
||||
if not start_date or not end_date:
|
||||
raise ValueError("Missing required start_date or end_date from tool arguments")
|
||||
kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date}
|
||||
if fn_name == "get_usage_data":
|
||||
if not is_admin:
|
||||
if user_id is None:
|
||||
# Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
|
||||
# should have already rejected this. If we ever reach here it means
|
||||
# a future caller invoked the helper without scoping — fail loudly
|
||||
# rather than issuing an unfiltered global query.
|
||||
raise ValueError(
|
||||
"Non-admin caller has user_id=None; refusing to issue an "
|
||||
"unscoped query. Endpoint-level guard missing."
|
||||
)
|
||||
kwargs["user_id"] = user_id
|
||||
elif fn_args.get("user_id"):
|
||||
kwargs["user_id"] = fn_args["user_id"]
|
||||
elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"):
|
||||
kwargs["team_ids"] = fn_args["team_ids"]
|
||||
elif fn_name == "get_tag_usage_data" and fn_args.get("tags"):
|
||||
kwargs["tags"] = fn_args["tags"]
|
||||
return kwargs
|
||||
|
||||
|
||||
async def _execute_tool_call(
|
||||
handler: ToolHandler,
|
||||
fn_name: str,
|
||||
fn_args: Mapping[str, str],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
) -> str:
|
||||
"""Run a single tool and return the summarised result text, or the reason the query
|
||||
layer rejected its dates so the model can correct them."""
|
||||
kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin)
|
||||
try:
|
||||
raw_data: Final = await handler["fetch"](**kwargs)
|
||||
except HTTPException as e:
|
||||
reason: Final = _invalid_date_reason(e)
|
||||
if reason is None:
|
||||
raise
|
||||
return f"Invalid date range: {reason}"
|
||||
return handler["summarise"](raw_data)
|
||||
|
||||
|
||||
def _invalid_date_reason(error: HTTPException) -> str | None:
|
||||
"""The reason in the daily activity query layer's 400 for invalid or reversed dates."""
|
||||
detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict
|
||||
match detail:
|
||||
case {"error": str() as reason} if error.status_code == 400:
|
||||
return reason
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
async def _process_tool_call(
|
||||
tc: ChatCompletionMessageToolCall,
|
||||
chat_messages: list[Mapping[str, object]],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Execute a single tool call, yielding SSE events for status."""
|
||||
fn_name: Final = tc.function.name
|
||||
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
|
||||
|
||||
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
|
||||
handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None
|
||||
|
||||
if fn_name is None or fn_name not in allowed_names or not handler:
|
||||
chat_messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": f"Tool not available: {fn_name}",
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
tool_event_base: Final = {
|
||||
"type": "tool_call",
|
||||
"tool_name": fn_name,
|
||||
"tool_label": handler["label"],
|
||||
"arguments": fn_args,
|
||||
}
|
||||
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"}))
|
||||
|
||||
try:
|
||||
tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin)
|
||||
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"}))
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e)
|
||||
tool_result = f"Error fetching {handler['label']}. Please try again."
|
||||
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"}))
|
||||
|
||||
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
|
||||
|
||||
|
||||
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
|
||||
"""Stream the final LLM response after tool results are appended."""
|
||||
yield _sse({"type": "status", "message": "Analyzing results..."})
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
model=model,
|
||||
messages=chat_messages,
|
||||
stream=True,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
)
|
||||
async for chunk in response:
|
||||
delta = chunk.choices[0].delta.content
|
||||
if delta:
|
||||
yield _sse({"type": "chunk", "content": delta})
|
||||
|
||||
|
||||
async def stream_usage_ai_chat(
|
||||
messages: list[dict[str, str]],
|
||||
model: str | None = None,
|
||||
user_id: str | None = None,
|
||||
is_admin: bool = False,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Stream SSE events: status → tool_call → chunk → done."""
|
||||
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
|
||||
chat_messages: Final[list[Mapping[str, object]]] = [
|
||||
{"role": "system", "content": _build_system_prompt(is_admin)},
|
||||
*truncated,
|
||||
]
|
||||
|
||||
try:
|
||||
yield _sse({"type": "status", "message": "Thinking..."})
|
||||
tools: Final = get_tools_for_role(is_admin)
|
||||
response: Final = await litellm.acompletion(
|
||||
model=resolved_model,
|
||||
messages=chat_messages,
|
||||
tools=tools,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
)
|
||||
choice: Final = response.choices[0]
|
||||
|
||||
if not choice.message.tool_calls:
|
||||
if choice.message.content:
|
||||
yield _sse({"type": "chunk", "content": choice.message.content})
|
||||
yield _sse({"type": "done"})
|
||||
return
|
||||
|
||||
chat_messages.append(choice.message.model_dump())
|
||||
for tc in choice.message.tool_calls:
|
||||
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
|
||||
yield event
|
||||
async for event in _stream_final_response(resolved_model, chat_messages):
|
||||
yield event
|
||||
yield _sse({"type": "done"})
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("AI usage chat failed: %s", e)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "error",
|
||||
"message": "An internal error occurred. Please try again.",
|
||||
}
|
||||
)
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
"""
|
||||
USAGE AI CHAT ENDPOINTS
|
||||
|
||||
/usage/ai/chat - Stream AI chat responses about usage data
|
||||
"""
|
||||
|
||||
from typing import Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import Field
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING,
|
||||
wrap_sse_stream_with_keepalive_pings,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
||||
class ChatMessage(LiteLLMBaseModel):
|
||||
role: Literal["user", "assistant"]
|
||||
content: str
|
||||
|
||||
|
||||
class UsageAIChatRequest(LiteLLMBaseModel):
|
||||
messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)")
|
||||
model: str | None = Field(default=None, description="Model to use for AI chat")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/usage/ai/chat",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def usage_ai_chat(
|
||||
data: UsageAIChatRequest,
|
||||
request: Request,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
AI chat about usage data. Streams SSE events with the AI response.
|
||||
The AI agent has access to tools that query aggregated daily activity data.
|
||||
"""
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
require_caller_user_id_for_non_admin,
|
||||
user_api_key_has_admin_view,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
stream_usage_ai_chat,
|
||||
)
|
||||
|
||||
is_admin: Final = user_api_key_has_admin_view(user_api_key_dict)
|
||||
if is_admin:
|
||||
user_id = user_api_key_dict.user_id
|
||||
else:
|
||||
user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
|
||||
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
|
||||
|
||||
return StreamingResponse(
|
||||
wrap_sse_stream_with_keepalive_pings(
|
||||
stream_usage_ai_chat(
|
||||
messages=messages,
|
||||
model=data.model,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
),
|
||||
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
|
||||
ping_chunk=SSE_COMMENT_PING,
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
109
litellm/proxy/management_helpers/team_list_sorting.py
Normal file
109
litellm/proxy/management_helpers/team_list_sorting.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
|
||||
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
COMPUTED_TEAM_SORTS: Final = MappingProxyType(
|
||||
{
|
||||
"organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') "
|
||||
'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), '
|
||||
"NULLIF(t.organization_id, ''))",
|
||||
"members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' "
|
||||
"THEN jsonb_array_length(t.members_with_roles) ELSE 0 END",
|
||||
"models": "cardinality(t.models)",
|
||||
"keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)',
|
||||
}
|
||||
)
|
||||
_WHERE: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnsupportedTeamFilter:
|
||||
field: str
|
||||
|
||||
|
||||
def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
|
||||
if field not in ("team_id", "team_alias", "organization_id"):
|
||||
return UnsupportedTeamFilter(field)
|
||||
column: Final = f't."{field}"'
|
||||
match value:
|
||||
case str(text):
|
||||
return f"{column} = ${index}", (text,)
|
||||
case {"in": list(values)}:
|
||||
return f"{column} = ANY(${index}::text[])", (values,)
|
||||
case {"contains": str(text), "mode": "insensitive"}:
|
||||
return f"{column} ILIKE ${index}", (f"%{text}%",)
|
||||
case {"startsWith": str(text)}:
|
||||
return f"{column} LIKE ${index}", (f"{text}%",)
|
||||
case _:
|
||||
return UnsupportedTeamFilter(field)
|
||||
|
||||
|
||||
def _group_tokens(
|
||||
field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue]
|
||||
) -> tuple[str | tuple[str, JsonValue], ...]:
|
||||
children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value)
|
||||
if not children:
|
||||
return ("FALSE" if field == "OR" else "TRUE",)
|
||||
return (
|
||||
"(",
|
||||
*chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)),
|
||||
")",
|
||||
)
|
||||
|
||||
|
||||
def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
|
||||
pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal
|
||||
fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying
|
||||
params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order
|
||||
while pending:
|
||||
match pending.pop():
|
||||
case str(fragment):
|
||||
fragments.append(fragment)
|
||||
case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)):
|
||||
pending.extend(reversed(_group_tokens(field, value)))
|
||||
case (field, value):
|
||||
match _field_sql(field, value, len(params) + 1):
|
||||
case UnsupportedTeamFilter() as failure:
|
||||
return failure
|
||||
case (fragment, values):
|
||||
fragments.append(fragment)
|
||||
params.extend(values)
|
||||
return "".join(fragments), tuple(params)
|
||||
|
||||
|
||||
class _TeamId(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
team_id: str
|
||||
|
||||
|
||||
_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...])
|
||||
|
||||
|
||||
async def sorted_team_ids(
|
||||
prisma_client: PrismaClient,
|
||||
*,
|
||||
where: Mapping[str, object],
|
||||
sort_by: str,
|
||||
descending: bool,
|
||||
skip: int,
|
||||
take: int,
|
||||
) -> Sequence[str] | UnsupportedTeamFilter:
|
||||
result: Final = _where_sql(_WHERE.validate_python(where))
|
||||
if isinstance(result, UnsupportedTeamFilter):
|
||||
return result
|
||||
predicate, params = result
|
||||
expression: Final = COMPUTED_TEAM_SORTS[sort_by]
|
||||
direction: Final = "DESC" if descending else "ASC"
|
||||
sql: Final = (
|
||||
f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} '
|
||||
f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC "
|
||||
f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
|
||||
)
|
||||
rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip))
|
||||
return tuple(row.team_id for row in rows)
|
||||
77
litellm/proxy/management_helpers/user_list_sorting.py
Normal file
77
litellm/proxy/management_helpers/user_list_sorting.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
from collections.abc import Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
class _UserId(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
user_id: str
|
||||
|
||||
|
||||
_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...])
|
||||
|
||||
|
||||
async def sorted_user_ids(
|
||||
prisma_client: PrismaClient,
|
||||
*,
|
||||
sort_by: Literal["status", "key_count"],
|
||||
descending: bool,
|
||||
skip: int,
|
||||
take: int,
|
||||
role: str | None,
|
||||
user_ids: str | None,
|
||||
user_email: str | None,
|
||||
team: str | None,
|
||||
sso_user_ids: str | None,
|
||||
organization_ids: str | None,
|
||||
search: str | None,
|
||||
) -> Sequence[str]:
|
||||
ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip())
|
||||
sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip())
|
||||
org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip())
|
||||
expression: Final = (
|
||||
"CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END"
|
||||
if sort_by == "status"
|
||||
else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k '
|
||||
"WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)"
|
||||
)
|
||||
direction: Final = "DESC" if descending else "ASC"
|
||||
sql: Final = f"""
|
||||
SELECT u.user_id FROM "LiteLLM_UserTable" u
|
||||
WHERE ($1::text IS NULL OR u.user_role = $1)
|
||||
AND ($2::text[] IS NULL
|
||||
OR CASE WHEN cardinality($2::text[]) = 1
|
||||
THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%'
|
||||
ELSE u.user_id = ANY($2::text[]) END)
|
||||
AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%')
|
||||
AND ($4::text IS NULL OR $4 = ANY(u.teams))
|
||||
AND ($5::boolean OR u.sso_user_id = ANY($6::text[]))
|
||||
AND (cardinality($7::text[]) = 0 OR EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_OrganizationMembership" m
|
||||
WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[])))
|
||||
AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%'
|
||||
OR u.user_email ILIKE '%' || $8 || '%')
|
||||
ORDER BY {expression} {direction}, u.user_id ASC
|
||||
LIMIT $9 OFFSET $10
|
||||
"""
|
||||
rows: Final = _USER_IDS.validate_python(
|
||||
await prisma_client.db.query_raw(
|
||||
sql,
|
||||
role or None,
|
||||
ids if user_ids else None,
|
||||
user_email,
|
||||
team,
|
||||
sso_user_ids is None,
|
||||
sso_ids,
|
||||
org_ids,
|
||||
search or None,
|
||||
take,
|
||||
skip,
|
||||
*((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()),
|
||||
)
|
||||
)
|
||||
return tuple(row.user_id for row in rows)
|
||||
|
|
@ -12210,7 +12210,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
|
|||
async def model_list(
|
||||
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
return_wildcard_routes: bool | None = False,
|
||||
return_wildcard_routes: bool | None = None,
|
||||
team_id: str | None = None,
|
||||
include_model_access_groups: bool | None = False,
|
||||
only_model_access_groups: bool | None = False,
|
||||
|
|
@ -12226,6 +12226,9 @@ async def model_list(
|
|||
This is just for compatibility with openai projects like aider.
|
||||
|
||||
Query Parameters:
|
||||
- return_wildcard_routes: When true, also list wildcard routes such as `openai/*`.
|
||||
Defaults to `general_settings.return_wildcard_routes`, and an explicit
|
||||
value on the request wins over it
|
||||
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
|
||||
provider's model catalog. When present, the response is Codex's own
|
||||
catalog shape (`{"models": [...]}`) built from the same listing, with
|
||||
|
|
@ -12254,6 +12257,9 @@ async def model_list(
|
|||
global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
|
||||
|
||||
settings: Final = cast(dict[str, object], general_settings) # any-ok: legacy settings
|
||||
include_wildcard_routes: Final = (
|
||||
return_wildcard_routes if return_wildcard_routes is not None else settings.get("return_wildcard_routes") is True
|
||||
)
|
||||
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
create_anthropic_model_list_response,
|
||||
|
|
@ -12343,7 +12349,7 @@ async def model_list(
|
|||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=return_wildcard_routes or False,
|
||||
return_wildcard_routes=include_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups or False,
|
||||
|
|
@ -12411,7 +12417,7 @@ async def model_list(
|
|||
team_id=team_id,
|
||||
include_model_access_groups=include_model_access_groups or False,
|
||||
only_model_access_groups=only_model_access_groups or False,
|
||||
return_wildcard_routes=return_wildcard_routes or False,
|
||||
return_wildcard_routes=include_wildcard_routes,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
|
||||
|
|
@ -19657,6 +19663,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro
|
|||
"proxy_config_reload_interval_seconds": "Integer",
|
||||
"pass_through_endpoints": "PydanticModel",
|
||||
"store_model_in_db": "Boolean",
|
||||
"return_wildcard_routes": "Boolean",
|
||||
"store_prompts_in_spend_logs": "Boolean",
|
||||
"disable_requester_ip_address_logging": "Boolean",
|
||||
"disable_fallbacks_on_per_model_rate_limits": "Boolean",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import (
|
|||
get_blog_posts,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.proxy._types import (
|
||||
CommonProxyErrors,
|
||||
)
|
||||
|
|
@ -564,7 +565,7 @@ async def _resolve_autorouter_presets(
|
|||
url: str,
|
||||
fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],
|
||||
) -> Mapping[str, AutoRouterPresetRecord]:
|
||||
if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
|
||||
return _load_bundled_autorouter_presets()
|
||||
try:
|
||||
return await fetch(url)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import Counter
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -12,12 +14,14 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest"
|
||||
CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/"
|
||||
CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml"
|
||||
RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}"
|
||||
LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5
|
||||
LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60
|
||||
|
|
@ -26,7 +30,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info"
|
|||
|
||||
_RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S")
|
||||
_NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b")
|
||||
_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
|
||||
_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/")
|
||||
|
||||
_Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"]
|
||||
_PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"})
|
||||
|
|
@ -45,6 +49,16 @@ class LatestReleaseUnavailable:
|
|||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChangelogEntry:
|
||||
version: tuple[int, int, int]
|
||||
url: str
|
||||
|
||||
@property
|
||||
def tag(self) -> str:
|
||||
return "v" + ".".join(str(part) for part in self.version)
|
||||
|
||||
|
||||
class _GitHubRelease(LiteLLMBaseModel):
|
||||
tag_name: str
|
||||
html_url: str
|
||||
|
|
@ -76,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock:
|
|||
return _latest_release_fetch_lock
|
||||
|
||||
|
||||
def _default_release_check_enabled() -> bool:
|
||||
return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true"
|
||||
|
||||
|
||||
def _bucket_for(line: str) -> _Bucket | None:
|
||||
if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None:
|
||||
return None
|
||||
|
|
@ -91,9 +109,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]:
|
|||
return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None))
|
||||
|
||||
|
||||
def _stable_version(tag_name: str) -> tuple[int, int, int] | None:
|
||||
match: Final = _STABLE_TAG_PATTERN.match(tag_name)
|
||||
return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||||
def _changelog_entry(link: str) -> ChangelogEntry | None:
|
||||
match: Final = _CHANGELOG_PAGE_PATTERN.match(link)
|
||||
if match is None:
|
||||
return None
|
||||
return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link)
|
||||
|
||||
|
||||
def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None:
|
||||
"""The highest stable release with a page in the changelog feed. Release candidate pages don't count."""
|
||||
links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link"))
|
||||
entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None)
|
||||
return max(entries, key=lambda entry: entry.version, default=None)
|
||||
|
||||
|
||||
def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
|
||||
|
|
@ -102,7 +129,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
|
|||
return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch))
|
||||
|
||||
|
||||
def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo:
|
||||
def summarize_release_line(
|
||||
entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]
|
||||
) -> LatestReleaseInfo:
|
||||
"""A patch release's notes only list its backports, so count the whole minor line it belongs to."""
|
||||
counts: Final = sum(
|
||||
(Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)),
|
||||
|
|
@ -113,7 +142,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease
|
|||
new_features=counts.get("new_features", 0),
|
||||
bug_fixes=counts.get("bug_fixes", 0),
|
||||
other_updates=counts.get("other_updates", 0),
|
||||
release_url=latest.html_url,
|
||||
release_url=entry.url,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -137,25 +166,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe
|
|||
return release
|
||||
|
||||
|
||||
async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable:
|
||||
response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
|
||||
if response.status_code != 200:
|
||||
return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}")
|
||||
try:
|
||||
entry: Final = latest_changelog_entry(response.content)
|
||||
except ET.ParseError as e:
|
||||
return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}")
|
||||
return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry
|
||||
|
||||
|
||||
async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable:
|
||||
try:
|
||||
entry: Final = await _fetch_changelog_entry(client)
|
||||
if isinstance(entry, LatestReleaseUnavailable):
|
||||
return entry
|
||||
latest: Final = parse_github_release(
|
||||
await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
|
||||
await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
|
||||
)
|
||||
if isinstance(latest, LatestReleaseUnavailable):
|
||||
return latest
|
||||
version: Final = _stable_version(latest.tag_name)
|
||||
if version is None:
|
||||
return summarize_release_line(latest, ())
|
||||
earlier: Final = await asyncio.gather(
|
||||
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version))
|
||||
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version))
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}")
|
||||
unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None)
|
||||
if unavailable is not None:
|
||||
return unavailable
|
||||
return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)))
|
||||
return summarize_release_line(
|
||||
entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))
|
||||
)
|
||||
|
||||
|
||||
async def get_latest_release_info(
|
||||
|
|
@ -188,12 +230,16 @@ async def latest_release_info(
|
|||
client: Annotated[_AsyncGetClient, Depends(_default_client)],
|
||||
cache: Annotated[InMemoryCache, Depends(_default_cache)],
|
||||
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
|
||||
release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)],
|
||||
) -> LatestReleaseInfo | None:
|
||||
"""
|
||||
Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
|
||||
split into new features, bug fixes and other updates.
|
||||
Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
|
||||
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
|
||||
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
|
||||
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
|
||||
Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request.
|
||||
"""
|
||||
if not release_check_enabled:
|
||||
return None
|
||||
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
|
||||
if isinstance(result, LatestReleaseUnavailable):
|
||||
verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason)
|
||||
|
|
|
|||
|
|
@ -6032,13 +6032,12 @@ class PrismaClient:
|
|||
elif table_name == "user":
|
||||
db_data = self.jsonify_object(data=data)
|
||||
try:
|
||||
new_user_row: Final = await UserRepository(self).table.upsert(
|
||||
where={"user_id": data["user_id"]},
|
||||
data={
|
||||
"create": {**db_data},
|
||||
"update": {}, # don't do anything if it already exists
|
||||
},
|
||||
)
|
||||
new_user_row: Final = await UserRepository(self).upsert_user(db_data)
|
||||
if new_user_row is None:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.",
|
||||
)
|
||||
except Exception as e:
|
||||
if (
|
||||
"Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`"
|
||||
|
|
|
|||
|
|
@ -2,8 +2,11 @@
|
|||
User repository for database operations on LiteLLM_UserTable.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from contextlib import AbstractAsyncContextManager, contextmanager
|
||||
from contextvars import ContextVar
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
|
|
@ -35,6 +38,16 @@ ORDER BY p.user_id
|
|||
"""
|
||||
|
||||
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
|
||||
_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]:
|
||||
token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_FREE_SSO_CREATION.reset(token)
|
||||
|
||||
|
||||
class _UserDb(Protocol):
|
||||
|
|
@ -42,6 +55,12 @@ class _UserDb(Protocol):
|
|||
def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ...
|
||||
|
||||
|
||||
class _UserTransaction(_UserDb, Protocol):
|
||||
async def execute_raw(self, query: str) -> int: ...
|
||||
|
||||
async def query_raw(self, query: str) -> object: ...
|
||||
|
||||
|
||||
class _PrismaClientView(Protocol):
|
||||
@property
|
||||
def db(self) -> _UserDb: ...
|
||||
|
|
@ -49,6 +68,16 @@ class _PrismaClientView(Protocol):
|
|||
@property
|
||||
def writer_db(self) -> _UserDb: ...
|
||||
|
||||
def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ...
|
||||
|
||||
|
||||
async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int:
|
||||
from prisma import Json
|
||||
|
||||
total: Final = await table.count()
|
||||
deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
|
||||
return max(0, total - deactivated)
|
||||
|
||||
|
||||
class UserRepository(BaseRepository[LiteLLM_UserTable]):
|
||||
"""Repository for user database operations."""
|
||||
|
|
@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
|
|||
null, or true all count, so seats are counted as total users minus
|
||||
the deactivated ones.
|
||||
"""
|
||||
from prisma import Json # pyright: ignore[reportUnknownVariableType]
|
||||
return await _count_billable_users(self.table)
|
||||
|
||||
total: Final = await self.count()
|
||||
deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
|
||||
return max(0, total - deactivated)
|
||||
async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None":
|
||||
"""Return the user row, or None when a new free SSO user would exceed capacity."""
|
||||
scope: Final = _FREE_SSO_CREATION.get()
|
||||
if (
|
||||
scope is None
|
||||
or (scope[0] is not None and scope[0] != data["user_id"])
|
||||
or scope[1] is not asyncio.current_task()
|
||||
):
|
||||
return await self.table.upsert(
|
||||
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
|
||||
)
|
||||
|
||||
client: Final[_PrismaClientView] = self.prisma_client
|
||||
async with client.tx() as transaction:
|
||||
await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
|
||||
await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)")
|
||||
existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]})
|
||||
if existing is not None:
|
||||
return existing
|
||||
if await _count_billable_users(transaction.litellm_usertable) >= 5:
|
||||
return None
|
||||
return await transaction.litellm_usertable.upsert(
|
||||
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
|
||||
)
|
||||
|
||||
async def create_user(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -3663,18 +3663,20 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if count is not None
|
||||
}
|
||||
)
|
||||
response_usage.input_tokens_details = InputTokensDetails(
|
||||
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
|
||||
text_tokens=prompt_details.text_tokens,
|
||||
audio_tokens=prompt_details.audio_tokens,
|
||||
image_tokens=prompt_details.image_tokens,
|
||||
video_tokens=prompt_details.video_tokens,
|
||||
cached_tokens_details=(
|
||||
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
|
||||
),
|
||||
**cache_write_extra,
|
||||
**cache_type_extra,
|
||||
**grounding_request_counts,
|
||||
response_usage.input_tokens_details = InputTokensDetails.model_validate(
|
||||
{
|
||||
"cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
|
||||
"text_tokens": prompt_details.text_tokens,
|
||||
"audio_tokens": prompt_details.audio_tokens,
|
||||
"image_tokens": prompt_details.image_tokens,
|
||||
"video_tokens": prompt_details.video_tokens,
|
||||
"cached_tokens_details": (
|
||||
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
|
||||
),
|
||||
**cache_write_extra,
|
||||
**cache_type_extra,
|
||||
**grounding_request_counts,
|
||||
}
|
||||
)
|
||||
|
||||
# Translate completion_tokens_details to output_tokens_details
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ from contextlib import contextmanager
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from typing_extensions import (
|
||||
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
|
||||
assert_never,
|
||||
|
|
@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
with_compaction_items_as_summaries,
|
||||
|
|
@ -85,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
|
|||
from litellm.types.responses.main import *
|
||||
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
|
||||
from litellm.types.utils import (
|
||||
ADDRESSED_RESPONSE_ID_FIELD,
|
||||
CallTypes,
|
||||
CostPerToken,
|
||||
CustomPricingLiteLLMParams,
|
||||
all_litellm_params,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
|
|
@ -544,6 +552,47 @@ async def aresponses_api_with_mcp(
|
|||
return response
|
||||
|
||||
|
||||
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
|
||||
return MappingProxyType(
|
||||
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
|
||||
)
|
||||
|
||||
|
||||
class _DeploymentModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
id: object = None
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
|
||||
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
|
||||
)
|
||||
|
||||
|
||||
def _is_router_deployment(deployment_model_info: object) -> bool:
|
||||
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
|
||||
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
|
||||
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
|
||||
|
||||
|
||||
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
|
||||
|
||||
|
||||
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
|
||||
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
|
||||
return None
|
||||
return _COST_PER_TOKEN.validate_python(
|
||||
litellm_params.model_dump(
|
||||
include={
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
},
|
||||
exclude_none=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _bridges_to_chat_completions(
|
||||
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
|
||||
) -> bool:
|
||||
|
|
@ -1466,6 +1515,12 @@ def responses(
|
|||
|
||||
# get llm provider logic
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
affinity_api_base: Final = (
|
||||
get_openai_credentials(api_base=litellm_params.api_base).api_base
|
||||
if custom_llm_provider == "openai"
|
||||
else litellm_params.api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
try:
|
||||
effective_extra_headers: Final = (
|
||||
add_provider_affinity_header(
|
||||
|
|
@ -1479,8 +1534,9 @@ def responses(
|
|||
"litellm_metadata": kwargs.get("litellm_metadata"),
|
||||
}
|
||||
),
|
||||
default_header=default_affinity_header,
|
||||
)
|
||||
if litellm_params.provider_affinity_header is not None
|
||||
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
|
||||
else extra_headers
|
||||
)
|
||||
except ValueError as affinity_error:
|
||||
|
|
@ -1664,6 +1720,7 @@ def responses(
|
|||
optional_params=dict(responses_api_request_params),
|
||||
litellm_params={
|
||||
**responses_api_request_params,
|
||||
**_per_call_custom_pricing(litellm_params),
|
||||
"aresponses": _is_async,
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": kwargs.get("model_info"),
|
||||
|
|
@ -1674,6 +1731,9 @@ def responses(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
if not _is_router_deployment(deployment_model_info):
|
||||
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
|
||||
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
|
||||
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
|
||||
input = with_compaction_items_as_summaries(
|
||||
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
|||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
InputTokensDetails,
|
||||
OutputTokensDetails,
|
||||
ResponseAPIUsage,
|
||||
ResponseInputParam,
|
||||
|
|
@ -23,6 +24,7 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.responses.main import DecodedResponseId
|
||||
from litellm.types.utils import (
|
||||
CacheCreationTokenDetails,
|
||||
CompletionTokensDetailsWrapper,
|
||||
PromptTokensDetailsWrapper,
|
||||
SpecialEnums,
|
||||
|
|
@ -41,6 +43,17 @@ def _output_token_detail(details: object, field: str) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
def _cache_write_lifetimes(cache_write_tokens_details: object) -> CacheCreationTokenDetails | None:
|
||||
"""OpenAI Responses splits `cache_write_tokens` into `ttl_30m_tokens` and `ttl_12h_tokens`."""
|
||||
ttl_30m_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_30m_tokens")
|
||||
ttl_12h_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_12h_tokens")
|
||||
if ttl_30m_tokens is None and ttl_12h_tokens is None:
|
||||
return None
|
||||
return CacheCreationTokenDetails(
|
||||
ephemeral_30m_input_tokens=ttl_30m_tokens, ephemeral_12h_input_tokens=ttl_12h_tokens
|
||||
)
|
||||
|
||||
|
||||
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
|
||||
return isinstance(value, list)
|
||||
|
||||
|
|
@ -1247,7 +1260,11 @@ class ResponseAPILoggingUtils:
|
|||
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
|
||||
if response_api_usage.input_tokens_details:
|
||||
if isinstance(response_api_usage.input_tokens_details, dict):
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details)
|
||||
input_details: Final = InputTokensDetails.model_validate(response_api_usage.input_tokens_details)
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
**input_details.model_dump(exclude={"cache_write_tokens_details"}),
|
||||
cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details),
|
||||
)
|
||||
else:
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
|
||||
|
|
@ -1259,6 +1276,9 @@ class ResponseAPILoggingUtils:
|
|||
),
|
||||
video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None),
|
||||
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
|
||||
cache_creation_token_details=_cache_write_lifetimes(
|
||||
getattr(response_api_usage.input_tokens_details, "cache_write_tokens_details", None)
|
||||
),
|
||||
cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None),
|
||||
web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None),
|
||||
google_maps_grounding_requests=getattr(
|
||||
|
|
|
|||
|
|
@ -14186,6 +14186,43 @@ class Router:
|
|||
deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool:
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
declared: Final = model_info.get("supports_vision")
|
||||
if declared is not None:
|
||||
return declared is False
|
||||
raw_params: Final = plain_deployment.get("litellm_params") or {}
|
||||
params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts
|
||||
if not params:
|
||||
return False
|
||||
base_model: Final = model_info.get("base_model") or params.get("base_model")
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return litellm.utils.is_vision_explicitly_disabled(
|
||||
base_model if isinstance(base_model, str) else model, custom_llm_provider
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
|
||||
)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _request_carries_image(
|
||||
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
|
||||
|
||||
structured: Final = resolve_structured_messages(
|
||||
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
|
||||
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
|
||||
)
|
||||
return bool(structured) and request_contains_image_content(structured)
|
||||
|
||||
@staticmethod
|
||||
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
|
||||
requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
|
||||
|
|
@ -14212,6 +14249,7 @@ class Router:
|
|||
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
|
||||
- filter models above rpm limits
|
||||
- if region given, filter out models not in that region / unknown region
|
||||
- request carries an image and the deployment declares supports_vision false, unless every candidate does
|
||||
- [TODO] function call and model doesn't support function calling
|
||||
"""
|
||||
|
||||
|
|
@ -14221,7 +14259,7 @@ class Router:
|
|||
# We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
|
||||
_returned_deployments = list(healthy_deployments)
|
||||
|
||||
invalid_model_indices: Final = set() # Use set for O(1) membership checks
|
||||
invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks
|
||||
|
||||
_context_window_error = False
|
||||
_potential_error_str = ""
|
||||
|
|
@ -14371,9 +14409,20 @@ class Router:
|
|||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
if len(excluded_indices) > 0:
|
||||
remaining: Final = tuple(
|
||||
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
|
||||
)
|
||||
text_only_indices: Final[frozenset[int]] = (
|
||||
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
|
||||
if remaining and self._request_carries_image(messages, request_settings)
|
||||
else frozenset()
|
||||
)
|
||||
dropped_indices: Final[frozenset[int]] = excluded_indices | (
|
||||
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
|
||||
)
|
||||
if len(dropped_indices) > 0:
|
||||
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
|
||||
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
|
||||
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
|
||||
|
||||
return _returned_deployments
|
||||
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
|
|||
}
|
||||
|
||||
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
|
||||
Feature.PROVIDER_AFFINITY,
|
||||
Feature.SAFEGUARDS_CLASSIFIER,
|
||||
Feature.PRE_REQUEST_HOOK,
|
||||
Feature.WEBSEARCH_INTERCEPTION,
|
||||
|
|
|
|||
|
|
@ -5,10 +5,15 @@ from __future__ import annotations
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.detectors import overridden, present, streaming
|
||||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
|
||||
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
|
||||
|
||||
|
||||
def provider_affinity(request: NativeCall) -> bool:
|
||||
return is_opencode_api_base(api_base(request))
|
||||
|
||||
|
||||
def anthropic_workload_identity(request: NativeCall) -> bool:
|
||||
"""Python mints an Anthropic federation token when no static credential is configured."""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present
|
||||
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
|
||||
|
||||
|
|
@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool:
|
|||
|
||||
def provider_affinity(request: NativeCall) -> bool:
|
||||
"""Python validates and fills the affinity header from the session ID."""
|
||||
return present(request, "provider_affinity_header")
|
||||
return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request))
|
||||
|
||||
|
||||
def text_format(request: NativeCall) -> bool:
|
||||
|
|
|
|||
0
litellm/text_completions/__init__.py
Normal file
0
litellm/text_completions/__init__.py
Normal file
88
litellm/text_completions/streaming_iterator.py
Normal file
88
litellm/text_completions/streaming_iterator.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
|
||||
|
||||
|
||||
class TextCompletionStreamWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
completion_stream,
|
||||
model,
|
||||
stream_options: dict | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
):
|
||||
self.completion_stream = completion_stream
|
||||
self.model = model
|
||||
self.stream_options = stream_options
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
def convert_to_text_completion_object(self, chunk: ModelResponse):
|
||||
try:
|
||||
response: Final = TextCompletionResponse()
|
||||
response["id"] = chunk.get("id", None)
|
||||
response["object"] = "text_completion"
|
||||
response["created"] = chunk.get("created", None)
|
||||
response["model"] = chunk.get("model", None)
|
||||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
response.usage = chunk.get("usage", None)
|
||||
else:
|
||||
response.usage = None
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
|
||||
|
||||
def __next__(self):
|
||||
# model_response = ModelResponse(stream=True, model=self.model)
|
||||
TextCompletionResponse()
|
||||
try:
|
||||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
except Exception as e:
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
|
||||
raise exception_type(
|
||||
model=self.model,
|
||||
custom_llm_provider=self.custom_llm_provider or "",
|
||||
original_exception=e,
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
async def __anext__(self):
|
||||
try:
|
||||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration
|
||||
|
|
@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[
|
|||
|
||||
DecisionQuestionMap: TypeAlias = Annotated[
|
||||
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
|
||||
Field(min_length=1, max_length=MAX_DECISION_QUESTIONS),
|
||||
Field(min_length=1),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,16 +9,38 @@ https://ai.google.dev/static/api/interactions.openapi.json
|
|||
|
||||
### How to Regenerate
|
||||
|
||||
When the API spec changes, regenerate the types with:
|
||||
The compliance tests in `tests/unit/interactions/test_openapi_compliance.py` read the snapshot at
|
||||
`tests/unit/interactions/interactions.openapi.json`, so a PR never depends on Google's live spec. The scheduled
|
||||
`interactions-spec-canary` workflow runs those tests against the live spec once a day (`--interactions-spec-url`),
|
||||
files the issue "Interactions spec canary is red" on a red run and closes it once a run is green again. A red run
|
||||
means the spec moved in a way these types no longer cover
|
||||
|
||||
When the red run names one new status, the small fix is a hand edit adding that member to the enum the failing
|
||||
schema's model carries, the way `queued` was added in #34135. When the spec moved further than that, refresh the
|
||||
snapshot and regenerate the types from it:
|
||||
|
||||
```bash
|
||||
uv tool run --from datamodel-code-generator datamodel-codegen \
|
||||
--url "https://ai.google.dev/static/api/interactions.openapi.json" \
|
||||
curl -fsSL https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
-o tests/unit/interactions/interactions.openapi.json
|
||||
uv tool run --from datamodel-code-generator==0.66.0 datamodel-codegen \
|
||||
--input tests/unit/interactions/interactions.openapi.json \
|
||||
--input-file-type openapi \
|
||||
--output litellm/types/interactions/generated.py \
|
||||
--output-model-type pydantic_v2.BaseModel \
|
||||
--target-python-version 3.9
|
||||
--target-python-version 3.10 \
|
||||
--base-class litellm.types.llms.base.LiteLLMBaseModel
|
||||
```
|
||||
|
||||
The pin is the same `datamodel-code-generator` version `scripts/generate_trace_types.py` uses. The generator names
|
||||
inline enums by order of appearance (`Status`, `Status1`, `Status2`, ...), so a regeneration can rename them. The
|
||||
compliance tests resolve each status enum through its model's `status` field, but
|
||||
`tests/unit/interactions/test_background_cost_polling.py` and
|
||||
`tests/unit/proxy/hooks/test_proxy_track_cost_callback.py` import `Status1` by name and need the new name, and
|
||||
`litellm/types/interactions/__init__.py` exports the interaction status enum as `InteractionStatus`, which has to
|
||||
point at the enum the regenerated `Interaction.status` carries (a compliance test checks their members match). The
|
||||
spec has grown a lot since the 2025-12-16 generation (agents, triggers, credentials, environments), so a
|
||||
regeneration today is a review-sized change rather than a refresh
|
||||
|
||||
Then add the LiteLLM-specific types at the bottom of the generated file:
|
||||
- `InteractionsAPIResponse`
|
||||
- `InteractionsAPIStreamingResponse`
|
||||
|
|
|
|||
|
|
@ -683,12 +683,12 @@ class ChatCompletionCachedContent(TypedDict):
|
|||
|
||||
class PromptCacheBreakpoint(TypedDict):
|
||||
mode: ReadOnly[Literal["explicit"]]
|
||||
ttl: NotRequired[ReadOnly[Literal["30m"]]]
|
||||
ttl: NotRequired[ReadOnly[Literal["30m", "12h"]]]
|
||||
|
||||
|
||||
class PromptCacheOptions(TypedDict, total=False):
|
||||
mode: ReadOnly[Literal["implicit", "explicit"]]
|
||||
ttl: ReadOnly[Literal["30m"]]
|
||||
ttl: ReadOnly[Literal["30m", "12h"]]
|
||||
|
||||
|
||||
class ChatCompletionThinkingBlock(TypedDict, total=False):
|
||||
|
|
@ -1382,10 +1382,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
|||
model_config = {"extra": "allow"}
|
||||
|
||||
|
||||
class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
||||
ttl_30m_tokens: int
|
||||
ttl_12h_tokens: int
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
def _is_none(value: object) -> bool:
|
||||
return value is None
|
||||
|
||||
|
||||
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
||||
audio_tokens: int | None = None
|
||||
cached_tokens: int = 0
|
||||
cached_tokens_details: CachedTokensDetails | None = None
|
||||
cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none)
|
||||
cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none)
|
||||
image_tokens: int | None = None
|
||||
text_tokens: int | None = None
|
||||
video_tokens: int | None = None
|
||||
|
|
|
|||
|
|
@ -322,11 +322,19 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_creation_input_token_cost_above_272k_tokens_flex: float | None
|
||||
cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_1hr: float | None
|
||||
cache_creation_input_token_cost_above_12hr: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_flex: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_priority: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
|
||||
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
|
||||
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
cache_read_input_token_cost: float | None
|
||||
cache_read_input_audio_token_cost: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_flex: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_priority: ReadOnly[float | None]
|
||||
cache_read_input_image_token_cost: ReadOnly[float | None]
|
||||
explicit_cache_read_input_token_cost: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
|
||||
|
|
@ -342,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_512k_tokens: float | None
|
||||
cache_read_input_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
|
|
@ -355,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates
|
||||
input_cost_per_character: float | None # only for vertex ai models
|
||||
input_cost_per_audio_token: float | None
|
||||
input_cost_per_audio_token_flex: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_priority: ReadOnly[float | None]
|
||||
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
|
||||
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
|
||||
input_cost_per_token_above_100k_tokens: ReadOnly[float | None]
|
||||
|
|
@ -1804,7 +1815,9 @@ class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for ol
|
|||
|
||||
class CacheCreationTokenDetails(LiteLLMBaseModel):
|
||||
ephemeral_5m_input_tokens: int | None = None
|
||||
ephemeral_30m_input_tokens: int | None = None
|
||||
ephemeral_1h_input_tokens: int | None = None
|
||||
ephemeral_12h_input_tokens: int | None = None
|
||||
|
||||
|
||||
class PromptTokensDetailsWrapper(
|
||||
|
|
@ -3954,6 +3967,12 @@ class MirroredPricingParams(LiteLLMBaseModel):
|
|||
explicit_cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_priority: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
|
||||
tiered_pricing: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
|
|
@ -4006,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
cache_read_input_token_cost_above_100k_tokens_batches: float | None = None
|
||||
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
|
||||
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
|
||||
cache_read_input_audio_token_cost_batches: float | None = None
|
||||
cache_creation_input_token_cost_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None
|
||||
cache_read_input_audio_token_cost: float | None = None
|
||||
cache_read_input_audio_token_cost_flex: float | None = None
|
||||
cache_read_input_audio_token_cost_priority: float | None = None
|
||||
cache_read_input_image_token_cost: float | None = None
|
||||
input_cost_per_character_above_128k_tokens: float | None = None
|
||||
input_cost_per_audio_token: float | None = None
|
||||
input_cost_per_audio_token_flex: float | None = None
|
||||
input_cost_per_audio_token_priority: float | None = None
|
||||
input_cost_per_token_cache_hit: float | None = None
|
||||
input_cost_per_token_above_128k_tokens: float | None = None
|
||||
input_cost_per_token_above_100k_tokens: float | None = None
|
||||
|
|
|
|||
130
litellm/utils.py
130
litellm/utils.py
|
|
@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
|
||||
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
|
||||
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
|
||||
|
|
@ -218,6 +219,7 @@ def _get_cached_audio_utils():
|
|||
return _audio_utils_module
|
||||
|
||||
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
AllPromptValues,
|
||||
|
|
@ -262,13 +264,17 @@ from litellm.types.utils import (
|
|||
SearchProviders,
|
||||
SelectTokenizerResponse,
|
||||
StreamingChoices,
|
||||
TextChoices,
|
||||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
credential_api_key_secret_reference,
|
||||
is_litellm_owned_kwarg,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
TextChoices as _TextChoices,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
TextCompletionResponse as _TextCompletionResponse,
|
||||
)
|
||||
|
||||
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
|
||||
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
|
||||
|
|
@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError):
|
|||
claude_json_str = json.dumps(json_data)
|
||||
import importlib.metadata
|
||||
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
TextChoices: TypeAlias = _TextChoices
|
||||
TextCompletionResponse: TypeAlias = _TextCompletionResponse
|
||||
TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper
|
||||
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
# These are lazy loaded via __getattr__
|
||||
|
|
@ -340,7 +350,8 @@ if TYPE_CHECKING:
|
|||
|
||||
# Type stubs for lazy-loaded functions to help mypy understand their types
|
||||
# These imports allow mypy to understand the types when these are accessed via __getattr__
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
exception_type: Callable[..., Exception]
|
||||
|
||||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
get_base_model_from_litellm_call_metadata,
|
||||
get_litellm_params,
|
||||
|
|
@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None:
|
|||
|
||||
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None:
|
||||
kind: Final = huggingface_tokenizer_kind(model)
|
||||
if kind is None:
|
||||
if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()):
|
||||
return None
|
||||
backend: Final = _huggingface_tokenizer_backend()
|
||||
key: Final = _huggingface_tokenizer_failure_key(kind, backend)
|
||||
|
|
@ -6762,6 +6773,13 @@ def get_model_info_helper(
|
|||
),
|
||||
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
|
||||
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
|
||||
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
|
||||
cache_read_input_audio_token_cost_priority=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_priority", None
|
||||
),
|
||||
cache_read_input_audio_token_cost_batches=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_batches", None
|
||||
),
|
||||
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
|
|
@ -6806,6 +6824,24 @@ def get_model_info_helper(
|
|||
cache_creation_input_token_cost_above_1hr=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_1hr", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_priority", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None
|
||||
),
|
||||
off_peak_pricing=_model_info.get("off_peak_pricing", None),
|
||||
input_cost_per_character=_model_info.get("input_cost_per_character", None),
|
||||
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
|
||||
|
|
@ -6829,6 +6865,8 @@ def get_model_info_helper(
|
|||
input_cost_per_second=_model_info.get("input_cost_per_second", None),
|
||||
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
|
||||
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
|
||||
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
|
||||
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
|
||||
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
|
||||
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
|
||||
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
|
||||
|
|
@ -7945,88 +7983,6 @@ def register_prompt_template(
|
|||
return litellm.custom_prompt_dict
|
||||
|
||||
|
||||
class TextCompletionStreamWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
completion_stream,
|
||||
model,
|
||||
stream_options: dict | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
):
|
||||
self.completion_stream = completion_stream
|
||||
self.model = model
|
||||
self.stream_options = stream_options
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
def convert_to_text_completion_object(self, chunk: ModelResponse):
|
||||
try:
|
||||
response: Final = TextCompletionResponse()
|
||||
response["id"] = chunk.get("id", None)
|
||||
response["object"] = "text_completion"
|
||||
response["created"] = chunk.get("created", None)
|
||||
response["model"] = chunk.get("model", None)
|
||||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
response["usage"] = chunk.get("usage", None)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
|
||||
|
||||
def __next__(self):
|
||||
# model_response = ModelResponse(stream=True, model=self.model)
|
||||
TextCompletionResponse()
|
||||
try:
|
||||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
except Exception as e:
|
||||
exception_type: Final = litellm_utils.exception_type
|
||||
raise exception_type(
|
||||
model=self.model,
|
||||
custom_llm_provider=self.custom_llm_provider or "",
|
||||
original_exception=e,
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
async def __anext__(self):
|
||||
try:
|
||||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration
|
||||
|
||||
|
||||
def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id=model_response.id,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
@ -61,7 +61,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -118,7 +118,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 nodejs libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
|
|||
|
|
@ -12827,6 +12827,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 16384,
|
||||
|
|
@ -12840,6 +12841,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"azure_ai/Phi-4-mini-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12851,6 +12853,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-multimodal-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_audio_token": 4e-06,
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -12865,6 +12868,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4-mini-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12876,6 +12880,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 32768,
|
||||
|
|
@ -24892,27 +24897,118 @@
|
|||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_multilingual_v2": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4": {
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4_turbo": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3_conversational": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
|
|
@ -31041,11 +31137,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost_batches": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31370,11 +31470,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 3e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 3e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 5.4e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
"input_cost_per_audio_token_flex": 1.5e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31664,12 +31768,16 @@
|
|||
},
|
||||
"gemini/gemini-3.1-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 9e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_audio_token_flex": 2.5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"input_cost_per_token_flex": 1.25e-07,
|
||||
|
|
@ -31790,10 +31898,14 @@
|
|||
},
|
||||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 5e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -36699,6 +36811,12 @@
|
|||
},
|
||||
"gpt-6-astra": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
|
||||
|
|
@ -36784,6 +36902,12 @@
|
|||
},
|
||||
"gpt-6-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -36861,17 +36985,23 @@
|
|||
},
|
||||
"gpt-6-luna": {
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_decisions": 1e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
|
||||
"cache_creation_input_token_cost_decisions": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
|
||||
"cache_creation_input_token_cost_flex": 6.25e-08,
|
||||
"cache_creation_input_token_cost_priority": 2.5e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_decisions": 1e-07,
|
||||
"cache_read_input_token_cost_decisions": 1e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-09,
|
||||
|
|
@ -36947,6 +37077,12 @@
|
|||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37021,6 +37157,12 @@
|
|||
},
|
||||
"gpt-5.6-sol": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37097,6 +37239,12 @@
|
|||
},
|
||||
"gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -37172,6 +37320,12 @@
|
|||
},
|
||||
"gpt-5.6-luna": {
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
|
|
@ -42337,6 +42491,134 @@
|
|||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3-0": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3-0": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3-0": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3-0": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-small-2507": {
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -45623,6 +45905,7 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -45648,6 +45931,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -48499,6 +48783,17 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
|
||||
"cache_read_input_token_cost": 1.5e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://www.together.ai/pricing",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
@ -49352,6 +49647,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"us-gov.openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -49371,6 +49667,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"us-gov.openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -52906,12 +53203,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
|
||||
"input_cost_per_token": 3.5e-07,
|
||||
"input_cost_per_token_batches": 1.75e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"max_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.15e-06,
|
||||
"output_cost_per_token_batches": 5.75e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -52926,12 +53225,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 10000000,
|
||||
"max_output_tokens": 10000000,
|
||||
"max_tokens": 10000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-07,
|
||||
"output_cost_per_token_batches": 3.5e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -59917,6 +60218,7 @@
|
|||
}
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7.5e-08,
|
||||
"input_cost_per_token_flex": 7.5e-08,
|
||||
|
|
@ -59942,6 +60244,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3.5e-08,
|
||||
"input_cost_per_token_flex": 3.5e-08,
|
||||
|
|
@ -59967,6 +60270,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7e-08,
|
||||
"input_cost_per_token_flex": 7e-08,
|
||||
|
|
@ -59990,6 +60294,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
|
|
@ -67841,6 +68146,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -67860,6 +68166,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68138,6 +68445,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68157,6 +68465,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68545,6 +68854,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68564,6 +68874,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68704,6 +69015,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68723,6 +69035,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -73330,6 +73643,7 @@
|
|||
]
|
||||
},
|
||||
"vertex_ai/veo-2.0-generate-001": {
|
||||
"deprecation_date": "2026-06-30",
|
||||
"litellm_provider": "vertex_ai-video-models",
|
||||
"max_input_tokens": 1024,
|
||||
"max_tokens": 1024,
|
||||
|
|
@ -74677,6 +74991,12 @@
|
|||
},
|
||||
"gpt-6.1-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v"
|
|||
ignoreUntil = 2026-11-01
|
||||
reason = "diskcache has no fixed release published; remove this entry once one exists"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-v4x8-3wr5-3wq8"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-vfj7-8cjw-p6xm"
|
||||
ignoreUntil = 2026-11-03
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ dependencies = [
|
|||
"pydantic>=2.12.0,<3.0.0; python_version >= '3.14'",
|
||||
"pydantic-settings>=2.14.1,<3.0",
|
||||
"jsonschema>=4.0.0,<5.0",
|
||||
"certifi>=2026.10.19",
|
||||
"typing-extensions>=4.13.0,<5.0",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ dependencies = [
|
|||
"pydantic>=2.12.0,<3.0.0; python_version >= '3.14'",
|
||||
"pydantic-settings>=2.14.1,<3.0",
|
||||
"jsonschema>=4.0.0,<5.0",
|
||||
"boto3>=1.43.1,<2.0",
|
||||
"boto3>=1.43.117,<2.0",
|
||||
"certifi>=2026.10.19",
|
||||
"typing-extensions>=4.13.0,<5.0",
|
||||
]
|
||||
|
||||
|
|
@ -72,7 +73,7 @@ proxy = [
|
|||
"cryptography>=49.0.0,<51.0",
|
||||
"pynacl>=1.6.2,<2.0",
|
||||
"websockets>=15.0.1,<16.0",
|
||||
"boto3>=1.43.1,<2.0",
|
||||
"boto3>=1.43.117,<2.0",
|
||||
"azure-identity>=1.25.2,<2.0",
|
||||
"azure-storage-blob>=12.28.0,<13.0",
|
||||
"mcp>=2.3.0,<3",
|
||||
|
|
@ -136,10 +137,10 @@ semantic-router = [
|
|||
"semantic-router>=0.1.15,<1.0; python_version < '3.14'",
|
||||
"aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'",
|
||||
]
|
||||
mlflow = ["mlflow>=3.11.1,<4.0"]
|
||||
mlflow = ["mlflow>=3.17.2,<4.0"]
|
||||
grpc = [
|
||||
# Newest non-yanked release older than the 30-day cutoff.
|
||||
"grpcio==1.78.0",
|
||||
# First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff
|
||||
"grpcio==1.84.1",
|
||||
]
|
||||
stt-vertex-chirp = [
|
||||
# Google Cloud Speech-to-Text v2 streaming (gRPC) for Chirp models on
|
||||
|
|
@ -173,19 +174,19 @@ proxy-runtime = [
|
|||
"google-cloud-speech>=2.40.0,<3.0",
|
||||
"google-genai>=1.37.0,<2.0",
|
||||
"anthropic[vertex]>=0.84.0,<1.0",
|
||||
"grpcio==1.78.0",
|
||||
"grpcio==1.84.1",
|
||||
"prometheus-client>=0.20.0,<1.0",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.54b1",
|
||||
"opentelemetry-api==1.45.1",
|
||||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"ddtrace>=4.8.2,<5.0",
|
||||
"sentry-sdk>=2.21.0,<3.0",
|
||||
"mangum>=0.17.0,<1.0",
|
||||
"azure-ai-contentsafety>=1.0.0,<2.0",
|
||||
"azure-storage-file-datalake>=12.20.0,<13.0",
|
||||
"pypdf>=6.16.1,<7.0",
|
||||
"pypdf>=6.21.2,<7.0",
|
||||
"llm-sandbox>=0.3.39,<1.0",
|
||||
"detect-secrets>=1.5.0,<2.0",
|
||||
]
|
||||
|
|
@ -222,10 +223,10 @@ dev = [
|
|||
"types-PyYAML==6.0.12.20250915",
|
||||
"botocore-stubs==1.43.14",
|
||||
"types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.54b1",
|
||||
"opentelemetry-api==1.45.1",
|
||||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"fastapi-offline==1.7.6",
|
||||
"fakeredis==2.34.1",
|
||||
|
|
@ -254,10 +255,10 @@ proxy-dev = [
|
|||
"hypercorn==0.17.3",
|
||||
"prometheus-client==0.20.0",
|
||||
"sentry-sdk==2.21.0",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.54b1",
|
||||
"opentelemetry-api==1.45.1",
|
||||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"azure-identity==1.25.2",
|
||||
"a2a-sdk==1.1.0",
|
||||
"backports-zstd==1.8.0; python_version < '3.14'",
|
||||
|
|
@ -279,7 +280,7 @@ ci = [
|
|||
"langchain==1.3.9",
|
||||
"lunary==1.4.36; python_version == '3.10'",
|
||||
"lunary==1.4.37; python_version >= '3.11'",
|
||||
"logfire==4.6.0",
|
||||
"logfire==5.2.0",
|
||||
"traceloop-sdk==0.34.0",
|
||||
"detect-secrets==1.5.0",
|
||||
"PyGithub==2.8.1",
|
||||
|
|
@ -358,7 +359,7 @@ constraint-dependencies = [
|
|||
"packaging>=24.0",
|
||||
"soupsieve>=2.8.4",
|
||||
"httplib2>=0.32.0",
|
||||
"setuptools>=83.0.0",
|
||||
"setuptools>=84.1.0",
|
||||
]
|
||||
override-dependencies = [
|
||||
# a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0.
|
||||
|
|
@ -370,6 +371,23 @@ default-groups = ["dev"]
|
|||
required-version = ">=0.10.9"
|
||||
exclude-newer = "3 days"
|
||||
|
||||
[tool.uv.exclude-newer-package]
|
||||
# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window
|
||||
certifi = "2026-10-20T03:00:00Z"
|
||||
# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window
|
||||
grpcio = "2026-10-22T15:00:00Z"
|
||||
# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window
|
||||
boto3 = "2026-10-21T19:31:00Z"
|
||||
botocore = "2026-10-21T19:31:00Z"
|
||||
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window
|
||||
mlflow = "2026-10-21T03:00:00Z"
|
||||
mlflow-skinny = "2026-10-21T03:00:00Z"
|
||||
mlflow-tracing = "2026-10-21T03:00:00Z"
|
||||
# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window
|
||||
pypdf = "2026-10-21T17:00:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
litellm-enterprise = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ langgraph: >=1.0.10 # MIT License
|
|||
langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE
|
||||
hypothesis: >=6.165.10 # MPL 2.0 license
|
||||
pytest-rerunfailures: >=15.1 # MPL 2.0 license
|
||||
certifi: >=2026.10.19 # MPL 2.0 license - https://github.com/certifi/python-certifi/blob/master/LICENSE
|
||||
pytest-recording: >=0.13.4 # MIT license
|
||||
expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE
|
||||
typing-extensions: >=4.13.0 # PSF-2.0 license - https://github.com/python/typing_extensions/blob/main/LICENSE
|
||||
|
|
|
|||
|
|
@ -1,125 +0,0 @@
|
|||
"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check
|
||||
the daily activity routes answer 400 with, instead of a generic fetch error."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway, object_value
|
||||
from integration._support.upstream import delete_scenario, register_scenario
|
||||
from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway
|
||||
from pydantic import JsonValue
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(180)
|
||||
|
||||
_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UsageChat:
|
||||
gateway: Gateway
|
||||
identity: str
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]:
|
||||
"""A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream."""
|
||||
identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}"
|
||||
handle: Final = register_scenario(identity, json_response({}))
|
||||
try:
|
||||
with owned_gateway(
|
||||
tmp_path_factory.mktemp("audit-usage-chat"),
|
||||
{"model_list": [], "router_settings": {"num_retries": 0}},
|
||||
environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity},
|
||||
) as owned:
|
||||
yield _UsageChat(owned, identity)
|
||||
finally:
|
||||
delete_scenario(handle)
|
||||
|
||||
|
||||
def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]:
|
||||
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
|
||||
return {
|
||||
"id": "chatcmpl-usage",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}}
|
||||
for tool in _TOOLS
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]:
|
||||
"""Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up
|
||||
model call carried, keyed by tool, and the tool statuses the SSE stream reported."""
|
||||
register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date)))
|
||||
Upstream(usage_chat.gateway.upstream_url).reset()
|
||||
with usage_chat.gateway.client.stream(
|
||||
"POST",
|
||||
"/usage/ai/chat",
|
||||
json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]},
|
||||
headers={"Authorization": f"Bearer {usage_chat.gateway.key}"},
|
||||
timeout=60,
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
events: Final = tuple(
|
||||
object_value(JSON_OBJECT.validate_json(line.removeprefix("data: ")))
|
||||
for line in response.iter_lines()
|
||||
if line.startswith("data: ")
|
||||
)
|
||||
calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity)
|
||||
assert len(calls) == 2, calls
|
||||
messages: Final = object_value(calls[1]["body"]).get("messages")
|
||||
assert isinstance(messages, list), calls[1]
|
||||
tool_messages: Final = {
|
||||
str(message["tool_call_id"]).removeprefix("call-"): str(message["content"])
|
||||
for message in messages
|
||||
if isinstance(message, dict) and message.get("role") == "tool"
|
||||
}
|
||||
statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call")
|
||||
return tool_messages, statuses
|
||||
|
||||
|
||||
_REVERSED: Final = "Invalid date range: end_date must be on or after start_date"
|
||||
_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates"
|
||||
_NO_TEAM_DATA: Final = "No Team usage data found for the given date range."
|
||||
_NO_TAG_DATA: Final = "No Tag usage data found for the given date range."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("start_date", "end_date", "team_message", "tag_message"),
|
||||
(
|
||||
pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"),
|
||||
pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"),
|
||||
pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"),
|
||||
pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"),
|
||||
),
|
||||
)
|
||||
def test_usage_chat_tools_answer_with_the_query_layers_date_check(
|
||||
usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str
|
||||
) -> None:
|
||||
"""The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query
|
||||
takes these dates without that check, still answers with its usage summary."""
|
||||
tool_messages, statuses = _tool_results(usage_chat, start_date, end_date)
|
||||
assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages
|
||||
assert tool_messages["get_team_usage_data"] == team_message, tool_messages
|
||||
assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages
|
||||
assert statuses == ("running", "complete") * len(_TOOLS), statuses
|
||||
|
|
@ -6985,6 +6985,106 @@ GPT_5_6_RESPONSES_CACHE_READ: Final = CostTrackingTestCase(
|
|||
),
|
||||
)
|
||||
|
||||
GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase(
|
||||
name="gpt-5.6-responses_cache_write_12h",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gpt-5.6",
|
||||
endpoint="/v1/responses",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"input": "summarize this text",
|
||||
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 12928,
|
||||
"output_tokens": 380,
|
||||
"total_tokens": 13308,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 5000,
|
||||
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0344865,
|
||||
input_cost=0.0291865,
|
||||
output_cost=0.00532,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
cache_creation_cost=0.0153125,
|
||||
),
|
||||
)
|
||||
|
||||
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase(
|
||||
name="gpt-6-luna-responses_cache_write_12h_no_12h_rate",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gpt-6-luna",
|
||||
endpoint="/v1/responses",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"input": "summarize this text",
|
||||
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "gpt-6-luna",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 12928,
|
||||
"output_tokens": 380,
|
||||
"total_tokens": 13308,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 5000,
|
||||
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0016078,
|
||||
input_cost=0.0014178,
|
||||
output_cost=0.00019,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
cache_creation_cost=0.000625,
|
||||
),
|
||||
)
|
||||
|
||||
GPT_5_6_RESPONSES_REASONING: Final = CostTrackingTestCase(
|
||||
name="gpt-5.6-responses_reasoning",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
|
|
@ -11913,6 +12013,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
OMNI_MODERATIONS_NEXT_SINGLE,
|
||||
OMNI_MODERATIONS_NEXT_LIST,
|
||||
GPT_5_6_RESPONSES_CACHE_READ,
|
||||
GPT_5_6_RESPONSES_CACHE_WRITE_12H,
|
||||
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
|
||||
GPT_5_6_RESPONSES_REASONING,
|
||||
GPT_5_6_RESPONSES_STREAM,
|
||||
GPT_5_6_RESPONSES_STREAM_CACHE_READ,
|
||||
|
|
|
|||
|
|
@ -174,6 +174,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
},
|
||||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 2.1875e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 4.375e-06,
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
"input_cost_per_audio_token": 4e-05,
|
||||
"input_cost_per_token": 1.75e-06,
|
||||
|
|
|
|||
|
|
@ -85,6 +85,12 @@ class CostMapEntry(BaseModel):
|
|||
cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_priority: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None
|
||||
cache_read_input_token_cost_above_200k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens: float | None = None
|
||||
|
|
|
|||
94
tests/integration/management/test_budget_list_sorting.py
Normal file
94
tests/integration/management/test_budget_list_sorting.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
|
||||
|
||||
@pytest.mark.parametrize("descending", (False, True))
|
||||
def test_budget_reset_sort_orders_durations_before_pagination(
|
||||
descending: bool,
|
||||
gateway: Gateway,
|
||||
scenario: Scenario,
|
||||
) -> None:
|
||||
prefix: Final = f"sort-{uuid4().hex}"
|
||||
durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "5d", "𝟝d", "5d", None)
|
||||
ids: Final = tuple(
|
||||
scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10)
|
||||
for index, duration in enumerate(durations)
|
||||
)
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
expected: Final = tuple(
|
||||
budget_id
|
||||
for _, budget_id in sorted(
|
||||
zip(durations, ids, strict=True),
|
||||
key=lambda item: (
|
||||
item[0] is None,
|
||||
duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0,
|
||||
item[1],
|
||||
),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/management/v1/budgets",
|
||||
params={
|
||||
"q": prefix,
|
||||
"sort": "-budget_duration" if descending else "budget_duration",
|
||||
"page_size": "3",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in range(1, -(-len(ids) // 3) + 1)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
|
||||
filtered: Final = gateway.request(
|
||||
"GET",
|
||||
"/management/v1/budgets",
|
||||
params={
|
||||
"q": prefix,
|
||||
"sort": "budget_duration",
|
||||
"filter[budget_duration][in]": "7d,30d",
|
||||
},
|
||||
)
|
||||
assert filtered.status_code == 200, filtered.text
|
||||
assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit"))
|
||||
@pytest.mark.parametrize("descending", (False, True))
|
||||
def test_budget_limits_sort_unlimited_as_infinity(
|
||||
field: str, descending: bool, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-unlimited-{uuid4().hex}"
|
||||
values: Final = (20, None, 0, 3)
|
||||
ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values))
|
||||
expected: Final = tuple(
|
||||
identity
|
||||
for value, identity in sorted(
|
||||
zip(values, ids, strict=True),
|
||||
key=lambda item: float("inf") if item[0] is None else item[0],
|
||||
reverse=descending,
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/management/v1/budgets",
|
||||
params={
|
||||
"q": prefix,
|
||||
"sort": ("-" if descending else "") + field,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
|
||||
|
|
@ -1,11 +1,16 @@
|
|||
from typing import FrozenSet
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
from typing import Final, FrozenSet
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
from tests.integration._support.actors import TEAM_ALPHA, Actor
|
||||
from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources
|
||||
from tests.integration._support.actors import create_scratch_key
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
|
|
@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor
|
|||
},
|
||||
)
|
||||
assert resp.status_code == 403, resp.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit")
|
||||
)
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted"))
|
||||
async def test_key_list_sorts_across_pages_with_nulls_last(
|
||||
field: str,
|
||||
direction: str,
|
||||
deleted: bool,
|
||||
gateway: Gateway,
|
||||
scenario: Scenario,
|
||||
db: ManagementDatabase,
|
||||
resources: ScratchResources,
|
||||
) -> None:
|
||||
team_admin: Final = scenario.user(user_role="internal_user")
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
team_id: Final = scenario.team(
|
||||
members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}]
|
||||
)
|
||||
team_admin_key: Final = scenario.key(user_id=team_admin)
|
||||
table: Final = (
|
||||
DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table
|
||||
)
|
||||
values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None)
|
||||
tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values)))
|
||||
rows: Final = [
|
||||
{
|
||||
"token": token,
|
||||
"key_alias": resources.prefix,
|
||||
"team_id": team_id,
|
||||
"user_id": owner,
|
||||
field: (
|
||||
datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value)
|
||||
if field in ("expires", "last_active", "budget_reset_at") and value is not None
|
||||
else value
|
||||
),
|
||||
}
|
||||
for token, value in zip(tokens, values, strict=True)
|
||||
]
|
||||
await table.create_many(rows)
|
||||
try:
|
||||
valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None)
|
||||
expected: Final = [
|
||||
token
|
||||
for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1]))
|
||||
] + [token for value, token in zip(values, tokens, strict=True) if value is None]
|
||||
for caller in (gateway.key, team_admin_key):
|
||||
pages: Final = [
|
||||
gateway.client.get(
|
||||
"/key/list",
|
||||
headers={"Authorization": f"Bearer {caller}"},
|
||||
params={
|
||||
"key_alias": resources.prefix,
|
||||
"team_id": team_id,
|
||||
"include_team_keys": "true",
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"size": "2",
|
||||
"page": str(page),
|
||||
**({"status": "deleted"} if deleted else {}),
|
||||
},
|
||||
)
|
||||
for page in (1, 2, 3)
|
||||
]
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total_count"] == len(tokens) for page in pages)
|
||||
assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected
|
||||
finally:
|
||||
await table.delete_many(where={"token": {"in": tokens}})
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import contextlib
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
|
@ -15,7 +17,7 @@ from pydantic import JsonValue
|
|||
|
||||
from tests.integration._support.client import Gateway, gateway_from_environment
|
||||
from tests.integration._support.database import read_rows
|
||||
from tests.integration._support.process import owned_proxy
|
||||
from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy
|
||||
from tests.integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
|
||||
|
|
@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role(
|
|||
"metadata": {"auth_provider": IDP_PROVIDER},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value)
|
||||
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
|
||||
with _sso_gateway(tmp_path, {}) as gateway:
|
||||
for index in range(4):
|
||||
signed_in: Final = _sign_in(gateway, f"free-sso-{index}")
|
||||
assert signed_in.status_code == 303, signed_in.text
|
||||
|
||||
contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8))
|
||||
with ThreadPoolExecutor(max_workers=len(contenders)) as pool:
|
||||
responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders))
|
||||
|
||||
assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403]
|
||||
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}]
|
||||
for subject, response in zip(contenders, responses):
|
||||
if response.status_code == 403:
|
||||
assert "requires a LiteLLM Enterprise license" in response.text
|
||||
assert not response.cookies
|
||||
assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == []
|
||||
|
||||
existing: Final = _sign_in(gateway, "free-sso-0")
|
||||
assert existing.status_code == 303, existing.text
|
||||
|
||||
admin_created: Final = gateway.request(
|
||||
"POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False}
|
||||
)
|
||||
assert admin_created.status_code == 200, admin_created.text
|
||||
above_capacity: Final = _sign_in(gateway, "free-sso-0")
|
||||
refused: Final = _sign_in(gateway, "free-sso-extra")
|
||||
assert (above_capacity.status_code, refused.status_code) == (303, 403)
|
||||
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}]
|
||||
|
|
|
|||
244
tests/integration/management/test_team_list_sorting.py
Normal file
244
tests/integration/management/test_team_list_sorting.py
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
import json
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
from tests.integration._support.database import write_rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count")
|
||||
)
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_team_list_sorts_across_pages_and_respects_membership(
|
||||
field: str,
|
||||
direction: str,
|
||||
gateway: Gateway,
|
||||
scenario: Scenario,
|
||||
) -> None:
|
||||
prefix: Final = f"sort-{uuid4().hex}"
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
caller: Final = scenario.key(user_id=owner)
|
||||
values: Final = (20, 3, 3, 100, 0)
|
||||
teams: Final = tuple(
|
||||
scenario.team(
|
||||
team_id=f"{prefix}-{index}",
|
||||
team_alias=f"{prefix}-{index}",
|
||||
members_with_roles=[{"user_id": owner, "role": "admin"}],
|
||||
)
|
||||
for index in range(len(values))
|
||||
)
|
||||
outsider: Final = scenario.team(team_alias=f"{prefix}-outside")
|
||||
for team_id, value in zip(teams, values, strict=True):
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, '
|
||||
"tpm_limit = %s::bigint, rpm_limit = %s::bigint, "
|
||||
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', "
|
||||
"members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) "
|
||||
"WHERE team_id = %s",
|
||||
(
|
||||
str(value),
|
||||
str(value),
|
||||
str(value),
|
||||
str(value),
|
||||
str(value),
|
||||
json.dumps(
|
||||
[{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)]
|
||||
),
|
||||
str(value),
|
||||
team_id,
|
||||
),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
|
||||
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
|
||||
(team_id, team_id, str(value)),
|
||||
)
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s',
|
||||
(teams[-1],),
|
||||
)
|
||||
try:
|
||||
expected: Final = tuple(
|
||||
team_id
|
||||
for value, team_id in sorted(
|
||||
zip(values, teams, strict=True),
|
||||
key=lambda item: (
|
||||
(item[0] != 0 if direction == "desc" else item[0] == 0)
|
||||
if field in ("max_budget", "tpm_limit", "rpm_limit")
|
||||
else False,
|
||||
item[0] * (1 if direction == "asc" else -1),
|
||||
item[1],
|
||||
),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
key=caller,
|
||||
params={
|
||||
"search": prefix,
|
||||
"user_id": owner,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2, 3)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages))
|
||||
assert returned == expected
|
||||
assert outsider not in returned
|
||||
assert all(page.json()["total"] == len(teams) for page in pages)
|
||||
finally:
|
||||
for team_id in teams:
|
||||
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_team_list_sorts_organization_names_and_missing_values(
|
||||
direction: str, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-org-{uuid4().hex}"
|
||||
budget: Final = scenario.budget(max_budget=10)
|
||||
organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", ""))
|
||||
teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4))
|
||||
try:
|
||||
for (organization_id, alias), team_id in zip(organizations, teams):
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationTable" '
|
||||
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
|
||||
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
|
||||
(organization_id, alias, budget),
|
||||
)
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s',
|
||||
(organization_id, team_id),
|
||||
)
|
||||
ascending: Final = (teams[0], teams[2], teams[1])
|
||||
ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending))
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
params={
|
||||
"search": prefix,
|
||||
"sort_by": "organization_alias",
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1])
|
||||
finally:
|
||||
for team_id in teams:
|
||||
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,))
|
||||
for organization_id, _ in organizations:
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count"))
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_computed_team_sorts_preserve_org_and_membership_union(
|
||||
field: str, direction: str, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-org-scope-{uuid4().hex}"
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
budget: Final = scenario.budget(max_budget=10)
|
||||
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
|
||||
teams: Final = tuple(
|
||||
scenario.team(
|
||||
team_id=f"{prefix}-{i}",
|
||||
team_alias=f"{prefix}-{i}",
|
||||
members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [],
|
||||
)
|
||||
for i in range(4)
|
||||
)
|
||||
try:
|
||||
for org, alias in zip(orgs, ("alpha", "zulu"), strict=True):
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationTable" '
|
||||
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
|
||||
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
|
||||
(org, alias, budget),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
|
||||
"VALUES (%s, %s, 'org_admin')",
|
||||
(owner, orgs[0]),
|
||||
)
|
||||
for index, team in enumerate(teams):
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, '
|
||||
"models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s",
|
||||
(
|
||||
orgs[0] if index < 2 else orgs[1],
|
||||
json.dumps(
|
||||
[
|
||||
{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"}
|
||||
for i in range(index + 1)
|
||||
]
|
||||
),
|
||||
str(index + 1),
|
||||
team,
|
||||
),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
|
||||
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
|
||||
(team, team, str(index + 1)),
|
||||
)
|
||||
caller: Final = scenario.key(user_id=owner)
|
||||
values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3)
|
||||
expected: Final = tuple(
|
||||
team
|
||||
for value, team in sorted(
|
||||
zip(values, teams[:3], strict=True),
|
||||
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
key=caller,
|
||||
params={
|
||||
"user_id": owner,
|
||||
"search": prefix,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total"] == 3 for page in pages)
|
||||
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected
|
||||
denied: Final = gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
key=caller,
|
||||
params={
|
||||
"organization_id": orgs[1],
|
||||
"sort_by": field,
|
||||
},
|
||||
)
|
||||
assert denied.status_code == 403, denied.text
|
||||
finally:
|
||||
for team in teams:
|
||||
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
|
||||
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,))
|
||||
for org in orgs:
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))
|
||||
179
tests/integration/management/test_user_list_sorting.py
Normal file
179
tests/integration/management/test_user_list_sorting.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
from tests.integration._support.database import write_rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count"))
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None:
|
||||
prefix: Final = f"sort-{uuid4().hex}"
|
||||
ids: Final = tuple(
|
||||
scenario.user(
|
||||
user_id=f"{prefix}-{index}",
|
||||
user_role="internal_user",
|
||||
user_alias=f"Alias {value if index != 2 else 4:03}",
|
||||
user_email=f"{prefix}-{index}@example.com",
|
||||
max_budget=value,
|
||||
metadata={"scim_active": value != 3},
|
||||
)
|
||||
for index, value in enumerate((20, 3, 3, 100, 0))
|
||||
)
|
||||
for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True):
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, '
|
||||
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s",
|
||||
(f"{prefix}-{value:03}", str(value), user_id),
|
||||
) if value != 3 or user_id == ids[1] else write_rows(
|
||||
"UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s",
|
||||
(f"{prefix}-004", user_id),
|
||||
)
|
||||
for index in range(value if value < 4 else 1):
|
||||
scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}")
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s',
|
||||
(ids[-1],),
|
||||
)
|
||||
scenario.key(user_id=ids[-1])
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1])
|
||||
)
|
||||
values: Final = (
|
||||
(0, 1, 1, 0, 0)
|
||||
if field == "status"
|
||||
else (1, 3, 3, 1, 0)
|
||||
if field == "key_count"
|
||||
else (20, 3, 4, 100, None)
|
||||
if field == "sso_user_id"
|
||||
else (20, 3, 4, 100, 0)
|
||||
if field == "user_alias"
|
||||
else (20, 3, 3, 100, None)
|
||||
)
|
||||
expected: Final = tuple(
|
||||
user_id
|
||||
for _, user_id in sorted(
|
||||
((value, user_id) for value, user_id in zip(values, ids, strict=True)),
|
||||
key=lambda item: (
|
||||
(item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None),
|
||||
(item[0] or 0) * (1 if direction == "asc" else -1),
|
||||
item[1],
|
||||
),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/user/list",
|
||||
params={
|
||||
"search": prefix,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2, 3)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total"] == len(ids) for page in pages)
|
||||
assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("status", "key_count"))
|
||||
def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None:
|
||||
team: Final = scenario.team()
|
||||
member: Final = scenario.member(team)
|
||||
other: Final = scenario.user(user_role="internal_user")
|
||||
for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}):
|
||||
baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"})
|
||||
response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field})
|
||||
assert response["total"] == baseline["total"]
|
||||
assert sorted(user["user_id"] for user in response["users"]) == sorted(
|
||||
user["user_id"] for user in baseline["users"]
|
||||
)
|
||||
ordinary_key: Final = scenario.key(user_id=other)
|
||||
denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field})
|
||||
assert denied.status_code == 403, denied.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("status", "key_count"))
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_computed_user_sorts_preserve_org_admin_scope(
|
||||
field: str, direction: str, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-org-users-{uuid4().hex}"
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
members: Final = tuple(
|
||||
scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1})
|
||||
for index in range(4)
|
||||
)
|
||||
budget: Final = scenario.budget(max_budget=10)
|
||||
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
|
||||
try:
|
||||
for org in orgs:
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationTable" '
|
||||
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
|
||||
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
|
||||
(org, org, budget),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
|
||||
"VALUES (%s, %s, 'org_admin')",
|
||||
(owner, orgs[0]),
|
||||
)
|
||||
for index, member in enumerate(members):
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
|
||||
"VALUES (%s, %s, 'internal_user')",
|
||||
(member, orgs[0] if index < 3 else orgs[1]),
|
||||
)
|
||||
for _ in range(index):
|
||||
scenario.key(user_id=member)
|
||||
caller: Final = scenario.key(user_id=owner)
|
||||
values: Final = (0, 1, 0) if field == "status" else (0, 1, 2)
|
||||
expected: Final = tuple(
|
||||
member
|
||||
for value, member in sorted(
|
||||
zip(values, members[:3], strict=True),
|
||||
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/user/list",
|
||||
key=caller,
|
||||
params={
|
||||
"search": prefix,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total"] == 3 for page in pages)
|
||||
returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages))
|
||||
assert returned == expected
|
||||
denied: Final = gateway.request(
|
||||
"GET",
|
||||
"/user/list",
|
||||
key=caller,
|
||||
params={
|
||||
"organization_ids": orgs[1],
|
||||
"sort_by": field,
|
||||
},
|
||||
)
|
||||
assert denied.status_code == 403, denied.text
|
||||
finally:
|
||||
for org in orgs:
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))
|
||||
180
tests/integration/security/test_offline_mode.py
Normal file
180
tests/integration/security/test_offline_mode.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, object_value
|
||||
from integration._support.forward_proxy import refusing_forward_proxy
|
||||
from integration._support.process import owned_proxy
|
||||
from pydantic import JsonValue
|
||||
|
||||
_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm"
|
||||
_CONTROL_HOST: Final = "provider.offline-mode.example"
|
||||
_REMOTE_URLS: Final = {
|
||||
"LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json",
|
||||
"LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json",
|
||||
"LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json",
|
||||
"LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml",
|
||||
}
|
||||
_SELF_INITIATED_HOSTS: Final = frozenset(
|
||||
{
|
||||
"cost-map.offline-mode.example:443",
|
||||
"beta-headers.offline-mode.example:443",
|
||||
"autorouter.offline-mode.example:443",
|
||||
"blog.offline-mode.example:443",
|
||||
"raw.githubusercontent.com:443",
|
||||
"docs.litellm.ai:443",
|
||||
"api.pwnedpasswords.com:443",
|
||||
"huggingface.co:443",
|
||||
}
|
||||
)
|
||||
_PER_FEATURE_FLAGS: Final = (
|
||||
"LITELLM_OFFLINE_MODE",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP",
|
||||
"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS",
|
||||
"LITELLM_LOCAL_AUTOROUTER_PRESETS",
|
||||
"LITELLM_LOCAL_POLICY_TEMPLATES",
|
||||
"LITELLM_LOCAL_BLOG_POSTS",
|
||||
"LITELLM_DISABLE_LATEST_RELEASE_CHECK",
|
||||
"LITELLM_LICENSE",
|
||||
)
|
||||
|
||||
|
||||
def _bundled(name: str) -> JsonValue:
|
||||
return json.loads((_PACKAGE / name).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _write_config(directory: Path) -> Path:
|
||||
config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "offline-mode-control",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": f"https://{_CONTROL_HOST}/v1",
|
||||
"api_key": "synthetic-provider-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "offline-mode-kimi",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/us.moonshotai.kimi-k3",
|
||||
"aws_region_name": "us-east-1",
|
||||
"aws_access_key_id": "synthetic-access-key",
|
||||
"aws_secret_access_key": "synthetic-secret-key",
|
||||
},
|
||||
},
|
||||
],
|
||||
"router_settings": {"num_retries": 0},
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue:
|
||||
response: Final = proxy.request(method, path, body)
|
||||
assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}"
|
||||
return json.loads(response.content)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
@pytest.mark.parametrize(
|
||||
("offline_mode", "expected_hosts", "reload_status"),
|
||||
[
|
||||
pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"),
|
||||
pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"),
|
||||
],
|
||||
)
|
||||
def test_offline_mode_stops_every_self_initiated_request(
|
||||
gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int
|
||||
) -> None:
|
||||
with refusing_forward_proxy() as egress:
|
||||
environment: Final = {
|
||||
"HTTPS_PROXY": egress.url,
|
||||
"HTTP_PROXY": egress.url,
|
||||
"ALL_PROXY": egress.url,
|
||||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
"AIOHTTP_TRUST_ENV": "True",
|
||||
**_REMOTE_URLS,
|
||||
**({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}),
|
||||
}
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
environment,
|
||||
config=_write_config(tmp_path),
|
||||
remove_environment=_PER_FEATURE_FLAGS,
|
||||
workers=2,
|
||||
) as isolated:
|
||||
source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source"))
|
||||
reloaded: Final = isolated.request("POST", "/reload/model_cost_map")
|
||||
beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers"))
|
||||
release: Final = _json(isolated, "GET", "/get/latest_release_info")
|
||||
presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets"))
|
||||
templates: Final = _json(isolated, "GET", "/policy/templates")
|
||||
blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts"))
|
||||
user: Final = object_value(
|
||||
_json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"})
|
||||
)
|
||||
password_set: Final = object_value(
|
||||
_json(
|
||||
isolated,
|
||||
"POST",
|
||||
"/user/update",
|
||||
{"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"},
|
||||
)
|
||||
)
|
||||
tokens: Final = object_value(
|
||||
_json(
|
||||
isolated,
|
||||
"POST",
|
||||
"/utils/token_counter",
|
||||
{"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]},
|
||||
)
|
||||
)
|
||||
kimi_tokens: Final = object_value(
|
||||
_json(
|
||||
isolated,
|
||||
"POST",
|
||||
"/utils/token_counter",
|
||||
{"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]},
|
||||
)
|
||||
)
|
||||
control: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]},
|
||||
)
|
||||
reached: Final = frozenset(egress.targets())
|
||||
|
||||
assert reached == expected_hosts
|
||||
assert control.status_code != 200, control.text
|
||||
assert source["source"] == "local", source
|
||||
assert source["is_env_forced"] is (offline_mode is not None), source
|
||||
assert reloaded.status_code == reload_status, reloaded.text
|
||||
assert beta_headers["status"] == "success", beta_headers
|
||||
assert release is None
|
||||
assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json")))
|
||||
assert templates == _bundled("policy_templates_backup.json")
|
||||
assert [object_value(post)["url"] for post in _list(blog["posts"])] == [
|
||||
object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"])
|
||||
]
|
||||
assert password_set["user_id"] == user["user_id"], password_set
|
||||
assert tokens["tokenizer_type"] == "openai_tokenizer", tokens
|
||||
assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens
|
||||
|
||||
|
||||
def _list(value: JsonValue) -> list[JsonValue]:
|
||||
assert isinstance(value, list), value
|
||||
return value
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue