merge: sync main into bedrock s3 media branch

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-11-14 00:51:58 +00:00
commit cbf9f49504
899 changed files with 81905 additions and 14100 deletions

View file

@ -0,0 +1,39 @@
name: "Cache the Go build"
description: >-
Restore the Go module and build caches on every run and save them only from
main, so pull requests reuse main's cache instead of evicting it with their
own copies.
actions/setup-go's built-in cache saves from every ref, and a cache saved by a
pull request is readable only by that pull request. Callers must pass
`cache: false` to setup-go and run this action after it.
inputs:
dependency-path:
description: "Path to the go.sum that keys the cache"
required: true
runs:
using: composite
steps:
- name: Restore and save the Go module and build caches
if: github.ref == 'refs/heads/main'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }}
restore-keys: |
${{ runner.os }}-go-
- name: Restore the Go module and build caches
if: github.ref != 'refs/heads/main'
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }}
restore-keys: |
${{ runner.os }}-go-

View file

@ -0,0 +1,37 @@
name: "Restore the Rust bridge"
description: >-
Restore the prebuilt Rust bridge that the unit workflow's rust-bridge job saves
on main, keyed on every input of the Rust build.
The bridge is abi3, so one build serves every supported Python version. A job
whose Rust inputs match main installs the restored wheel and native library
instead of compiling them inside `uv sync`. On a miss the caller builds from
source as before.
The restored `rust-bridge-dist` directory holds the editable litellm wheel, the
litellm-proxy gateway wheel, and `litellm/rust_bridge/_native.abi3.so`.
outputs:
key:
description: "Cache key for the bridge built from this checkout"
value: ${{ steps.key.outputs.key }}
cache-hit:
description: "'true' when the bridge for this exact key was restored"
value: ${{ steps.restore.outputs.cache-hit }}
runs:
using: composite
steps:
- name: Define the Rust bridge cache key
id: key
shell: bash
env:
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }}
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
- name: Restore the Rust bridge
id: restore
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: rust-bridge-dist
key: ${{ steps.key.outputs.key }}

View file

@ -11,22 +11,27 @@ reason = "filelock WindowsFileLock ignores mode in every release from 3.10.0 bef
[[IgnoredVulns]]
id = "GHSA-pcg2-g6cj-q92c"
ignoreUntil = 2027-02-01
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped"
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped"
[[IgnoredVulns]]
id = "GHSA-pw9p-v7r6-gx4p"
ignoreUntil = 2027-02-01
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
[[IgnoredVulns]]
id = "GHSA-r4j5-9xpp-3985"
ignoreUntil = 2027-02-01
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
[[IgnoredVulns]]
id = "GHSA-45c3-73f7-pv4m"
ignoreUntil = 2027-02-01
reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.2; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
[[IgnoredVulns]]
id = "GHSA-mmhr-3c5m-f64f"
ignoreUntil = 2027-02-01
reason = "soupsieve 4.0 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup in tests/code_coverage_tests/bedrock_pricing.py to parse AWS pricing HTML with html.parser and find table rows and cells, not CSS selectors; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5"
[[PackageOverrides]]
name = "pillow"
@ -35,3 +40,23 @@ ecosystem = "PyPI"
vulnerability.ignore = true
effectiveUntil = 2027-02-01
reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups and of the mlflow extra stays on 12.3.0; 3.11+ is on 13.0.0; pillow has never shipped an older-major release after the next major (no 10.x after 11.0.0, no 11.x after 12.0.0), so 13.x is the only fix line; extended to 2027-02-01 with maintainer approval, same date as every other Python 3.10 entry; remove once 3.10 is dropped"
[[IgnoredVulns]]
id = "GHSA-vvf4-9x63-8xvv"
ignoreUntil = 2027-02-01
reason = "dnspython 2.9.2 is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.2; the advisory is memory and CPU use when dns.rdata.from_text parses a WKS record from attacker-controlled zone file text, and litellm only reaches dnspython through pydantic EmailStr, which never parses zone files or does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
[[IgnoredVulns]]
id = "GHSA-5cjw-f85j-cv6r"
ignoreUntil = 2027-02-01
reason = "tornado 6.6.1 is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
[[IgnoredVulns]]
id = "GHSA-6655-6qjp-jx7x"
ignoreUntil = 2027-02-01
reason = "soupsieve 3.0.2 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. The advisory is exponential regex backtracking when compiling an untrusted CSS selector. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup only in tests/code_coverage_tests/bedrock_pricing.py, which calls find and find_all and never compiles a CSS selector; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5"
[[IgnoredVulns]]
id = "GHSA-7g44-g9rc-x53w"
ignoreUntil = 2027-02-01
reason = "soupsieve 3.0.2 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. The advisory is unbounded recursion when parsing an untrusted CSS selector with deeply nested pseudo-classes. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup only in tests/code_coverage_tests/bedrock_pricing.py, which calls find and find_all and never parses a CSS selector; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5"

View file

@ -0,0 +1,210 @@
## Captured API proof
These are actual curl requests to a local proxy and a controlled HTTP provider, using synthetic records and keys. Addresses, header files and IDs below are placeholders. Response excerpts omit unrelated fields and credentials.
### Permission change: verified before and after
The baseline process ran clean relevant source at `6685525fa2a809e0d4bc2c13cb07aef8eec242c3`. The requests used the same bound key and org policy; the after request was rerun after the schema update: the org allowed only `gpt-4.1-mini`; the key allowed `usage-proof-model` and was bound with `metadata.customer_org_id`. No team/member was needed for this org-level comparison.
```sh
curl --silent --show-error --request POST "$PROXY_BASE/v1/chat/completions" \
--header "@$SCOPED_KEY_HEADERS" --header 'Content-Type: application/json' \
--data-binary @request.json
```
`request.json`:
```json
{
"model": "usage-proof-model",
"messages": [
{
"role": "user",
"content": "Synthetic before-after permission check"
}
]
}
```
Baseline: **HTTP 200**.
```json
{
"object": "chat.completion",
"model": "usage-proof-model",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Controlled usage proof response.",
"role": "assistant"
},
"provider_specific_fields": {}
}
]
}
```
Current implementation: **HTTP 403**, with **zero controlled-provider calls**.
```json
{
"error": {
"message": "The requested model 'usage-proof-model' is not in the allowed models for this customer. Check the models this customer can use and try again.",
"type": "customer_model_access_denied",
"param": "model",
"code": "403"
}
}
```
### Current hierarchy and key intersection
Admin requests created the records below; each returned **201**. Member `budget_id` selects its template and `budget_override_id` identifies its optional override:
`POST /management/v1/customer_organizations`
```json
{
"customer_org_id": "<org-id>",
"alias": "Permissions and deletion proof",
"budget": {
"max_budget": 500,
"allowed_models": [
"gpt-4.1-mini",
"gpt-5.5"
]
},
"default_budgets": [
{
"name": "Standard",
"is_default": true,
"max_budget": 100,
"allowed_models": [
"gpt-4.1-mini"
]
}
]
}
```
`POST /management/v1/customer_teams`
```json
{
"customer_team_id": "<team-id>",
"parent_org_id": "<org-id>",
"alias": "Proof customer team",
"budget": {
"max_budget": 250,
"allowed_models": [
"gpt-4.1-mini",
"gpt-5.5"
]
}
}
```
`POST /management/v1/customer_members`
```json
{
"customer_member_id": "<member-id>",
"parent_team_id": "<team-id>",
"alias": "Proof customer member",
"budget_override": {
"max_budget": 100,
"allowed_models": [
"gpt-4.1-mini"
]
}
}
```
A synthetic `internal_user` was created with `auto_create_key: false`. The admin then created these keys through `POST /key/generate` (each **200**; returned credentials omitted):
| Key | models | metadata / budget binding |
| --- | --- | --- |
| A | `["gpt-4.1-mini", "gpt-5.5"]` | `{"customer_org_id":"<org-id>","customer_member_id":"<member-id>"}` |
| B | `["gpt-4.1-mini"]` | Same org/member binding |
| C | `["gpt-4.1-mini"]` | `budget_id: "<shared-budget-id>"`; no customer binding |
Inference curl uses the appropriate key header file. Captured request bodies:
```json
{
"model": "gpt-4.1-mini",
"messages": [
{
"role": "user",
"content": "Synthetic permission check"
}
]
}
```
```json
{
"model": "gpt-4.1-mini",
"input": "Synthetic permission check"
}
```
| State / key | Requested model | Chat HTTP | Responses HTTP | Provider calls per request |
| --- | --- | --- | --- | --- |
| Member restriction / A | gpt-4.1-mini | 200 | 200 | 1 |
| Member restriction / A | gpt-5.5 | 403 | 403 | 0 |
| Member widened; key B remains narrow | gpt-5.5 | 403 | 403 | 0 |
| Member widened; org narrowed / A | gpt-5.5 | 403 | 403 | 0 |
The widening PATCH was `{"budget_override":{"allowed_models":["gpt-4.1-mini","gpt-5.5"]}}` on the member. The org narrowing PATCH was `{"budget":{"allowed_models":["gpt-4.1-mini"]}}`. Key B still denied the model after the member widened.
### Cascade, retained history and shared key budget
**Same-revision comparison:** deleting the populated org without `cascade=true` returned **409**:
```json
{
"type": "urn:litellm:error:customer-hierarchy-has-children",
"title": "Has children",
"status": 409,
"detail": "Delete the customer's children before deleting the parent."
}
```
The confirmed cascade request returned **204**, with an empty body:
```sh
curl --silent --show-error --request DELETE \
"$PROXY_BASE/management/v1/customer_organizations/$ORG_ID?cascade=true" \
--header "@$ADMIN_HEADERS"
```
Subsequent GETs for that org, its team and its member each returned **404**. The existing demo organization still returned **200**.
Key C referenced the org-owned budget. After PATCHing that budget to `max_budget: 0`, its inference returned **422 `budget_exceeded` before and after cascade**, with zero provider calls. Its authenticated `GET /key/info` retained the same relation and limit (selected fields):
```json
{
"info": {
"budget_id": "<shared-budget-id>",
"litellm_budget_table": {
"max_budget": 0.0
}
}
}
```
The same admin daily query returned **200** before and after cascade:
```sh
curl --silent --show-error \
"$PROXY_BASE/management/v1/customer_organizations/daily/activity?customer_org_ids=$ORG_ID&start_date=$UTC_DATE&end_date=$UTC_DATE" \
--header "@$ADMIN_HEADERS"
```
Selected captured metadata was identical before and after; API-key/entity breakdowns are omitted:
```json
{
"metadata": {
"total_spend": 0.28,
"total_prompt_tokens": 200,
"total_completion_tokens": 40,
"total_tokens": 240,
"total_api_requests": 4,
"total_successful_requests": 2,
"total_failed_requests": 2
}
}
```

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 93 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 90 KiB

View file

@ -0,0 +1,69 @@
The OpenAI passthrough route now resolves customer organization, team and effective member model restrictions before forwarding the request. It uses the existing customer resolver and model matcher, then checks the model again after hooks. Gateway key permissions still apply.
Actual curl proof used a controlled HTTP upstream. Before: PR head `43d55ab2b91e4cccccf014c5f813a2b70a4e89f9` merged with target `eb811913f1`, before the fix. After: the implementation linked from the current PR description. Counts below come from the upstream request log.
Setup requests, using synthetic identifiers:
```http
POST /management/v1/customer_organizations
{"customer_org_id":"proof-org","alias":"Synthetic passthrough permission proof","budget":{"max_budget":100,"allowed_models":["gpt-4.1-mini"]}}
POST /user/new
{"user_id":"proof-user","user_role":"internal_user","auto_create_key":false}
POST /key/generate
{"user_id":"proof-user","models":["gpt-4.1-mini","gpt-5.5"],"metadata":{"customer_org_id":"proof-org"}}
POST /key/generate
{"user_id":"proof-user","models":["gpt-4.1-mini"],"metadata":{"customer_org_id":"proof-org"}}
```
The first key permits both models; the second permits only mini. Credentials from both responses are omitted.
```bash
curl "$GATEWAY_BASE_URL/openai/v1/chat/completions" \
--header "@$CUSTOMER_KEY_AB_HEADERS" \
--header 'Content-Type: application/json' \
--data '{"model":"gpt-5.5","messages":[{"role":"user","content":"Synthetic passthrough policy check"}]}'
```
Before: HTTP 200, one upstream call. Captured response excerpt:
```json
{
"model": "gpt-5.5",
"usage": {
"prompt_tokens": 100,
"completion_tokens": 20,
"total_tokens": 120
}
}
```
After: HTTP 403, zero upstream calls. Captured response:
```json
{
"error": {
"message": "The requested model 'gpt-5.5' is not in the allowed models for this customer. Check the models this customer can use and try again.",
"type": "customer_model_access_denied",
"param": "model",
"code": "403"
}
}
```
The same key still receives HTTP 200 for `gpt-4.1-mini`, with one upstream call. After widening the organization to permit both models, the mini-only key still receives HTTP 403 for `gpt-5.5`, with zero upstream calls:
```json
{
"error": {
"message": "The requested model 'gpt-5.5' is not available for this API key, or the model name is invalid. Check the models available to you and try again.",
"type": "key_model_access_denied",
"param": "model",
"code": "403"
}
}
```
The proof deletes its synthetic organization afterward. Registered-route regressions also cover team restrictions, member templates and overrides, hook model rewrites, model-less file retrieval, and the existing configured passthrough model contract.

Binary file not shown.

After

Width:  |  Height:  |  Size: 73 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 61 KiB

View file

@ -73,14 +73,14 @@ jobs:
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
${{ runner.os }}-uv-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
@ -396,7 +396,7 @@ jobs:
- name: Cache uv dependencies
if: steps.changes.outputs.relevant == 'true'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv

View file

@ -38,6 +38,16 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# These jobs check packaging, installation and imports, so they build the Rust
# workspace unoptimized. The release profile's fat LTO and opt-level 3 pushed
# every job to its timeout. The overrides go through Cargo's environment
# because a maturin `--profile ci` argument also reaches dependencies that
# build from source, such as jiter on Python 3.15, which have no ci profile
env:
CARGO_PROFILE_RELEASE_OPT_LEVEL: "0"
CARGO_PROFILE_RELEASE_LTO: "off"
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "256"
jobs:
resolve:
runs-on: ubuntu-latest

View file

@ -3,6 +3,8 @@ name: smoke
on:
pull_request:
branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"]
push:
branches: [main]
workflow_dispatch:
permissions:
@ -52,8 +54,26 @@ jobs:
with:
version: "0.10.9"
- name: Restore the Rust bridge
id: rust-bridge
uses: ./.github/actions/restore-rust-bridge
- name: Restore the Cargo build cache
if: steps.rust-bridge.outputs.cache-hit != 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
env:
RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }}
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION"
exit 0
fi
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/litellm-*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
- name: Create the loopback-only network namespace
run: |
@ -87,6 +107,7 @@ jobs:
- name: Upload smoke diagnostics
if: always()
continue-on-error: true
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: merge-smoke-diagnostics-py${{ matrix.python-version }}
@ -97,6 +118,10 @@ jobs:
if: always()
run: sudo ip netns delete smoke
- name: Keep only source-built wheels in the uv cache
if: always() && github.ref == 'refs/heads/main'
run: uv cache prune --ci
all-extras:
name: all-extras-py${{ matrix.python-version }}
runs-on: ubuntu-24.04
@ -123,12 +148,34 @@ jobs:
with:
version: "0.10.9"
- name: Restore the Rust bridge
id: rust-bridge
uses: ./.github/actions/restore-rust-bridge
- name: Restore the Cargo build cache
if: steps.rust-bridge.outputs.cache-hit != 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install every extra and dependency group
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }}
env:
RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }}
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION"
exit 0
fi
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
- name: Import the SDK, the proxy server and the enterprise package
run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise"
- name: Keep only source-built wheels in the uv cache
if: always() && github.ref == 'refs/heads/main'
run: uv cache prune --ci
windows-sdk:
name: windows-sdk
runs-on: windows-2025
@ -154,15 +201,16 @@ jobs:
with:
version: "0.10.9"
- name: Set up the Rust bridge
uses: ./.github/actions/rust-bridge
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
- name: Run the Windows tests
run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests
- name: Keep only source-built wheels in the uv cache
if: always() && github.ref == 'refs/heads/main'
run: uv cache prune --ci
proxy-build-from-pip:
name: proxy-build-from-pip
runs-on: ubuntu-24.04
@ -298,6 +346,10 @@ jobs:
if: always()
run: cat /tmp/litellm-pip.log || true
- name: Keep only source-built wheels in the uv cache
if: always() && github.ref == 'refs/heads/main'
run: uv cache prune --ci
smoke-passed:
name: smoke passed
needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]

View file

@ -42,10 +42,6 @@ concurrency:
env:
CARGO_TERM_COLOR: always
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
SCCACHE_GHA_VERSION: litellm-ci-v1
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
CARGO_INCREMENTAL: "0"
jobs:
@ -91,11 +87,7 @@ jobs:
with:
workspaces: litellm/rust
cache-on-failure: true
- name: Set up sccache
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
with:
version: "v0.18.0"
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install Python dependencies
working-directory: .

View file

@ -78,10 +78,6 @@ concurrency:
env:
CARGO_TERM_COLOR: always
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
SCCACHE_GHA_VERSION: litellm-ci-v1
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
CARGO_INCREMENTAL: "0"
jobs:
@ -108,11 +104,6 @@ jobs:
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Set up sccache
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
with:
version: "v0.18.0"
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
rust-test:
@ -155,11 +146,6 @@ jobs:
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Set up sccache
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
with:
version: "v0.18.0"
- name: Check generated trace contracts
working-directory: .
run: uv run litellm/scripts/generate_trace_types.py --check
@ -206,11 +192,6 @@ jobs:
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Set up sccache
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
with:
version: "v0.18.0"
- run: uv build --wheel --out-dir dist
- run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl

View file

@ -4,6 +4,7 @@ on:
push:
paths:
- "litellm/deploy/terraform/provider/**"
- ".github/actions/cache-go-build/**"
- ".github/workflows/test-terraform-provider.yml"
pull_request:
branches:
@ -13,6 +14,7 @@ on:
- "litellm/deploy/terraform/provider/**"
- "litellm/proxy/**"
- "litellm/enterprise/**"
- ".github/actions/cache-go-build/**"
- ".github/workflows/test-terraform-provider.yml"
permissions:
@ -38,8 +40,12 @@ jobs:
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
with:
go-version-file: litellm/deploy/terraform/provider/go.mod
cache: true
cache-dependency-path: litellm/deploy/terraform/provider/go.sum
cache: false
- name: Cache the Go build
uses: ./.github/actions/cache-go-build
with:
dependency-path: litellm/deploy/terraform/provider/go.sum
- name: gofmt
run: |
@ -120,8 +126,12 @@ jobs:
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
with:
go-version-file: litellm/deploy/terraform/provider/go.mod
cache: true
cache-dependency-path: litellm/deploy/terraform/provider/go.sum
cache: false
- name: Cache the Go build
uses: ./.github/actions/cache-go-build
with:
dependency-path: litellm/deploy/terraform/provider/go.sum
- name: Audit provider endpoints against the schema
working-directory: litellm/deploy/terraform/provider

View file

@ -41,13 +41,6 @@ jobs:
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Define editable Rust bridge cache key
id: rust-bridge-key
if: steps.changes.outputs.decision != 'skip'
env:
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }}
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
@ -66,30 +59,17 @@ jobs:
id: rust-bridge-cache
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: rust-bridge-dist
key: ${{ steps.rust-bridge-key.outputs.key }}
uses: ./.github/actions/restore-rust-bridge
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Set up sccache
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
with:
version: "v0.18.0"
- name: Build editable Rust bridge
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
timeout-minutes: 15
env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
SCCACHE_GHA_VERSION: litellm-ci-v1
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
CARGO_INCREMENTAL: "0"
run: |
mkdir -p rust-bridge-dist
@ -103,10 +83,6 @@ jobs:
timeout-minutes: 15
working-directory: litellm/packaging/litellm-proxy
env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
SCCACHE_GHA_VERSION: litellm-ci-v1
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
CARGO_INCREMENTAL: "0"
run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../../rust-bridge-dist
@ -125,7 +101,7 @@ jobs:
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: rust-bridge-dist
key: ${{ steps.rust-bridge-key.outputs.key }}
key: ${{ steps.rust-bridge-cache.outputs.key }}
- name: Expose the Rust bridge artifact
id: rust-bridge-artifact

View file

@ -36,11 +36,6 @@ _billing_time: Final[ContextVar[datetime | None]] = ContextVar("billing_time", d
_post_response: Final[ContextVar[bool]] = ContextVar("post_response", default=False)
_service_target: Final[ContextVar[str | None]] = ContextVar("service_target", default=None)
# Event-metadata key under which a Redis pipeline reports the sorted, comma-joined families its ops were
# declared under when they span more than one.
REDIS_FAMILIES_METADATA_KEY: Final = "families"
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
@ -140,22 +135,6 @@ def with_service_target(
return decorate
@contextmanager
def service_caller(caller: str | None) -> Generator[None]:
"""Name the litellm code a datastore call was issued for when its own frames cannot: an operation
declared in one task and run in another (a batch op retried on the flush) carries the chain captured
where it was declared."""
token: Final = _service_caller.set(caller)
try:
yield
finally:
_restore(_service_caller, token)
def current_service_caller() -> str | None:
return _service_caller.get()
@contextmanager
def mirrored_call() -> Generator[None]:
"""A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included."""
@ -197,6 +176,20 @@ def current_request_parent_otel_span() -> Span | None:
return _request_parent_otel_span.get()
@contextmanager
def request_parent_otel_span(span: Span | None) -> Generator[None]:
"""Anchor the datastore service spans made inside this block, and by the tasks it starts, under the span an SDK
caller passed. A request that already anchored its own, as the proxy does at auth, keeps it."""
if span is None or _request_parent_otel_span.get() is not None:
yield
return
token: Final = _request_parent_otel_span.set(span)
try:
yield
finally:
_restore(_request_parent_otel_span, token)
@contextmanager
def outside_mirrored_call() -> Generator[None]:
"""Logging a mirrored call: the calls its callbacks start are not mirrors themselves."""

View file

@ -56,6 +56,9 @@ def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpda
"model_access_group_list_transactions": dict(
_entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable")
),
"customer_org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerOrganizationTable")),
"customer_team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerTeamTable")),
"customer_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerMemberTable")),
}

View file

@ -1,6 +1,6 @@
from collections.abc import Iterator, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import Final
@ -14,9 +14,10 @@ from litellm.constants import (
MODEL_INSIGHTS_TASK_TAG_PREFIX,
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry
from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution
from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids
from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks
from litellm.proxy.db.model_usage_rollup import date_from_start_time
from litellm.proxy.db.model_usage_rollup import date_from_start_time, utc_date_from_start_time
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens
@ -70,9 +71,32 @@ def _spend(table: str, key: tuple[tuple[str, str], ...], spend: float) -> Contri
return Contribution(table, key, (("spend", spend),))
_CUSTOMER_ID_KEYS: Final = ("customer_org_id", "customer_team_id", "customer_member_id")
def _customer_ids(metadata: Mapping[str, object]) -> Mapping[str, str | None]:
"""The customer hierarchy a request is charged to: the member the request named, else the key's attribution."""
entities: Final = _fields(metadata.get("accounting_entities"))
if isinstance(entities.get("customer_org_id"), str):
return {key: value if isinstance(value := entities.get(key), str) else None for key in _CUSTOMER_ID_KEYS}
serialized: Final = metadata.get(CUSTOMER_USAGE_METADATA_KEY)
if not isinstance(serialized, str):
return dict.fromkeys(_CUSTOMER_ID_KEYS)
try:
attribution: Final = CustomerUsageAttribution.model_validate_json(serialized)
except ValidationError:
return dict.fromkeys(_CUSTOMER_ID_KEYS)
return {
"customer_org_id": attribution.customer_org_id,
"customer_team_id": attribution.customer_team_id,
"customer_member_id": attribution.customer_member_id,
}
def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
spend: Final = _number(row.get("spend"))
entities: Final = _fields(metadata.get("accounting_entities"))
customer: Final = _customer_ids(metadata)
identities: Final = (
("LiteLLM_VerificationToken", "token", entities.get("api_key")),
("LiteLLM_UserTable", "user_id", entities.get("user")),
@ -80,6 +104,9 @@ def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, obje
("LiteLLM_TeamTable", "team_id", entities.get("team_id")),
("LiteLLM_OrganizationTable", "organization_id", entities.get("organization_id")),
("LiteLLM_EndUserTable", "user_id", entities.get("end_user")),
("LiteLLM_CustomerOrganizationTable", "customer_org_id", customer["customer_org_id"]),
("LiteLLM_CustomerTeamTable", "customer_team_id", customer["customer_team_id"]),
("LiteLLM_CustomerMemberTable", "customer_member_id", customer["customer_member_id"]),
("LiteLLM_AgentsTable", "agent_id", entities.get("agent_id")),
("LiteLLM_ProjectTable", "project_id", entities.get("project_id")),
)
@ -159,7 +186,14 @@ def _daily_contributions(row: Mapping[str, object], metadata: Mapping[str, objec
)
# The legacy daily organization writer uses the charged org argument, not
# SpendLogs' normalized empty-string identity for a request with no org.
daily_row: Final = {**row, "organization_id": _fields(metadata.get("accounting_entities")).get("organization_id")}
entities: Final = _fields(metadata.get("accounting_entities"))
customer: Final = _customer_ids(metadata)
daily_row: Final = {
**row,
"organization_id": entities.get("organization_id"),
"customer_org_id": customer["customer_org_id"],
**{key: customer[key] or "" for key in ("customer_team_id", "customer_member_id")},
}
return tuple(_daily_rows(daily_row, shared_key, counters))
@ -169,13 +203,26 @@ def _daily_rows(
counters: tuple[tuple[str, int | float], ...],
) -> Iterator[Contribution]:
for entity, table in DAILY_SPEND_TABLES.items():
customer_date = _utc_date(row) if entity == "customer_member" else None
if entity == "customer_member" and customer_date is None:
continue
daily_key = tuple(
(column, customer_date if column == "date" and customer_date is not None else value)
for column, value in shared_key
)
model_group = (("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),)
for identity in daily_spend_entity_ids(row, entity):
yield Contribution(
table.name,
((table.entity_id_column, _text(identity)), *shared_key),
(
(table.entity_id_column, _text(identity)),
*((column, _text(row.get(column))) for column in table.parent_id_columns),
*daily_key,
*((("model_group", _text(row.get("model_group"))),) if table.hashed_identity else ()),
),
counters,
(
("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),
*(() if table.hashed_identity else model_group),
*((("request_id", _text(row.get("request_id"))),) if table.carries_request_id else ()),
),
)
@ -259,18 +306,7 @@ def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, objec
def _utc_date(row: Mapping[str, object]) -> str | None:
start: Final = row.get("startTime")
try:
parsed: Final = datetime.fromisoformat(start.replace("Z", "+00:00")) if isinstance(start, str) else start
except ValueError:
return None
if not isinstance(parsed, datetime):
return None
return (
(parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed)
.astimezone(timezone.utc)
.date()
.isoformat()
)
return utc_date_from_start_time(start) if isinstance(start, (datetime, str)) else None
def _guardrail_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:

View file

@ -31,6 +31,7 @@ _METADATA_FIELDS: Final = frozenset(
"compression_savings",
"internal_call_origin",
"billed_retry_attempt",
"user_api_key_customer_usage",
}
)
@ -51,6 +52,9 @@ _ACCOUNTING_FIELDS: Final = frozenset(
"team_id",
"organization_id",
"end_user",
"customer_member_id",
"customer_team_id",
"customer_org_id",
"agent_id",
"request_tags",
"status",

View file

@ -10,8 +10,10 @@ from typing import (
from litellm._logging import verbose_proxy_logger
from litellm.accounting.application import apply_deliveries
from litellm.accounting.delivery_projection import entity_transactions
from litellm.accounting.metrics import record_backlog, record_storage
from litellm.accounting.projection import Contribution
from litellm.proxy.spend_tracking.customer_budget_alerts import CustomerBudgetAlerting
from litellm.repositories.accounting_delivery_repository import (
AccountingDeliveryRepository,
DeliveryBatchResult,
@ -65,6 +67,9 @@ async def flush_accounting_deliveries(
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
if result.applied:
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
await CustomerBudgetAlerting.detect_committed_batch(
entity_transactions(applied), prisma_client, proxy_logging_obj.internal_usage_cache.dual_cache
)
if result.parked:
verbose_proxy_logger.error(
"Accounting delivery parked %d requests after isolated failures", len(result.parked)

View file

@ -1,151 +1,42 @@
"""Atomic affinity claims shared by deployment and tier-model selection."""
import json
from collections.abc import Mapping
from typing import (
Final,
cast, # noqa: TID251 # Heterogeneous cache results are narrowed only to object, then validated
)
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing import Final
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_batch import active_request_redis_batch
from litellm.caching.redis_batch import Batch
from litellm.redis.stores.router_session_pins import ROUTER_SESSION_PINS_TARGET, PinKey, RouterSessionPinStore
ROUTER_SESSION_PINS_TARGET: Final = "router_session_pins"
_PIN_JSON_ADAPTER: Final = TypeAdapter[JsonValue](JsonValue)
_READ_PIN_SCRIPT: Final = """
local current = redis.call('GET', KEYS[1])
if current ~= false then
redis.call('EXPIRE', KEYS[1], ARGV[1])
end
return current
"""
_CLAIM_PIN_SCRIPT: Final = """
local current = redis.call('GET', KEYS[1])
if current == false then
redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
return ARGV[1]
end
if ARGV[3] then
local decoded, stored = pcall(cjson.decode, current)
if ARGV[4] then
local legacy = current
if decoded and type(stored) == 'string' then legacy = stored end
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
if eligible['model_id'] == legacy then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
if decoded and type(stored) == 'table' then
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
local matches = true
for key, value in pairs(eligible) do
if stored[key] ~= value then matches = false; break end
end
for key, _ in pairs(stored) do
if eligible[key] == nil then matches = false; break end
end
if matches then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
return ARGV[1]
end
if current == ARGV[1] then
redis.call('EXPIRE', KEYS[1], ARGV[2])
end
return current
"""
__all__ = [
"ROUTER_SESSION_PINS_TARGET",
"claim_affinity_pin",
"read_affinity_pin",
"set_local_affinity_pin",
]
def set_local_affinity_pin(cache: DualCache, cache_key: str, value: object, ttl_seconds: int) -> None:
"""Replace the entry because InMemoryCache.set_cache preserves a live key's expiry."""
cache.in_memory_cache.delete_cache(cache_key)
cache.in_memory_cache.set_cache(cache_key, value, ttl=ttl_seconds)
def set_local_affinity_pin(pins: RouterSessionPinStore, cache_key: PinKey, value: object, ttl_seconds: int) -> None:
pins.set_local(cache_key, value, ttl_seconds)
def _legacy_pin_matches(stored: object, pin_value: Mapping[str, str]) -> bool:
if isinstance(stored, dict):
return all(stored.get(key) is not None and str(stored[key]) == value for key, value in pin_value.items())
return isinstance(stored, str) and len(pin_value) == 1 and stored in pin_value.values()
def claim_affinity_pin_in_memory(
cache: DualCache,
cache_key: str,
pin_value: Mapping[str, str],
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""No await between read and write, so same-loop claims agree during a Redis outage."""
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
if existing is not None and eligible_values is None:
if _legacy_pin_matches(existing, pin_value):
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
return existing
eligible_existing: Final = existing in (eligible_values or ()) or (
preserve_legacy_model_ids
and isinstance(existing, str)
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
)
winner: Final = existing if existing is not None and eligible_existing else pin_value
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
def _decode_pin(value: str) -> object:
try:
return _PIN_JSON_ADAPTER.validate_json(value)
except ValidationError:
return value
async def read_affinity_pin(cache: DualCache, cache_key: str, ttl_seconds: int) -> object:
async def read_affinity_pin(pins: RouterSessionPinStore, cache_key: PinKey, ttl_seconds: int) -> object:
"""Refresh the authoritative pin without writing a previously observed value back."""
redis_cache: Final = cache.redis_cache
if redis_cache is not None:
if pins.has_redis:
try:
read_script: Final = redis_cache.async_register_script(_READ_PIN_SCRIPT)
batch: Final = active_request_redis_batch(redis_cache)
raw: Final = cast( # cast-ok: Redis script returns a serialized pin or a miss; validate before use
object,
await batch.script(_READ_PIN_SCRIPT, read_script, (cache_key,), (int(ttl_seconds),))
if batch is not None and batch.pending
else await read_script(keys=(cache_key,), args=(int(ttl_seconds),)),
)
decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
value: Final = _decode_pin(decoded) if isinstance(decoded, str) else None
if value is None:
cache.in_memory_cache.delete_cache(cache_key)
else:
set_local_affinity_pin(cache, cache_key, value, ttl_seconds)
value: Final = await pins.read(cache_key, ttl_seconds, batch=Batch.PRE_CALL)
except Exception as error: # noqa: BLE001 # Redis faults retain pod-local continuity, unlike a successful miss
verbose_router_logger.debug("Affinity cache: Redis read failed, using pod-local pin. error=%s", error)
else:
return value
local_value: Final = cast( # cast-ok: the heterogeneous local cache is parsed by the affinity caller
object, cache.in_memory_cache.get_cache(cache_key)
)
local_value: Final = pins.get_local(cache_key)
if local_value is not None:
set_local_affinity_pin(cache, cache_key, local_value, ttl_seconds)
set_local_affinity_pin(pins, cache_key, local_value, ttl_seconds)
return local_value
async def claim_affinity_pin(
cache: DualCache,
cache_key: str,
pins: RouterSessionPinStore,
cache_key: PinKey,
pin_value: Mapping[str, str],
ttl_seconds: int,
*,
@ -157,33 +48,20 @@ async def claim_affinity_pin(
Eligible claims refresh the returned winner. Legacy deployment claims only refresh
a matching candidate. Resolve Redis per call because the proxy attaches it lazily.
"""
redis_cache: Final = cache.redis_cache
if redis_cache is not None:
if pins.has_redis:
try:
claim_script: Final = redis_cache.async_register_script(_CLAIM_PIN_SCRIPT)
args: Final = (
json.dumps(dict(pin_value)),
int(ttl_seconds),
*(
(json.dumps(tuple(dict(value) for value in eligible_values)),)
if eligible_values is not None
else ()
),
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
winner: Final = await pins.claim(
cache_key,
pin_value,
ttl_seconds,
eligible_values=eligible_values,
preserve_legacy_model_ids=preserve_legacy_model_ids,
)
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
object, await claim_script(keys=(cache_key,), args=args)
)
decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
if not isinstance(decoded, str):
return pin_value
winner: Final = _decode_pin(decoded)
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
return claim_affinity_pin_in_memory(
cache,
else:
return pin_value if winner is None else winner
return pins.claim_local(
cache_key,
pin_value,
ttl_seconds,

View file

@ -8,11 +8,10 @@ Has 4 primary methods:
- async_get_cache
"""
import asyncio
import itertools
import logging
import time
from collections.abc import Awaitable, Mapping, Sequence
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from threading import Lock
from typing import TYPE_CHECKING, Any, Final
@ -24,7 +23,6 @@ from litellm.types.caching import RedisPipelineIncrementOperation
from .base_cache import BaseCache
from .in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache
from .redis_batch import BatchResult, RedisBatch, active_post_call_redis_batch, active_request_redis_batch
from .redis_cache import RedisCache, RedisCircuitBreakerOpenError, log_redis_failure
if TYPE_CHECKING:
@ -59,14 +57,6 @@ class PendingBatchRead:
previous_access_times: dict[str, float | None]
def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
if future.cancelled():
return
failure: Final = future.exception()
if failure is not None:
log_redis_failure(verbose_logger, logging.WARNING, "post-call Redis increment failed", failure)
class DualCache(BaseCache):
"""
DualCache is a cache implementation that updates both Redis and an in-memory cache simultaneously.
@ -269,9 +259,6 @@ class DualCache(BaseCache):
result = in_memory_result
if result is None and self.redis_cache is not None and local_only is False:
request_batch: Final = active_request_redis_batch(self.redis_cache)
if request_batch is not None and request_batch.read_as_missing(key):
return None
# If not found in in-memory cache, try fetching from Redis
redis_result: Final = await self.redis_cache.async_get_cache(key, parent_otel_span=parent_otel_span)
@ -316,20 +303,6 @@ class DualCache(BaseCache):
return sublist_keys, previous_access_times
def reserve_redis_batch_reads(self, keys: Sequence[str]) -> tuple[list[str], dict[str, float | None]]:
"""Reserve the memory-missed keys whose throttled Redis reads are due, as a batch read would."""
if self.redis_cache is None:
return [], {}
key_list: Final = list(keys)
memory: Final = self.in_memory_cache
in_memory_result: Final = (
None
if memory is None # pyright: ignore[reportUnnecessaryComparison] # handle an absent in-memory tier
else memory.batch_get_cache(key_list)
)
result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in key_list)
return self._reserve_redis_batch_keys(time.time(), key_list, result)
def _rollback_redis_batch_key_reservations(self, previous_access_times: dict[str, float | None]) -> None:
with self._last_redis_batch_access_time_lock:
for key, previous_time in previous_access_times.items():
@ -493,28 +466,6 @@ class DualCache(BaseCache):
verbose_logger, logging.ERROR, "LiteLLM Cache: exception in async add_cache", e, with_traceback=True
)
async def async_set_cache_pre_call(self, key: str, value: object, ttl: float | None) -> BatchResult[None] | None:
"""Memory now, the Redis SET on the request's pipeline, sent with the next read any caller awaits; None
when no pipeline is open, so the caller takes its direct path."""
batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache)
return None if batch is None else await self._set_on_batch(batch, key, value, ttl)
async def async_delete_cache_pre_call(self, key: str) -> Awaitable[None] | None:
"""Memory now, the Redis DEL on the request's pipeline; None when no pipeline is open, so the caller
takes its direct path."""
batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache)
if batch is None:
return None
if self.in_memory_cache is not None:
self.in_memory_cache.delete_cache(key)
return batch.delete(key)
async def _set_on_batch(self, batch: RedisBatch, key: str, value: object, ttl: float | None) -> BatchResult[None]:
effective_ttl: Final = self.default_in_memory_ttl if ttl is None else ttl
if self.in_memory_cache is not None:
await self.in_memory_cache.async_set_cache(key, value, ttl=effective_ttl)
return batch.set(key, value, effective_ttl)
# async_batch_set_cache
async def async_set_cache_pipeline(
self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs
@ -582,41 +533,6 @@ class DualCache(BaseCache):
)
return result
async def async_increment_cache_post_call(
self,
key: str,
value: float,
ttl: int | None,
parent_otel_span: Span | None = None,
) -> None:
"""Memory is incremented now; the Redis increment rides the request's post-call pipeline when one is
open, and runs on its own as ``async_increment_cache`` otherwise."""
await self.async_increment_cache_pipeline_post_call(
(RedisPipelineIncrementOperation(key=key, increment_value=value, ttl=ttl),), parent_otel_span
)
async def async_increment_cache_pipeline_post_call(
self,
increment_list: Sequence["RedisPipelineIncrementOperation"],
parent_otel_span: Span | None = None,
) -> None:
batch: Final = None if self.redis_cache is None else active_post_call_redis_batch(self.redis_cache)
operations: Final = list(increment_list)
if batch is None:
await self.async_increment_cache_pipeline(operations, parent_otel_span=parent_otel_span)
return
try:
if self.in_memory_cache is not None:
await self.in_memory_cache.async_increment_pipeline(
increment_list=operations, parent_otel_span=parent_otel_span
)
except Exception as e: # noqa: BLE001 # same tolerance as async_increment_cache_pipeline
log_redis_failure(verbose_logger, logging.WARNING, "in-memory increment failed", e)
for operation in increment_list:
batch.increment(operation["key"], operation["increment_value"], operation["ttl"]).on_settled(
_log_deferred_increment_failure
)
async def async_increment_cache_pipeline(
self,
increment_list: list["RedisPipelineIncrementOperation"],

View file

@ -1,453 +1,24 @@
"""One Redis pipeline for several independent operations, each with its own result and its own failure.
A ``RedisBatch`` collects MGETs, Lua scripts and increments declared by unrelated callers and sends them
in one ``pipeline(transaction=False)`` round trip. Every declaration returns an awaitable; awaiting one
flushes whatever has been declared so far, so callers keep their existing ``await`` shape and their own
error handling while sharing the wire. Redis Cluster clients run each operation on its own, as before:
a cluster pipeline is per node anyway and the existing per-operation paths already group by slot.
"""
"""The current request's native Redis batches: a pre-call batch for the reads it needs before answering and a
post-call batch for the writes nobody waits on, both owned by the native ``NativeRequestRedis``."""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import time
import weakref
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
from contextvars import ContextVar, Token, copy_context
from dataclasses import dataclass, field
from datetime import timedelta
from collections.abc import Awaitable, Callable, Generator, Sequence
from contextvars import ContextVar, Token
from enum import Enum, auto
from types import MappingProxyType, TracebackType
from types import TracebackType
from typing import TYPE_CHECKING, Final, Generic, Literal, Protocol, TypeAlias, TypeVar
from litellm._internal_context import (
REDIS_FAMILIES_METADATA_KEY,
current_service_target,
service_caller,
service_target,
)
from litellm._logging import verbose_logger
from litellm.caching.redis_cache import (
RedisCache,
_get_call_stack_info, # pyright: ignore[reportPrivateUsage] # same caller chain every RedisCache method reports
_run_under_circuit_breaker, # pyright: ignore[reportPrivateUsage] # same health signal as every RedisCache method
log_redis_failure,
)
from litellm.caching.redis_cluster_cache import RedisClusterCache
from litellm.types.services import ServiceTypes
if TYPE_CHECKING:
from litellm.rust_bridge._native import NativeRequestRedis, NativeSend
_T = TypeVar("_T")
_T_co: Final = TypeVar("_T_co", covariant=True)
_ScriptArg = str | bytes | int | float
SettledHook = Callable[[asyncio.Future[_T]], Awaitable[None] | None]
POST_CALL_FLUSH_DEADLINE_SECONDS: Final = 1.0
class RegisteredScript(Protocol):
def __call__(self, keys: Sequence[str], args: Sequence[_ScriptArg]) -> Awaitable[object]: ...
class _RedisPipeline(Protocol):
def mget(self, keys: Sequence[str]) -> object: ...
def evalsha(self, sha: str, numkeys: int, *keys_and_args: _ScriptArg) -> object: ...
def incrbyfloat(self, name: str, amount: float) -> object: ...
def expire(self, name: str, time: timedelta) -> object: ...
def set(self, name: str, value: str, ex: timedelta | None = None) -> object: ...
def delete(self, *names: str) -> object: ...
async def execute(self, raise_on_error: bool = True) -> list[object]: ...
class _Op(Generic[_T]):
"""One declared operation: how many pipeline replies it consumes, how to turn them into a result, and
how to run on its own when the batch cannot pipeline (cluster client, or a reply the pipeline cannot
settle, like NOSCRIPT)."""
__slots__ = ("caller", "future", "settled_hooks", "target")
def __init__(self) -> None:
self.future: Final[asyncio.Future[_T]] = asyncio.get_running_loop().create_future()
self.future.add_done_callback(_mark_retrieved)
self.settled_hooks: Final[list[SettledHook[_T]]] = [] # mutable-ok: append-only registry
self.target: Final = current_service_target()
self.caller: Final = _get_call_stack_info()
async def run_settled_hooks(self) -> None:
for hook in self.settled_hooks:
await self._run_settled_hook(hook)
async def _run_settled_hook(self, hook: SettledHook[_T]) -> None:
try:
follow_up: Final = hook(self.future)
if follow_up is not None:
await follow_up
except Exception as e: # noqa: BLE001 # one owner's follow-up must not stop the others
verbose_logger.warning("redis batch settled hook failed: %s", e)
def enqueue(self, pipe: _RedisPipeline) -> int:
raise NotImplementedError
def resolve(self, replies: Sequence[object]) -> _T:
raise NotImplementedError
async def run_alone(self) -> _T:
raise NotImplementedError
def settle(self, replies: Sequence[object]) -> Awaitable[None] | None:
"""Resolve from pipeline replies; return a coroutine when the op has to be retried on its own."""
failure: Final = next((reply for reply in replies if isinstance(reply, Exception)), None)
if failure is None:
try:
self.future.set_result(self.resolve(replies))
except Exception as e: # noqa: BLE001 # a reply this op cannot decode fails this op alone
self.future.set_exception(e)
return None
if _is_missing_script(failure):
return self._settle_alone()
self.future.set_exception(failure)
return None
async def _settle_alone(self) -> None:
try:
with service_target(self.target), service_caller(self.caller):
self.future.set_result(await self.run_alone())
except Exception as e: # noqa: BLE001 # the declaring caller owns the failure of its own operation
self.future.set_exception(e)
def _is_missing_script(failure: Exception) -> bool:
"""Imported lazily: this module is reachable from a base ``import litellm`` while redis is not a base dependency."""
from redis.exceptions import NoScriptError
return isinstance(failure, NoScriptError)
def _mark_retrieved(future: asyncio.Future[object]) -> None:
"""A caller that stops awaiting (cancelled request) must not leave an 'exception never retrieved' log."""
if not future.cancelled():
future.exception()
class _MGet(_Op[Mapping[str, object]]):
__slots__ = ("_keys", "_redis_cache")
def __init__(self, redis_cache: RedisCache, keys: Sequence[str]) -> None:
super().__init__()
self._redis_cache: Final = redis_cache
self._keys: Final[tuple[str, ...]] = tuple(dict.fromkeys(keys))
def enqueue(self, pipe: _RedisPipeline) -> int:
pipe.mget(tuple(self._redis_cache.check_and_fix_namespace(key=key) for key in self._keys))
return 1
def resolve(self, replies: Sequence[object]) -> Mapping[str, object]:
values: Final = replies[0]
if not isinstance(values, (list, tuple)):
raise TypeError(f"MGET reply is not a list: {type(values).__name__}")
return MappingProxyType(
{key: self._redis_cache._get_cache_logic(value) for key, value in zip(self._keys, values)} # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # shared decode with async_batch_get_cache
)
async def run_alone(self) -> Mapping[str, object]:
found: Mapping[str, object] = await self._redis_cache.async_batch_get_cache(key_list=list(self._keys)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped cache API
if any(key not in found for key in self._keys):
raise ConnectionError("batch get did not return every key")
return found
class _Script(_Op[object]):
__slots__ = ("_args", "_keys", "_redis_cache", "_run", "_sha")
def __init__(
self,
redis_cache: RedisCache,
source: str,
run: RegisteredScript,
keys: Sequence[str],
args: Sequence[_ScriptArg],
) -> None:
super().__init__()
self._redis_cache: Final = redis_cache
self._sha: Final = hashlib.sha1(source.encode()).hexdigest() # noqa: S324 # EVALSHA identifies scripts by SHA-1
self._run: Final = run
self._keys: Final[tuple[str, ...]] = tuple(keys)
self._args: Final[tuple[_ScriptArg, ...]] = tuple(args)
def enqueue(self, pipe: _RedisPipeline) -> int:
namespaced: Final = tuple(self._redis_cache.check_and_fix_namespace(key=key) for key in self._keys)
pipe.evalsha(self._sha, len(namespaced), *namespaced, *self._args)
return 1
def resolve(self, replies: Sequence[object]) -> object:
return replies[0]
async def run_alone(self) -> object:
return await self._run(keys=self._keys, args=self._args)
class _Increment(_Op[float]):
__slots__ = ("_key", "_redis_cache", "_ttl", "_value")
def __init__(self, redis_cache: RedisCache, key: str, value: float, ttl: int | None) -> None:
super().__init__()
self._redis_cache: Final = redis_cache
self._key: Final = key
self._value: Final = value
self._ttl: Final = ttl
def enqueue(self, pipe: _RedisPipeline) -> int:
name: Final = self._redis_cache.check_and_fix_namespace(key=self._key)
pipe.incrbyfloat(name, self._value)
if self._ttl is None:
return 1
pipe.expire(name, timedelta(seconds=self._ttl))
return 2
def resolve(self, replies: Sequence[object]) -> float:
reply: Final = replies[0]
if not isinstance(reply, (int, float, str, bytes)):
raise TypeError(f"INCRBYFLOAT reply is not numeric: {type(reply).__name__}")
return float(reply)
async def run_alone(self) -> float:
value: object = await self._redis_cache.async_increment(key=self._key, value=self._value, ttl=self._ttl) # pyright: ignore[reportUnknownMemberType] # untyped cache API
if not isinstance(value, (int, float)):
raise TypeError(f"increment did not return a number: {type(value).__name__}")
return float(value)
class _Set(_Op[None]):
"""SET with the cache's TTL rules, same encoding as ``async_set_cache_pipeline_with_ttls``."""
__slots__ = ("_key", "_redis_cache", "_ttl", "_value")
def __init__(self, redis_cache: RedisCache, key: str, value: object, ttl: float | None) -> None:
super().__init__()
self._redis_cache: Final = redis_cache
self._key: Final = key
self._value: Final = value
self._ttl: Final = ttl
def enqueue(self, pipe: _RedisPipeline) -> int:
ttl: Final = self._redis_cache.get_ttl(ttl=self._ttl) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped cache API
pipe.set(
self._redis_cache.check_and_fix_namespace(key=self._key),
json.dumps(self._value),
ex=None if ttl is None else timedelta(seconds=ttl),
)
return 1
def resolve(self, replies: Sequence[object]) -> None:
return None
async def run_alone(self) -> None:
await self._redis_cache.async_set_cache_pipeline_with_ttls(((self._key, self._value, self._ttl),))
class _Delete(_Op[None]):
"""DEL of one key, the pipelined twin of ``async_delete_cache``."""
__slots__ = ("_key", "_redis_cache")
def __init__(self, redis_cache: RedisCache, key: str) -> None:
super().__init__()
self._redis_cache: Final = redis_cache
self._key: Final = key
def enqueue(self, pipe: _RedisPipeline) -> int:
pipe.delete(self._redis_cache.check_and_fix_namespace(key=self._key))
return 1
def resolve(self, replies: Sequence[object]) -> None:
return None
async def run_alone(self) -> None:
await self._redis_cache.async_delete_cache(self._key)
class BatchResult(Generic[_T]):
"""Awaitable handle for one declared operation; awaiting it flushes the batch it belongs to."""
__slots__ = ("_batch", "_op")
def __init__(self, batch: RedisBatch, op: _Op[_T]) -> None:
self._batch: Final = batch
self._op: Final = op
def __await__(self) -> Generator[object, None, _T]:
return self._wait().__await__()
async def _wait(self) -> _T:
if not self._op.future.done():
await self._batch.flush()
return self._op.future.result()
@property
def done(self) -> bool:
return self._op.future.done()
def on_settled(self, hook: SettledHook[_T]) -> None:
"""For an owner that does not await: runs inside the flush once this operation has its result or
failure (or was cancelled with the pipeline), so the flush completes with the follow-up done."""
self._op.settled_hooks.append(hook)
@dataclass(slots=True)
class RedisBatch:
"""Operations declared here go out in one pipeline the next time any of them is awaited or ``flush`` runs."""
redis_cache: RedisCache
name: str = "redis_batch"
_pending: list[_Op[object]] = field(default_factory=list) # mutable-ok: drained by flush
_flush_hooks: list[Callable[[], None]] = field(default_factory=list) # mutable-ok: append-only registry
_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
_misses: set[str] = field(default_factory=set) # mutable-ok: keys an MGET of this request read as absent
flushes: int = 0
def mget(self, keys: Sequence[str]) -> BatchResult[Mapping[str, object]]:
op: Final = _MGet(self.redis_cache, keys)
op.future.add_done_callback(self._note_misses)
return self._declare(op)
def _note_misses(self, future: asyncio.Future[Mapping[str, object]]) -> None:
if future.cancelled() or future.exception() is not None:
return
self._misses.update(key for key, value in future.result().items() if value is None)
def read_as_missing(self, key: str) -> bool:
"""True when an MGET on this batch already found no value under ``key`` and nothing has set it since,
so a per-key GET later in the same request can be answered without another round trip."""
return key in self._misses
def script(
self, source: str, run: RegisteredScript, keys: Sequence[str], args: Sequence[_ScriptArg]
) -> BatchResult[object]:
return self._declare(_Script(self.redis_cache, source, run, keys, args))
def increment(self, key: str, value: float, ttl: int | None = None) -> BatchResult[float]:
return self._declare(_Increment(self.redis_cache, key, value, ttl))
def set(self, key: str, value: object, ttl: float | None = None) -> BatchResult[None]:
self._misses.discard(key)
return self._declare(_Set(self.redis_cache, key, value, ttl))
def delete(self, key: str) -> BatchResult[None]:
self._misses.add(key)
return self._declare(_Delete(self.redis_cache, key))
def add_flush_hook(self, hook: Callable[[], None]) -> None:
"""Called at the start of every flush so lazily bound readers can declare their keys into the same trip."""
self._flush_hooks.append(hook)
@property
def pending(self) -> int:
return len(self._pending)
def _declare(self, op: _Op[_T]) -> BatchResult[_T]:
self._pending.append(op) # pyright: ignore[reportArgumentType] # heterogeneous ops share the flush loop
return BatchResult(self, op)
async def flush(self) -> None:
async with self._lock:
for hook in self._flush_hooks:
hook()
ops: Final = tuple(self._pending)
self._pending.clear()
if not ops:
return
self.flushes += 1
try:
if isinstance(self.redis_cache, RedisClusterCache):
await asyncio.gather(*(op._settle_alone() for op in ops)) # pyright: ignore[reportPrivateUsage] # batch owns its ops
else:
await self._flush_pipeline(ops)
finally:
for op in ops:
if not op.future.done():
op.future.cancel()
await asyncio.gather(*(op.run_settled_hooks() for op in ops))
async def _flush_pipeline(self, ops: Sequence[_Op[object]]) -> None:
start_time: Final = time.time()
target, metadata = _pipeline_service_event(ops)
widths: list[int] = [] # mutable-ok: filled while enqueuing
async def run() -> list[object]:
client: Final = self.redis_cache.init_async_client()
async with client.pipeline(transaction=False) as pipe:
widths.extend(op.enqueue(pipe) for op in ops)
return await pipe.execute(raise_on_error=False)
try:
replies: Final = await _run_under_circuit_breaker(self.redis_cache._health, self.name, run) # pyright: ignore[reportPrivateUsage] # same breaker as the cache's own methods
except Exception as e: # noqa: BLE001 # each declaring caller applies its own Redis fallback
log_redis_failure(verbose_logger, logging.WARNING, f"{self.name}: pipeline of {len(ops)} ops failed", e)
with service_target(target):
asyncio.create_task(
self.redis_cache.service_logger_obj.async_service_failure_hook(
service=ServiceTypes.REDIS,
duration=time.time() - start_time,
error=e,
call_type=self.name,
start_time=start_time,
end_time=time.time(),
event_metadata=metadata,
)
)
for op in ops:
op.future.set_exception(e)
return
with service_target(target):
asyncio.create_task(
self.redis_cache.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=time.time() - start_time,
call_type=self.name,
start_time=start_time,
end_time=time.time(),
event_metadata=metadata,
)
)
retries: list[Awaitable[None]] = [] # mutable-ok: collected while slicing replies
offset = 0
for op, width in zip(ops, widths):
retry = op.settle(replies[offset : offset + width])
offset += width
if retry is not None:
retries.append(retry)
if retries:
await asyncio.gather(*retries)
MIXED_PIPELINE_TARGET: Final = "mixed"
def _pipeline_service_event(ops: Sequence[_Op[object]]) -> tuple[str | None, dict[str, int | str]]:
"""The target and metadata of one pipeline flush: the one key family every op was declared under, or
``"mixed"`` plus the sorted families when owners of several families share the trip."""
families: Final = sorted({op.target for op in ops if op.target is not None})
if len(families) > 1:
return MIXED_PIPELINE_TARGET, {"op_count": len(ops), REDIS_FAMILIES_METADATA_KEY: ",".join(families)}
return next(iter(families), None), {"op_count": len(ops)}
def _backend_key(redis_cache: RedisCache) -> object:
"""Two ``RedisCache`` instances built from the same connection settings and namespace talk to the same server
under the same key prefix, so the proxy's cache and the router's cache share one pipeline (the router gets its
port as a string, hence the ``str`` comparison); a cache whose settings cannot be compared (a test double) gets
its own."""
try:
settings: Final = tuple(sorted((str(k), str(v)) for k, v in redis_cache.redis_kwargs.items() if v is not None)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportUnknownArgumentType] # untyped cache API
except AttributeError:
return ("instance", id(redis_cache))
return (type(redis_cache), redis_cache.namespace, settings)
class NativePending(Protocol[_T_co]):
def resolve(self) -> Awaitable[_T_co]: ...
@ -511,30 +82,22 @@ def _native_send(phase: RequestPhase, request: NativeRequestRedis) -> NativeSend
_open_post_call: Final[weakref.WeakSet[RequestRedisBatches]] = weakref.WeakSet()
"""Requests whose post-call batch still holds declared ops, so a shutdown can send them before Redis goes away."""
"""Requests whose post-call batch may still hold declared ops, so a shutdown can send them before Redis goes away."""
class RequestRedisBatches:
"""One ``RedisBatch`` per Redis backend for the current request, so readers of different caches that
share a server (the proxy's and the router's) share the pipeline.
The post-call batches hold the writes nothing waits on (counters, token scripts, the response cache).
They flush once, when the success or failure callbacks have all run, or at ``post_call_deadline``
seconds after the first declaration when no callback phase closes them."""
"""The request's native batches. The post-call batch goes out once, when the success or failure callbacks have
all run, or ``post_call_deadline`` seconds after its first declaration when no callback phase closes it."""
__slots__ = (
"__weakref__",
"_batches",
"_deadline",
"_deadline_flush",
"_native",
"_native_context",
"_native_factory",
"_native_flush_hooks",
"_native_misses",
"_native_send_factory",
"_post_call",
"_post_call_settlements",
"_post_call_unsent",
"post_call_deadline",
"prefetched",
)
@ -545,54 +108,27 @@ class RequestRedisBatches:
native_factory: NativeRequestFactory = _native_request,
native_send_factory: NativeSendFactory = _native_send,
) -> None:
self._batches: Final[dict[object, RedisBatch]] = {} # mutable-ok: lazily filled per backend
self._post_call: Final[dict[object, RedisBatch]] = {} # mutable-ok: lazily filled per backend
self._post_call_settlements: tuple[PostCallSettlement, ...] = ()
self._post_call_unsent = False
self.post_call_deadline: Final = post_call_deadline
self._native_factory: Final = native_factory
self._native_send_factory: Final = native_send_factory
self._native: NativeRequestRedis | None = None
self._native_context: Final = copy_context()
self._native_flush_hooks: Final[list[Callable[[], None]]] = [] # mutable-ok: append-only registry
self._native_misses: frozenset[str] = frozenset()
self._deadline: asyncio.TimerHandle | None = None
self._deadline_flush: asyncio.Task[None] | None = None
# Reads declared early for a consumer that runs later in the request, keyed by consumer name.
self.prefetched: Final[dict[str, object]] = {} # mutable-ok: armed pre-admission, taken at use
def batch(self, redis_cache: RedisCache) -> RedisBatch:
key: Final = _backend_key(redis_cache)
batch = self._batches.get(key)
if batch is None:
batch = RedisBatch(redis_cache, name="request_redis_batch")
self._batches[key] = batch
return batch
def post_call(self, redis_cache: RedisCache) -> RedisBatch:
key: Final = _backend_key(redis_cache)
existing: Final = self._post_call.get(key)
batch: Final = (
existing
if existing is not None
else self._post_call.setdefault(key, RedisBatch(redis_cache, name="post_call_redis_batch"))
)
if self._deadline is None:
self._deadline = asyncio.get_running_loop().call_later(self.post_call_deadline, self._flush_on_deadline)
_open_post_call.add(self)
return batch
def native(self) -> NativeRequestRedis:
"""The request's native batches, built on first use so requests that touch no native store pay nothing.
They keep a copy of the context they are built in, and the cycle collector cannot see through native
objects, so they are built in the context this scope was opened from, which cannot reach the scope."""
"""The request's native batches, built on first use so requests that touch no native store pay nothing."""
if self._native is None:
self._native = self._native_context.run(self._native_factory, self.post_call_deadline)
self._native = self._native_factory(self.post_call_deadline)
return self._native
def native_post_call(self) -> NativeRequestRedis:
"""``native`` for a post-call declaration, so a shutdown drain also flushes it."""
_open_post_call.add(self)
self._post_call_unsent = True
return self.native()
def native_send(self, phase: RequestPhase) -> NativeSend:
@ -612,7 +148,7 @@ class RequestRedisBatches:
self._native_flush_hooks.append(hook)
def note_native_misses(self, keys: Sequence[str]) -> None:
"""Keys a native pre-call read found absent (or a native DEL removed), like ``RedisBatch.read_as_missing``."""
"""Keys a native pre-call read found absent, or a native DEL removed, so a later GET can skip Redis."""
self._native_misses = self._native_misses | frozenset(keys)
def forget_native_miss(self, key: str) -> None:
@ -628,31 +164,21 @@ class RequestRedisBatches:
hook()
finally:
_active_request_batches.reset(token)
await self.flush_pre_call()
async def flush_pre_call(self) -> None:
"""Send the pre-call ops declared so far, such as write-backs nobody awaits before the request scope closes.
The pre-call flush hooks are skipped: they declare reads for a reader about to await, and none is waiting."""
if self._native is not None:
await self._native.flush_pre()
def _flush_on_deadline(self) -> None:
self._deadline = None
self._deadline_flush = asyncio.ensure_future(self.flush_post_call())
async def flush_all(self) -> None:
"""Batch whatever is still declared (write-backs nobody awaits) before the request scope closes. The pre-call
flush hooks are skipped: they declare reads for later readers, and none are left."""
await asyncio.gather(
*(batch.flush() for batch in self._batches.values() if batch.pending),
*(() if self._native is None else (self._native.flush_pre(),)),
)
async def flush_post_call(self) -> None:
"""One pipeline per backend for the post-call writes; the deadline is disarmed since this is that flush."""
if self._deadline is not None:
self._deadline.cancel()
self._deadline = None
await asyncio.gather(
*(batch.flush() for batch in self._post_call.values() if batch.pending),
*(() if self._native is None else (self._native.flush_post(),)),
)
if not any(batch.pending for batch in self._post_call.values()):
"""Send the post-call writes. The request stays registered with the shutdown drain until they are back, so a
drain that overlaps this flush waits for it, and an op declared while it runs keeps the request registered."""
self._post_call_unsent = False
if self._native is not None:
await self._native.flush_post()
if not self._post_call_unsent:
_open_post_call.discard(self)
def track_post_call_settlement(self, settlement: PostCallSettlement) -> None:
@ -663,36 +189,16 @@ class RequestRedisBatches:
def post_call_settlements(self) -> tuple[PostCallSettlement, ...]:
return self._post_call_settlements
@property
def batches(self) -> tuple[RedisBatch, ...]:
return tuple(self._batches.values())
_active_request_batches: Final[ContextVar[RequestRedisBatches | None]] = ContextVar(
"request_redis_batches", default=None
)
def active_request_redis_batch(redis_cache: RedisCache) -> RedisBatch | None:
"""The request's batch for this backend, or None outside a ``request_redis_batch_scope``."""
batches: Final = _active_request_batches.get()
if batches is None:
return None
return batches.batch(redis_cache)
def active_request_redis_batches() -> RequestRedisBatches | None:
return _active_request_batches.get()
def active_post_call_redis_batch(redis_cache: RedisCache) -> RedisBatch | None:
"""The request's post-call batch for this backend, or None outside a ``request_redis_batch_scope``."""
batches: Final = _active_request_batches.get()
if batches is None:
return None
return batches.post_call(redis_cache)
async def flush_post_call_redis_batches() -> None:
"""Called where the success and failure callbacks of a request have all run."""
batches: Final = _active_request_batches.get()
@ -706,7 +212,7 @@ async def drain_post_call_redis_batches() -> None:
class request_redis_batch_scope:
"""Redis reads declared inside share one pipeline per backend; nested scopes join the outer one."""
"""Redis ops declared inside share the request's native batches; nested scopes join the outer one."""
__slots__ = ("_native_factory", "_native_send_factory", "_post_call_deadline", "_token")

View file

@ -29,7 +29,6 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import current_service_caller
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import (
DEFAULT_REDIS_MAJOR_VERSION,
@ -112,7 +111,6 @@ class _SupportsScriptLoad(Protocol):
_GENERIC_CALLER_MODULES: Final = frozenset(
{
__name__,
"litellm.caching.redis_batch",
"litellm.caching.dual_cache",
"litellm.caching.caching",
"litellm.rust_bridge.execution",
@ -125,8 +123,6 @@ _GENERIC_CALLER_FRAME_NAMES: Final = frozenset(
"wrapper",
"_run_under_circuit_breaker",
"_run_under_circuit_breaker_sync",
"run_alone",
"_settle_alone",
"get_cache",
"set_cache",
"async_get_cache",
@ -184,11 +180,9 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
Get the function names of the nearest meaningful callers of the cache method.
Frames that merely forward the call (this module's circuit-breaker guards, the
cache facades, the batch pipeline's retry path, generic cache verbs) are
skipped, and the walk stops at the event loop, so the chain names the litellm
code that wanted the call. When nothing but forwarding frames is found (the call
runs in a task of its own, like a batch op retried on the flush) the chain the
declaring code threaded through ``service_caller`` is reported, else ``unknown``.
cache facades, generic cache verbs) are skipped, and the walk stops at the event
loop, so the chain names the litellm code that wanted the call, or ``unknown``
when nothing but forwarding frames is found.
Args:
num_frames: Number of previous frames to include (default: 2)
@ -219,9 +213,7 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
)
function_names: Final = leading_names + further_names
if function_names:
return " <- ".join(function_names)
return current_service_caller() or "unknown"
return " <- ".join(function_names) if function_names else "unknown"
except Exception:
return "unknown"

View file

@ -163,6 +163,8 @@ DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COO
DEFAULT_REDIS_BATCH_CACHE_EXPIRY_SECONDS: Final = 10
DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5)
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
DEFAULT_REPLICATE_SYNC_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_SYNC_POLLING_RETRIES", 120)
REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES: Final = get_env_int("REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES", 512 * 1024 * 1024)
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
@ -448,6 +450,7 @@ MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body"
MCP_ADMISSION_BODY_PEEK_TIMEOUT_SECONDS: Final = 5.0
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
REALTIME_METERED_USAGE_KEY: Final = "realtime_metered_usage"
FAILED_PROMPT_TOKEN_ESTIMATE_KEY: Final = "failed_prompt_token_estimate"
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS: Final = 30
OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS: Final = 5
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
@ -493,6 +496,7 @@ REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_team_spend_upda
REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_org_spend_update_buffer"
REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_spend_update_buffer"
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_customer_member_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
@ -1997,7 +2001,7 @@ BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = get_env_float(
"BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", 60.0
)
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = get_env_float(
"BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 3600.0
"BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 7800.0
)
_background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"

View file

@ -101,6 +101,7 @@ from litellm.llms.openai.cost_calculation import (
cost_per_token as openai_cost_per_token,
)
from litellm.llms.openai.cost_calculation import (
video_clip_cost,
video_output_cost_per_second,
)
from litellm.llms.perplexity.cost_calculator import (
@ -126,6 +127,7 @@ from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
from litellm.types.extraction import ExtractionResponse, ExtractionUsage
from litellm.types.interactions import InteractionsAPIResponse
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -1144,11 +1146,29 @@ def _served_or_priced_requested_model(
if served in (requested, requested.partition("/")[2]):
return served
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
profile_of_served: Final = _is_bedrock_cross_region_profile_of(requested_model, served_model, pricing_provider)
if profile_of_served and _resolves_to_rates(requested, pricing_provider):
return requested
if _resolves_to_rates(served, pricing_provider):
return served
return requested if _resolves_to_rates(requested, pricing_provider) else served
def _is_bedrock_cross_region_profile_of(profile: str, served_model: str, custom_llm_provider: str | None) -> bool:
if custom_llm_provider != "bedrock":
return False
from litellm.llms.bedrock.common_utils import (
get_bedrock_cross_region_inference_regions,
strip_bedrock_routing_prefix,
)
region, _, profiled_model = strip_bedrock_routing_prefix(profile).partition(".")
return (
region in get_bedrock_cross_region_inference_regions()
and strip_bedrock_routing_prefix(served_model) == profiled_model
)
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
@ -1187,6 +1207,8 @@ def _get_response_model(completion_response: object) -> str | None:
if completion_response is None:
return None
if isinstance(completion_response, InteractionsAPIResponse):
return completion_response.model or completion_response.agent
if isinstance(completion_response, BaseModel):
return getattr(completion_response, "model", None)
elif isinstance(completion_response, dict):
@ -2070,25 +2092,21 @@ def completion_cost(
litellm_logging_obj, custom_pricing, router_model_id
)
usage_obj = getattr(completion_response, "usage", None)
_video_usage_obj: object = attribute_or_none(completion_response, "usage")
duration_seconds: float | None = None
video_resolution: str | None = None
provider_reported_cost: float | None = None
if completion_response is not None and usage_obj:
# Handle both dict and Pydantic Usage object
if isinstance(usage_obj, dict):
duration_seconds = usage_obj.get("duration_seconds", None)
_vr = usage_obj.get("video_resolution", None)
provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
_vc = usage_obj.get("video_count", None)
else:
duration_seconds = getattr(usage_obj, "duration_seconds", None)
_vr = getattr(usage_obj, "video_resolution", None)
provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
_vc = getattr(usage_obj, "video_count", None)
if _vr is not None:
video_resolution = str(_vr).strip().lower()
video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1
if completion_response is not None and _video_usage_obj:
duration_seconds = _VIDEO_USAGE_NUMBER.validate_python(
_video_usage_value(_video_usage_obj, "duration_seconds")
)
provider_reported_cost = _VIDEO_USAGE_NUMBER.validate_python(
_video_usage_value(_video_usage_obj, "provider_reported_cost_usd")
)
if (
video_resolution_value := _video_usage_value(_video_usage_obj, "video_resolution")
) is not None:
video_resolution = str(video_resolution_value).strip().lower()
if _video_model_info is None and provider_reported_cost is not None:
return float(provider_reported_cost)
@ -2105,7 +2123,7 @@ def completion_cost(
custom_llm_provider=custom_llm_provider,
model_info=_video_model_info,
video_resolution=video_resolution,
) * video_count + _video_input_image_cost(
) * _video_count(_video_usage_obj) + _video_input_image_cost(
requested_model or model,
custom_llm_provider,
_video_model_info,
@ -2118,7 +2136,7 @@ def completion_cost(
custom_llm_provider=custom_llm_provider,
model_info=_video_model_info,
video_resolution=video_resolution,
)
) * _video_count(_video_usage_obj)
elif call_type in _SPEECH_CALL_TYPES:
prompt_characters = _speech_billed_characters(
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
@ -2631,6 +2649,19 @@ def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
_VIDEO_USAGE_NUMBER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
def _video_usage_value(usage: object, field: str) -> object:
fields: Final = as_str_mapping(usage)
return fields.get(field) if fields is not None else attribute_or_none(usage, field)
def _video_count(usage: object) -> int:
count: Final = _video_usage_value(usage, "video_count")
return count if type(count) is int and count > 1 else 1
def _video_input_image_cost(
model: str,
custom_llm_provider: str | None,
@ -3122,6 +3153,9 @@ def default_image_cost_calculator(
return cost
_VIDEO_COST_INFO: Final = TypeAdapter(Mapping[str, object])
def default_video_cost_calculator(
model: str,
duration_seconds: float,
@ -3184,6 +3218,14 @@ def default_video_cost_calculator(
if cost_info is None:
raise Exception(f"Model not found in cost map for model={model}")
clip_cost: Final = video_clip_cost(_VIDEO_COST_INFO.validate_python(cost_info), duration_seconds, video_resolution)
if clip_cost is not None:
return clip_cost
video_cost: Final = cost_info.get("output_cost_per_video")
if video_cost is not None:
return video_cost
# Check for video-specific cost per second first
video_cost_per_second: Final = cost_info.get("output_cost_per_video_per_second")
if video_cost_per_second is not None:

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:ae9575b121f5776e2f04e192064bc2c5665af9d1c425d620dc773aca31698712
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:ae9575b121f5776e2f04e192064bc2c5665af9d1c425d620dc773aca31698712
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

View file

@ -96,25 +96,31 @@
/v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*,
/v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix.
*/}}
{{- $gatewayPrefixes := list
"/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings"
"/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images"
"/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning"
"/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
"/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
"/v1/models" "/models" "/openai" "/engines"
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" "/api/event_logging"
"/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
"/v1beta" "/interactions"
"/anthropic" "/azure" "/azure_ai" "/azure_speech" "/aws" "/bedrock" "/comprehendmedical" "/transcribe" "/cohere" "/gemini" "/google"
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
"/toolset"
"/v1/realtime" "/realtime"
"/health" "/metrics"
{{- $gatewayPrefixGroups := list
(list "/chat" "/v1/chat") (list "/completions" "/v1/completions") (list "/embeddings" "/v1/embeddings")
(list "/moderations" "/v1/moderations") (list "/audio" "/v1/audio") (list "/images" "/v1/images")
(list "/files" "/v1/files") (list "/batches" "/v1/batches") (list "/fine_tuning" "/v1/fine_tuning")
(list "/fine-tuning" "/v1/fine-tuning") (list "/responses" "/v1/responses") (list "/threads" "/v1/threads")
(list "/assistants" "/v1/assistants") (list "/vector_stores" "/v1/vector_stores") (list "/models" "/v1/models")
(list "/messages" "/v1/messages") (list "/a2a" "/v1/a2a") (list "/rerank" "/v1/rerank")
(list "/decisions" "/v1/decisions") (list "/systemone" "/v1/systemone") (list "/extraction" "/v1/extraction")
(list "/ocr" "/v1/ocr") (list "/rag" "/v1/rag") (list "/video" "/v1/video")
(list "/videos" "/v1/videos") (list "/search" "/v1/search") (list "/containers" "/v1/containers")
(list "/realtime" "/v1/realtime") (list "/live/sessions" "/v1/live/sessions") (list "/v1/skills" "/v1/indexes")
(list "/v1/evals" "/v2/rerank") (list "/v1beta" "/v1/memory") (list "/openai" "/engines")
(list "/queue/chat" "/api/event_logging") (list "/toolset" "/interactions") (list "/azure" "/anthropic")
(list "/azure_ai" "/azure_speech") (list "/aws" "/bedrock") (list "/transcribe" "/comprehendmedical")
(list "/cohere" "/gemini") (list "/google" "/gigachat") (list "/vertex_ai" "/vertex-ai")
(list "/assemblyai" "/eu.assemblyai") (list "/fal_ai" "/deepgram") (list "/vllm" "/langfuse")
(list "/mistral" "/typesafe") (list "/openrouter" "/nvidia_nim") (list "/groq" "/voyage")
(list "/cursor" "/milvus") (list "/tinyfish" "/openai_passthrough") (list "/watsonx")
(list "/health" "/metrics")
-}}
{{- $gatewayPrefixes := list -}}
{{- range $gatewayPrefixGroups -}}
{{- $gatewayPrefixes = concat $gatewayPrefixes . -}}
{{- end -}}
{{- $gatewayExactPaths := list "/test" "/debug/memory/summary" "/eu_jev/v1/systemone" -}}
{{/*
/test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its
children /test/connection and /test/tools/list are MCP-server management
@ -131,7 +137,16 @@
at "/" Prefix would swallow the whole backend management API) instead of
adding to it.
*/}}
{{- $builtinPathKeys := ternary (list "/|Prefix") (list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix") $monolith -}}
{{- $builtinPathKeys := list "/|Prefix" -}}
{{/*
An ALB allows 100 listener rules and the AWS Load Balancer Controller makes
one per Ingress path, so on alb each $gatewayPrefixGroups entry is one rule.
Its first, shorter prefix is the Ingress path, which keeps any longer child
path ahead of it in the controller's longest-first order, and a conditions
annotation adds the other prefix to the same rule.
*/}}
{{- $albGrouping := and (eq $controller "alb") (not $monolith) -}}
{{- $gatewayConditions := get (.Values.ingress.annotations | default dict) (printf "alb.ingress.kubernetes.io/conditions.%s" $gatewayName) | default "[]" | trim | trimPrefix "[" | trimSuffix "]" | trim -}}
{{- if $hasComponentHosts }}
apiVersion: networking.k8s.io/v1
kind: Ingress
@ -198,9 +213,20 @@ metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
{{- if or .Values.ingress.annotations $albGrouping }}
annotations:
{{- with .Values.ingress.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if $albGrouping }}
{{- range $i, $group := $gatewayPrefixGroups }}
alb.ingress.kubernetes.io/actions.litellm-gateway-{{ $i }}: {{ printf `{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"%s","servicePort":"http"}]}}` $gatewayName | quote }}
{{- $conditions := compact (list $gatewayConditions (ternary (printf `{"field":"path-pattern","pathPatternConfig":{"values":["%s","%s/*"]}}` (last $group) (last $group)) "" (eq (len $group) 2))) }}
{{- with $conditions }}
alb.ingress.kubernetes.io/conditions.litellm-gateway-{{ $i }}: {{ printf "[%s]" (join "," .) | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
spec:
{{- with .Values.ingress.className }}
@ -234,21 +260,31 @@ spec:
# /test/* MCP management endpoints fall to the backend catch-all.
# Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory
# gate); the rest of /debug/* stays on the backend.
- path: /test
{{- range $gatewayExactPaths }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Exact" .) }}
- path: {{ . }}
pathType: Exact
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
- path: /debug/memory/summary
pathType: Exact
{{- end }}
{{- if $albGrouping }}
{{- range $i, $group := $gatewayPrefixGroups }}
{{- range $group }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
{{- end }}
- path: {{ first $group }}
pathType: Prefix
backend:
service:
name: {{ $gatewayName }}
name: litellm-gateway-{{ $i }}
port:
number: {{ $gatewayPort }}
{{- range $gatewayPrefixes }}
name: use-annotation
{{- end }}
{{- end }}
{{- range (ternary list $gatewayPrefixes $albGrouping) }}
{{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
- path: {{ . }}

View file

@ -18,16 +18,8 @@ tests:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /eu.assemblyai
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- notExists:
path: spec.rules[0].http.paths[?(@.backend.service.port.name == "use-annotation" && @.pathType != "Prefix")]
- contains:
path: spec.rules[0].http.paths
content:
@ -39,6 +31,39 @@ tests:
port:
number: 3000
- it: fits the AWS Load Balancer Controller's one rule per path under the default 100 rules per ALB
set:
ingress.enabled: true
asserts:
- notExists:
path: spec.rules[0].http.paths[100]
- it: routes each grouped gateway prefix to the gateway Service through a use-annotation action on alb
set:
ingress.enabled: true
asserts:
- equal:
path: metadata.annotations["alb.ingress.kubernetes.io/actions.litellm-gateway-0"]
value: '{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"RELEASE-NAME-litellm-gateway","servicePort":"http"}]}}'
- matchRegex:
path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"]
pattern: '^\[\{"field":"path-pattern","pathPatternConfig":\{"values":\["(/[^"*]+)","/[^"*]+/\*"\]\}\}\]$'
- notExists:
path: spec.rules[0].http.paths[?(@.pathType == "Prefix" && @.backend.service.name == "RELEASE-NAME-litellm-gateway")]
- it: keeps an operator's conditions for the gateway Service on every grouped alb rule
set:
ingress.enabled: true
ingress.annotations:
alb.ingress.kubernetes.io/conditions.RELEASE-NAME-litellm-gateway: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]'
asserts:
- matchRegex:
path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"]
pattern: '^\[\{"field":"source-ip","sourceIpConfig":\{"values":\["10\.0\.0\.0/8"\]\}\},\{"field":"path-pattern",'
- equal:
path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-50"]
value: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]'
- it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
set:
ingress.enabled: true
@ -107,6 +132,26 @@ tests:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /eu_jev/v1/systemone
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /openrouter
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- equal:
path: spec.rules[0].http.paths[-1]
value:

View file

@ -7,6 +7,7 @@ tests:
- it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
set:
ingress.enabled: true
ingress.controller: nginx
asserts:
- equal:
path: spec.rules[0].http.paths[-1]
@ -33,12 +34,12 @@ tests:
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
- path: /hosted_vllm
asserts:
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /watsonx
path: /hosted_vllm
pathType: Prefix
backend:
service:
@ -59,8 +60,9 @@ tests:
- it: keeps every built-in path when extra paths are supplied
set:
ingress.enabled: true
ingress.controller: nginx
ingress.extraPaths:
- path: /watsonx
- path: /hosted_vllm
asserts:
- contains:
path: spec.rules[0].http.paths
@ -117,7 +119,7 @@ tests:
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
- path: /hosted_vllm
service: gateway
- path: /my-passthrough
pathType: Exact
@ -129,7 +131,7 @@ tests:
- equal:
path: spec.rules[0].http.paths[-4]
value:
path: /watsonx
path: /hosted_vllm
pathType: Prefix
backend:
service:
@ -164,7 +166,7 @@ tests:
backend.service.port: 8001
ui.service.port: 8080
ingress.extraPaths:
- path: /watsonx
- path: /hosted_vllm
- path: /my-passthrough
service: backend
- path: /brand.txt
@ -184,21 +186,21 @@ tests:
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
- path: /hosted_vllm
service: proxy
asserts:
- failedTemplate:
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown service "proxy", expected one of backend, gateway, ui'
- it: rejects an entry whose pathType is not a kubernetes pathType
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
- path: /hosted_vllm
pathType: prefix
asserts:
- failedTemplate:
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
- it: rejects an entry with no path
set:
@ -322,7 +324,7 @@ tests:
set:
ingress.enabled: true
ingress.extraPaths:
- /watsonx
- /hosted_vllm
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"
@ -372,7 +374,7 @@ tests:
- path: /moyai
serviceName: moyai-ingress
servicePort: http
- path: /watsonx
- path: /hosted_vllm
service: gateway
asserts:
- equal:

View file

@ -41,6 +41,7 @@ tests:
- it: componentized mode still splits traffic across ui, gateway and backend Services
set:
ingress.enabled: true
ingress.controller: nginx
asserts:
- equal:
path: spec.rules[0].http.paths[0].backend.service.name

View file

@ -102,7 +102,7 @@ ingress:
# The target component only answers paths its own route allowlist keeps, so
# a path here still has to be one that component serves.
extraPaths: []
# - path: /watsonx
# - path: /hosted_vllm
# pathType: Prefix
# service: gateway

View file

@ -1,326 +1,334 @@
"""
PagerDuty Alerting Integration
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
Note: This is a Free feature on the regular litellm docker image.
However, this is under the enterprise license
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.exceptions import MissingMessageRoleError
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.pagerduty import (
AlertingConfig,
PagerDutyInternalEvent,
PagerDutyPayload,
PagerDutyRequestBody,
)
from litellm.types.utils import (
CallTypesLiteral,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
class PagerDutyAlerting(SlackAlerting):
"""
Tracks failed requests and hanging requests separately.
If threshold is crossed for either type, triggers a PagerDuty alert.
"""
def __init__(
self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
):
super().__init__()
_api_key = os.getenv("PAGERDUTY_API_KEY")
if not _api_key:
raise ValueError("PAGERDUTY_API_KEY is not set")
self.api_key: str = _api_key
alerting_args = alerting_args or {}
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
failure_threshold=alerting_args.get(
"failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD
),
failure_threshold_window_seconds=alerting_args.get(
"failure_threshold_window_seconds",
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
),
hanging_threshold_seconds=alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
),
hanging_threshold_window_seconds=alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
),
)
# Separate storage for failures vs. hangs
self._failure_events: List[PagerDutyInternalEvent] = []
self._hanging_events: List[PagerDutyInternalEvent] = []
# ------------------ MAIN LOGIC ------------------ #
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
if isinstance(kwargs.get("exception"), MissingMessageRoleError):
return
now = datetime.now(timezone.utc)
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
)
if not standard_logging_payload:
raise ValueError(
"standard_logging_object is required for PagerDutyAlerting"
)
# Extract error details
error_info: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information") or {}
)
_meta = standard_logging_payload.get("metadata") or {}
self._failure_events.append(
PagerDutyInternalEvent(
failure_event_type="failed_response",
timestamp=now,
error_class=error_info.get("error_class"),
error_code=error_info.get("error_code"),
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_project_id=_meta.get("user_api_key_project_id"),
user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"failure_threshold_window_seconds", 60
)
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
# If threshold is crossed, send PD alert for failures
await self._send_alert_if_thresholds_crossed(
events=self._failure_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High LLM API Failure Rate",
)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
"""
Example of detecting hanging requests by waiting a given threshold.
If the request didn't finish by then, we treat it as 'hanging'.
"""
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
asyncio.create_task(
self.hanging_response_handler(
request_data=data, user_api_key_dict=user_api_key_dict
)
)
return None
async def hanging_response_handler(
self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth
):
"""
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
If not, we classify it as a hanging request.
"""
verbose_logger.debug(
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
)
await asyncio.sleep(
self.pagerduty_alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
)
if await self._request_is_completed(request_data=request_data):
return # It's not hanging if completed
# Otherwise, record it as hanging
self._hanging_events.append(
PagerDutyInternalEvent(
failure_event_type="hanging_response",
timestamp=datetime.now(timezone.utc),
error_class="HangingRequest",
error_code="HangingRequest",
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_user_spend=user_api_key_dict.user_spend,
user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
user_api_key_team_spend=user_api_key_dict.team_spend,
user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_org_alias=user_api_key_dict.organization_alias,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_project_alias=user_api_key_dict.project_alias,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
)
threshold: int = self.pagerduty_alerting_args.get(
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
# If threshold is crossed, send PD alert for hangs
await self._send_alert_if_thresholds_crossed(
events=self._hanging_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High Number of Hanging LLM Requests",
)
# ------------------ HELPERS ------------------ #
async def _send_alert_if_thresholds_crossed(
self,
events: List[PagerDutyInternalEvent],
window_seconds: int,
threshold: int,
alert_prefix: str,
):
"""
1. Prune old events
2. If threshold is reached, build alert, send to PagerDuty
3. Clear those events
"""
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
# Update the reference list
events.clear()
events.extend(pruned)
# Check threshold
verbose_logger.debug(
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
)
if len(events) >= threshold:
# Build short summary of last N events
error_summaries = self._build_error_summaries(events, max_errors=5)
alert_message = (
f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
)
custom_details = {"recent_errors": error_summaries}
await self.send_alert_to_pagerduty(
alert_message=alert_message,
custom_details=custom_details,
)
# Clear them after sending an alert, so we don't spam
events.clear()
def _build_error_summaries(
self, events: List[PagerDutyInternalEvent], max_errors: int = 5
) -> List[PagerDutyInternalEvent]:
"""
Build short text summaries for the last `max_errors`.
Example: "ValueError (code: 500, provider: openai)"
"""
recent = events[-max_errors:]
summaries = []
for fe in recent:
# If any of these is None, show "N/A" to avoid messing up the summary string
fe.pop("timestamp")
summaries.append(fe)
return summaries
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
"""
Send [critical] Alert to PagerDuty
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
"""
try:
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
async_client: AsyncHTTPHandler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
payload: PagerDutyRequestBody = PagerDutyRequestBody(
payload=PagerDutyPayload(
summary=alert_message,
severity="critical",
source="LiteLLM Alert",
component="LiteLLM",
custom_details=custom_details,
),
routing_key=self.api_key,
event_action="trigger",
)
return await async_client.post(
url="https://events.pagerduty.com/v2/enqueue",
json=dict(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")
"""
PagerDuty Alerting Integration
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
Note: This is a Free feature on the regular litellm docker image.
However, this is under the enterprise license
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import Final, Literal
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.exceptions import MissingMessageRoleError
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.pagerduty import (
AlertingConfig,
PagerDutyInternalEvent,
PagerDutyPayload,
PagerDutyRequestBody,
)
from litellm.types.utils import (
CallTypesLiteral,
StandardLoggingMetadata,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
def _metadata_of(payload: StandardLoggingPayload) -> StandardLoggingMetadata:
return payload["metadata"]
_CustomerIdKey = Literal[
"user_api_key_customer_member_id", "user_api_key_customer_team_id", "user_api_key_customer_org_id"
]
def _customer_id(metadata: StandardLoggingMetadata, key: _CustomerIdKey) -> str | None:
return metadata[key] if key in metadata else None
class PagerDutyAlerting(SlackAlerting):
"""
Tracks failed requests and hanging requests separately.
If threshold is crossed for either type, triggers a PagerDuty alert.
"""
def __init__(self, alerting_args: AlertingConfig | dict | None = None, **kwargs):
super().__init__()
_api_key = os.getenv("PAGERDUTY_API_KEY")
if not _api_key:
raise ValueError("PAGERDUTY_API_KEY is not set")
self.api_key: str = _api_key
alerting_args = alerting_args or {}
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
failure_threshold=alerting_args.get("failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD),
failure_threshold_window_seconds=alerting_args.get(
"failure_threshold_window_seconds",
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
),
hanging_threshold_seconds=alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
),
hanging_threshold_window_seconds=alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
),
)
# Separate storage for failures vs. hangs
self._failure_events: list[PagerDutyInternalEvent] = []
self._hanging_events: list[PagerDutyInternalEvent] = []
# ------------------ MAIN LOGIC ------------------ #
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
if isinstance(kwargs.get("exception"), MissingMessageRoleError):
return
now = datetime.now(timezone.utc)
standard_logging_payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object")
if not standard_logging_payload:
raise ValueError("standard_logging_object is required for PagerDutyAlerting")
# Extract error details
error_info: StandardLoggingPayloadErrorInformation | None = (
standard_logging_payload.get("error_information") or {}
)
_meta: Final = _metadata_of(standard_logging_payload)
self._failure_events.append(
PagerDutyInternalEvent(
failure_event_type="failed_response",
timestamp=now,
error_class=error_info.get("error_class"),
error_code=error_info.get("error_code"),
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_project_id=_meta.get("user_api_key_project_id"),
user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_customer_member_id=_customer_id(_meta, "user_api_key_customer_member_id"),
user_api_key_customer_team_id=_customer_id(_meta, "user_api_key_customer_team_id"),
user_api_key_customer_org_id=_customer_id(_meta, "user_api_key_customer_org_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get("failure_threshold_window_seconds", 60)
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
# If threshold is crossed, send PD alert for failures
await self._send_alert_if_thresholds_crossed(
events=self._failure_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High LLM API Failure Rate",
)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Exception | str | dict | None:
"""
Example of detecting hanging requests by waiting a given threshold.
If the request didn't finish by then, we treat it as 'hanging'.
"""
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
asyncio.create_task(self.hanging_response_handler(request_data=data, user_api_key_dict=user_api_key_dict))
return None
async def hanging_response_handler(self, request_data: dict | None, user_api_key_dict: UserAPIKeyAuth):
"""
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
If not, we classify it as a hanging request.
"""
verbose_logger.debug(
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
)
await asyncio.sleep(
self.pagerduty_alerting_args.get("hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)
)
if await self._request_is_completed(request_data=request_data):
return # It's not hanging if completed
# Otherwise, record it as hanging
self._hanging_events.append(
PagerDutyInternalEvent(
failure_event_type="hanging_response",
timestamp=datetime.now(timezone.utc),
error_class="HangingRequest",
error_code="HangingRequest",
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None
),
user_api_key_user_spend=user_api_key_dict.user_spend,
user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
user_api_key_team_spend=user_api_key_dict.team_spend,
user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_org_alias=user_api_key_dict.organization_alias,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_project_alias=user_api_key_dict.project_alias,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_customer_member_id=(
user_api_key_dict.customer_hierarchy.customer_member_id
if user_api_key_dict.customer_hierarchy
else None
),
user_api_key_customer_team_id=(
user_api_key_dict.customer_hierarchy.customer_team_id
if user_api_key_dict.customer_hierarchy
else None
),
user_api_key_customer_org_id=(
user_api_key_dict.customer_hierarchy.customer_org_id
if user_api_key_dict.customer_hierarchy
else None
),
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
)
threshold: int = self.pagerduty_alerting_args.get(
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
# If threshold is crossed, send PD alert for hangs
await self._send_alert_if_thresholds_crossed(
events=self._hanging_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High Number of Hanging LLM Requests",
)
# ------------------ HELPERS ------------------ #
async def _send_alert_if_thresholds_crossed(
self,
events: list[PagerDutyInternalEvent],
window_seconds: int,
threshold: int,
alert_prefix: str,
):
"""
1. Prune old events
2. If threshold is reached, build alert, send to PagerDuty
3. Clear those events
"""
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
# Update the reference list
events.clear()
events.extend(pruned)
# Check threshold
verbose_logger.debug(
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
)
if len(events) >= threshold:
# Build short summary of last N events
error_summaries = self._build_error_summaries(events, max_errors=5)
alert_message = f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
custom_details = {"recent_errors": error_summaries}
await self.send_alert_to_pagerduty(
alert_message=alert_message,
custom_details=custom_details,
)
# Clear them after sending an alert, so we don't spam
events.clear()
def _build_error_summaries(
self, events: list[PagerDutyInternalEvent], max_errors: int = 5
) -> list[PagerDutyInternalEvent]:
"""
Build short text summaries for the last `max_errors`.
Example: "ValueError (code: 500, provider: openai)"
"""
recent = events[-max_errors:]
summaries = []
for fe in recent:
# If any of these is None, show "N/A" to avoid messing up the summary string
fe.pop("timestamp")
summaries.append(fe)
return summaries
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
"""
Send [critical] Alert to PagerDuty
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
"""
try:
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
async_client: AsyncHTTPHandler = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
payload: PagerDutyRequestBody = PagerDutyRequestBody(
payload=PagerDutyPayload(
summary=alert_message,
severity="critical",
source="LiteLLM Alert",
component="LiteLLM",
custom_details=custom_details,
),
routing_key=self.api_key,
event_action="trigger",
)
return await async_client.post(
url="https://events.pagerduty.com/v2/enqueue",
json=dict(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")

View file

@ -82,6 +82,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
TeamProviderDeployment,
_is_base64_encoded_unified_file_id,
apply_unified_file_ids,
canonical_provider_file_id,
decode_model_from_file_id,
ensure_batch_response_managed_file_ids,
get_batch_id_from_unified_batch_id,
@ -91,6 +92,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
is_litellm_executed_batch,
map_raw_file_ids_to_unified,
normalize_mime_type_for_provider,
provider_file_id_spellings,
resolve_managed_output_file_model_name,
validate_file_list_limit,
validate_file_list_purpose,
@ -597,14 +599,27 @@ async def _scoped_file_rows(
return _SCOPED_FILE_ROWS.validate_python(rows)
def _iter_provider_file_id_pairs(
rows: Sequence[_ManagedFileRow],
requested_provider_file_ids: frozenset[str],
) -> Iterator[tuple[str, str]]:
def _iter_canonical_provider_file_ids(rows: Sequence[_ManagedFileRow]) -> Iterator[tuple[str, str]]:
for row in rows:
for provider_file_id in row.flat_model_file_ids:
if provider_file_id in requested_provider_file_ids:
yield provider_file_id, row.unified_file_id
yield canonical_provider_file_id(provider_file_id), row.unified_file_id
def _iter_requested_provider_file_id_spellings(provider_file_ids: Sequence[str]) -> Iterator[str]:
for provider_file_id in provider_file_ids:
yield from provider_file_id_spellings(provider_file_id)
def _iter_provider_file_id_pairs(
rows: Sequence[_ManagedFileRow],
requested_provider_file_ids: Sequence[str],
) -> Iterator[tuple[str, str]]:
unified_by_canonical_id: Final = dict(_iter_canonical_provider_file_ids(rows))
for requested_provider_file_id in requested_provider_file_ids:
if (
unified_file_id := unified_by_canonical_id.get(canonical_provider_file_id(requested_provider_file_id))
) is not None:
yield requested_provider_file_id, unified_file_id
def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedObjectTableActions:
@ -767,12 +782,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> None:
verbose_logger.info(f"Storing LiteLLM Managed File object with id={file_id} in cache")
storage_metadata: Final = _storage_metadata_of(file_object)
flat_model_file_ids: Final = [
canonical_provider_file_id(provider_file_id) for provider_file_id in model_mappings.values()
]
if file_object is not None:
litellm_managed_file_object = LiteLLM_ManagedFileTable(
unified_file_id=file_id,
file_object=file_object,
model_mappings=model_mappings,
flat_model_file_ids=list(model_mappings.values()),
flat_model_file_ids=flat_model_file_ids,
created_by=resolve_resource_owner_id(user_api_key_dict),
team_id=user_api_key_dict.team_id,
updated_by=user_api_key_dict.user_id,
@ -790,14 +808,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
db_data = {
"unified_file_id": file_id,
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"flat_model_file_ids": flat_model_file_ids,
"created_by": resolve_resource_owner_id(user_api_key_dict),
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
update_data = {
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"flat_model_file_ids": flat_model_file_ids,
"updated_by": user_api_key_dict.user_id,
}
@ -1094,8 +1112,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""
if self.prisma_client is None:
return
spellings: Final = list(provider_file_id_spellings(file_id))
managed_file = await _managed_file_table(self.prisma_client).find_first(
where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
where={
"OR": [
{"unified_file_id": {"in": spellings}}, # bounded-ok: at most four spellings of one file id
{"flat_model_file_ids": {"hasSome": spellings}},
]
}
)
if managed_file is None:
return
@ -1139,7 +1163,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
for file_id in provider_file_ids:
model_name = decode_model_from_file_id(file_id)
raw_file_id = get_original_file_id(file_id)
raw_file_id = canonical_provider_file_id(get_original_file_id(file_id))
await self.store_batch_output_file(
unified_file_id=file_id,
provider_file_id=raw_file_id,
@ -1629,7 +1653,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return MappingProxyType({})
provider_file_ids_list: Final = [ # mutable-ok: Prisma hasSome requires a list
provider_file_id for provider_file_id in unique_provider_file_ids
*dict.fromkeys(_iter_requested_provider_file_id_spellings(unique_provider_file_ids))
]
rows: Final = await _managed_file_table(self.prisma_client).find_many(
where={ # mutable-ok: Prisma requires a plain dictionary for where
@ -1641,10 +1665,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return MappingProxyType(
dict(
_iter_provider_file_id_pairs(
rows,
frozenset(unique_provider_file_ids),
)
_iter_provider_file_id_pairs(rows, unique_provider_file_ids)
)
)

View file

@ -43,6 +43,7 @@ from litellm.proxy.management_helpers.model_quota_policy import (
)
from litellm.proxy.management_helpers.utils import (
management_endpoint_wrapper,
validate_scalar_budget_attachment,
)
from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
from litellm.repositories.base_repository import record_to_dict
@ -637,6 +638,7 @@ async def new_project(
)
# Create budget if not provided
await validate_scalar_budget_attachment(prisma_client, data.budget_id)
if data.budget_id is None:
data.budget_id = await _create_budget_for_project(
data=data,
@ -869,6 +871,7 @@ async def update_project(
)
# Prepare update data
await validate_scalar_budget_attachment(prisma_client, data.budget_id)
update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name

View file

@ -35,6 +35,7 @@ class SendKeyRotatedEmailEvent(WebhookEvent):
class EmailEvent(str, enum.Enum):
model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT
performance_alerts = "Performance Alerts"
virtual_key_created = "Virtual Key Created"
new_user_invitation = "New User Invitation"
virtual_key_rotated = "Virtual Key Rotated"
@ -60,7 +61,8 @@ class DefaultEmailSettings(LiteLLMBaseModel):
settings: dict[EmailEvent, bool] = Field(
default_factory=lambda: {
EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders.
EmailEvent.model_deprecations: False,
EmailEvent.performance_alerts: False,
EmailEvent.virtual_key_created: True, # On by default
EmailEvent.new_user_invitation: True, # On by default
EmailEvent.virtual_key_rotated: True, # On by default

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.83"
version = "0.1.85"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.83"
version = "0.1.85"
version_files = [
"pyproject.toml:^version",
"../../pyproject.toml:litellm-enterprise==",

View file

@ -465,6 +465,7 @@ def image_generation(
litellm.LlmProviders.AIML,
litellm.LlmProviders.GEMINI,
litellm.LlmProviders.FAL_AI,
litellm.LlmProviders.REPLICATE,
litellm.LlmProviders.STABILITY,
litellm.LlmProviders.RUNWAYML,
litellm.LlmProviders.VERTEX_AI,
@ -496,6 +497,7 @@ def image_generation(
logging_obj=litellm_logging_obj,
timeout=timeout,
client=client,
_is_async=aimg_generation is True and custom_llm_provider == litellm.LlmProviders.REPLICATE,
)
elif custom_llm_provider == "black_forest_labs":
# Route to BFL-specific handler (polling required)

View file

@ -2,10 +2,11 @@
# Class for sending Slack Alerts #
import asyncio
import datetime
import json
import os
import random
import time
from collections.abc import Callable
from collections.abc import Awaitable, Callable
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final, Literal
@ -38,6 +39,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.repositories.config_repository import ConfigRepository
from litellm.repositories.table_repositories import InvitationLinkRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
@ -87,6 +89,12 @@ def _proxy_llm_router() -> Router | None:
_DAILY_REPORT_TARGET: Final = "daily_report_schedule"
async def _model_deprecations_enabled() -> bool:
from litellm.proxy.proxy_server import prisma_client
return prisma_client is None or await ConfigRepository(prisma_client, use_writer=True).model_deprecations_enabled()
class SlackAlerting(CustomBatchLogger):
"""
Class for sending Slack Alerts
@ -97,12 +105,13 @@ class SlackAlerting(CustomBatchLogger):
self,
internal_usage_cache: DualCache | None = None,
alerting_threshold: float | None = None, # threshold for slow / hanging llm responses (in seconds)
alerting: list | None = [],
alerting: list[str] | None = [],
alert_types: list[AlertType] = DEFAULT_ALERT_TYPES,
alert_to_webhook_url: dict[AlertType, list[str] | str]
| None = None, # if user wants to separate alerts to diff channels
alerting_args={},
default_webhook_url: str | None = None,
model_deprecations_enabled: Callable[[], Awaitable[bool]] = _model_deprecations_enabled,
alert_type_config: dict[str, dict] | None = None,
async_http_handler: AsyncHTTPHandler | None = None,
**kwargs,
@ -120,6 +129,7 @@ class SlackAlerting(CustomBatchLogger):
self.is_running = False
self.alerting_args = SlackAlertingArgs(**alerting_args)
self.default_webhook_url = default_webhook_url
self.model_deprecations_enabled: Final = model_deprecations_enabled
self.flush_lock = asyncio.Lock()
self.periodic_started = False
self._periodic_flush_task: asyncio.Task[None] | None = None
@ -142,7 +152,7 @@ class SlackAlerting(CustomBatchLogger):
def update_values(
self,
alerting: list | None = None,
alerting: list[str] | None = None,
alerting_threshold: float | None = None,
alert_types: list[AlertType] | None = None,
alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None,
@ -1114,7 +1124,7 @@ Model Info:
The daily lock is claimed only once there is something to say, so an empty pass never blocks a
later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking
"""
if not self._deprecation_alerts_enabled():
if not self._deprecation_alerts_enabled() or not await self.model_deprecations_enabled():
return False
from litellm.proxy.common_utils.model_deprecation import (
@ -1194,6 +1204,26 @@ Model Info:
continue
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
def performance_alert_webhooks(self) -> tuple[str, ...]:
if not self.alerting or "slack" not in self.alerting or AlertType.performance_alerts not in self.alert_types:
return ()
configured: Final = (self.alert_to_webhook_url or {}).get(AlertType.performance_alerts)
destination: Final = (
configured
or self.default_webhook_url
or os.getenv("SLACK_WEBHOOK_URL")
or os.getenv("ALERTING_WEBHOOK_URL")
)
if isinstance(destination, list):
return tuple(dict.fromkeys(destination))
return (destination,) if destination else ()
async def send_performance_alert_summary(self, webhook: str, message: str) -> bool:
response: Final = await self.async_http_handler.post(
url=webhook, headers={"Content-Type": "application/json"}, data=json.dumps({"text": message})
)
return response.status_code == 200
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
"""
Sends structured alert to webhook, if set.

View file

@ -3,6 +3,8 @@
import re
import traceback
from collections.abc import AsyncGenerator, Mapping, Sequence
from functools import cache
from itertools import takewhile
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
@ -36,6 +38,7 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.types.integrations.customer_budget import CustomerBudgetEvent
from litellm.types.mcp import (
MCPPostCallResponseObject,
MCPPreCallRequestObject,
@ -66,6 +69,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
# Class variables or attributes
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
wants_failed_prompt_token_estimate: ClassVar[bool] = False
enforces_request_content: bool = False
"""
@ -196,6 +200,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_log_audit_log_event(self, audit_log: "StandardAuditLogPayload"):
"""Called when an audit log is created. Override in subclasses to handle."""
async def async_on_customer_budget_event(self, event: "CustomerBudgetEvent") -> None:
pass
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
"""Called once per part of a finished batch's lines, when this logger is listed in batch_line_item_callbacks."""
@ -1115,3 +1122,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
msg[key] = self._redact_base64(value=val, max_depth=max_depth)
filtered_messages.append(msg)
return filtered_messages
@cache
def subclass_overrides_hook(cls: type, hook_name: str, base: type = CustomLogger) -> bool:
below_base: Final = takewhile(lambda klass: klass is not base, cls.__mro__)
return any(hook_name in vars(klass) for klass in below_base)

View file

@ -0,0 +1,74 @@
from litellm.integrations.custom_logger_v2.call import CallContext, CallFacts, CallIds, CallKind, CallTarget, CallTiming
from litellm.integrations.custom_logger_v2.dispatcher import Dispatcher, HookResult, RequestScope
from litellm.integrations.custom_logger_v2.enforcement import ContractViolation, UndeclaredReadError
from litellm.integrations.custom_logger_v2.enums import Activation, Field, FieldTier, Hook, OptOutBy
from litellm.integrations.custom_logger_v2.hooks import (
HOOK_SPECS,
HookSpec,
InvalidDeclarationError,
Subscription,
)
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest, build_manifest
from litellm.integrations.custom_logger_v2.offpath import InProcessSink, OffPathSink
from litellm.integrations.custom_logger_v2.outcomes import (
Abort,
InvalidPatchError,
JsonValue,
MergeField,
Outcome,
Patch,
Reroute,
Respond,
SetField,
)
from litellm.integrations.custom_logger_v2.patches import apply_patch, apply_patches
from litellm.integrations.custom_logger_v2.planner import HookPlan, plan_hook
from litellm.integrations.custom_logger_v2.registry import CallbackRegistry
from litellm.integrations.custom_logger_v2.selection import RequestSelection, Selection, is_selected
__all__ = [
"HOOK_SPECS",
"Abort",
"Activation",
"CallContext",
"CallFacts",
"CallIds",
"CallKind",
"CallTarget",
"CallTiming",
"CallbackManifest",
"CallbackRegistry",
"ContractViolation",
"CustomLoggerV2",
"Dispatcher",
"Field",
"FieldTier",
"Hook",
"HookPlan",
"HookResult",
"HookSpec",
"InProcessSink",
"InvalidDeclarationError",
"InvalidPatchError",
"JsonValue",
"MergeField",
"OffPathSink",
"OptOutBy",
"Outcome",
"Patch",
"RequestScope",
"RequestSelection",
"Reroute",
"Respond",
"Selection",
"SetField",
"Subscription",
"UndeclaredReadError",
"apply_patch",
"apply_patches",
"build_manifest",
"is_selected",
"on",
"plan_hook",
]

View file

@ -0,0 +1,55 @@
from dataclasses import dataclass
from typing import Final, Generic, TypeVar
from litellm.integrations.custom_logger_v2.enums import Hook
StateT: Final = TypeVar("StateT")
@dataclass(frozen=True, slots=True)
class CallIds:
call_id: str
trace_id: str | None = None
session_id: str | None = None
parent_call_id: str | None = None
@dataclass(frozen=True, slots=True)
class CallKind:
call_type: str
route: str | None = None
stream: bool = False
@dataclass(frozen=True, slots=True)
class CallTarget:
model: str
model_group: str | None = None
provider: str | None = None
api_base: str | None = None
deployment_id: str | None = None
attempt: int | None = None
@dataclass(frozen=True, slots=True)
class CallTiming:
start: float
provider_start: float | None = None
first_token: float | None = None
end: float | None = None
@dataclass(frozen=True, slots=True)
class CallFacts:
ids: CallIds
kind: CallKind
target: CallTarget
timing: CallTiming
cache_hit: bool | None = None
@dataclass(frozen=True, slots=True)
class CallContext(Generic[StateT]):
hook: Hook
call: CallFacts | None
state: StateT

View file

@ -0,0 +1,400 @@
import asyncio
import inspect
import logging
from collections import Counter
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from operator import itemgetter
from types import TracebackType
from typing import Final, Protocol, TypeAlias, TypeVar
from typing_extensions import Self
from litellm.integrations.custom_logger_v2.call import CallContext, CallFacts
from litellm.integrations.custom_logger_v2.enforcement import ContractViolation, Enforced, Policy, Unchecked
from litellm.integrations.custom_logger_v2.enums import Field, Hook
from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS
from litellm.integrations.custom_logger_v2.manifest import LegacyBinding
from litellm.integrations.custom_logger_v2.offpath import InProcessSink, OffPathSink
from litellm.integrations.custom_logger_v2.outcomes import Abort, Outcome, Patch, Reroute, Respond
from litellm.integrations.custom_logger_v2.patches import apply_patch, apply_patches
from litellm.integrations.custom_logger_v2.planner import EMPTY_PLAN, Entry, HookPlan
from litellm.integrations.custom_logger_v2.registry import CallbackRegistry, RegistrySnapshot
from litellm.integrations.custom_logger_v2.selection import RequestSelection, is_selected
LegacyT: Final = TypeVar("LegacyT")
Between: TypeAlias = Callable[[Mapping[Field, object]], Awaitable[Mapping[Field, object]]]
SyncBinding: TypeAlias = Callable[[object, object], object]
SWALLOWING_HOOKS: Final = frozenset({Hook.response_headers, Hook.post_call_failure})
_LOGGER: Final = logging.getLogger("LiteLLM")
class _Handler(Protocol):
def __call__(self, view: object, ctx: CallContext[object], /) -> object: ...
@dataclass(frozen=True, slots=True)
class HookResult:
fields: Mapping[Field, object]
patches: tuple[Patch, ...] = ()
outcome: Outcome | None = None
@dataclass(frozen=True, slots=True)
class _Chain:
fields: Mapping[Field, object]
patches: tuple[tuple[int, Patch], ...] = ()
reroute: Reroute | None = None
terminal: Abort | Respond | None = None
@property
def ordered_patches(self) -> tuple[Patch, ...]:
return tuple(patch for _, patch in sorted(self.patches, key=itemgetter(0)))
def _handler(logger: object, name: str) -> _Handler:
found: Final[object] = getattr(logger, name, None)
if not callable(found):
raise ContractViolation(f"{name} is not callable")
return found
async def _awaited(returned: object) -> object:
return await returned if inspect.isawaitable(returned) else returned
def _decides(hook: Hook, result: object) -> bool:
match result:
case Abort() | Respond():
return True
case ContractViolation():
return True
case Exception():
return hook not in SWALLOWING_HOOKS
case BaseException():
return True
case _:
return False
def _fatal(error: BaseException) -> bool:
return isinstance(error, ContractViolation) or not isinstance(error, Exception)
def _split(
stages: tuple[tuple[Entry, ...], ...],
) -> tuple[tuple[tuple[Entry, ...], ...], tuple[tuple[Entry, ...], ...]]:
if stages and stages[-1][0].subscription.parallel:
return stages[:-1], stages[-1:]
return stages, ()
def _legacy_start(
entry: Entry, binding: Callable[[object, object], LegacyT] | None, fields: Mapping[Field, object]
) -> LegacyT | None:
call: Final = entry.manifest.legacy_call
if call is None or binding is None:
raise ContractViolation(
f"{entry.manifest.selection.name} is a legacy entry, so the hook point must pass a binding"
)
return call(binding, fields)
async def _run_legacy(entry: Entry, binding: LegacyBinding | None, fields: Mapping[Field, object]) -> object:
started: Final = _legacy_start(entry, binding, fields)
return None if started is None else await started
async def _gather_started(started: tuple[Awaitable[object] | None, ...], fail_fast: bool) -> tuple[object, ...]:
pending: Final = tuple(run for run in started if run is not None)
if not pending:
return tuple(None for _ in started)
done: Final = iter(
await asyncio.gather(*pending) if fail_fast else await asyncio.gather(*pending, return_exceptions=True)
)
return tuple(None if run is None else next(done) for run in started)
def _fail_fast(hook: Hook, stage: tuple[Entry, ...]) -> bool:
return HOOK_SPECS[hook].fail_fast_parallel and all(entry.subscription.parallel for entry in stage)
async def run_legacy_plan(
hook: Hook,
plan: HookPlan,
fields: Mapping[Field, object],
binding: LegacyBinding,
between: Between | None = None,
) -> Mapping[Field, object]:
serial, parallel = _split(plan.stages)
current: Mapping[Field, object] = fields # rebind-ok: each legacy entry may replace the kwargs
for stage in serial:
for entry in stage:
result = await _run_legacy(entry, binding, current)
if isinstance(result, Patch):
current = apply_patch(current, result)
if between is not None:
current = await between(current)
for stage in parallel:
started = tuple(_legacy_start(entry, binding, current) for entry in stage)
results = await _gather_started(started, _fail_fast(hook, stage))
failure = next((result for result in results if isinstance(result, BaseException)), None)
if failure is not None:
raise failure
for entry in plan.inline:
_ = await _run_legacy(entry, binding, current)
return current
def run_legacy_plan_sync(
plan: HookPlan, fields: Mapping[Field, object], binding: SyncBinding
) -> Mapping[Field, object]:
staged: Final = tuple(chain.from_iterable(plan.stages))
if any(entry.subscription.parallel for entry in staged):
raise ContractViolation("a synchronous hook cannot run parallel legacy entries")
current: Mapping[Field, object] = fields # rebind-ok: each legacy entry may replace the kwargs
for entry in (*staged, *plan.inline):
result = _legacy_start(entry, binding, current)
if isinstance(result, Patch):
current = apply_patch(current, result)
return current
class Dispatcher:
def __init__(
self,
registry: CallbackRegistry,
sink: OffPathSink | None = None,
enforce: bool = False,
) -> None:
self.registry: Final = registry
self.sink: Final[OffPathSink] = sink or InProcessSink()
self.policy: Final[Policy] = Enforced() if enforce else Unchecked()
self.failures: Final[Counter[tuple[str, Hook]]] = Counter() # mutable-ok: process-wide failure counts
def open_scope(self, selection: RequestSelection) -> "RequestScope":
return RequestScope(self, self.registry.snapshot, selection)
def record_failure(self, entry: Entry, hook: Hook, error: BaseException) -> None:
self.failures[(entry.manifest.selection.name, hook)] += 1
_LOGGER.warning("custom logger v2 %s failed on %s: %r", entry.manifest.selection.name, hook.value, error)
class RequestScope:
def __init__(self, dispatcher: Dispatcher, snapshot: RegistrySnapshot, selection: RequestSelection) -> None:
self._dispatcher: Final = dispatcher
self._snapshot: Final = snapshot
self._selection: Final = selection
self._states: dict[int, object] = {} # mutable-ok: request_state instances are created lazily per logger
self.reroute: Reroute | None = None
def __enter__(self) -> Self:
return self
def __exit__(
self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None
) -> None:
self.close()
def close(self) -> None:
self._states.clear()
def selects(self, entry: Entry) -> bool:
return is_selected(entry.manifest.selection, entry.activation, self._selection)
def plan(self, hook: Hook) -> HookPlan:
entries: Final = self._snapshot.entries[hook]
if not entries:
return EMPTY_PLAN
selected: Final = tuple(index for index, entry in enumerate(entries) if self.selects(entry))
return self._snapshot.plan(hook, selected) if selected else EMPTY_PLAN
def reads(self, hook: Hook) -> frozenset[Field]:
return self.plan(hook).reads
async def run_hook(
self,
hook: Hook,
fields: Mapping[Field, object],
call: CallFacts | None,
) -> HookResult:
plan: Final = self.plan(hook)
if plan is EMPTY_PLAN:
return HookResult(fields=fields)
chain: Final = await self._run_stages(hook, plan.stages, _Chain(fields=fields), call, None)
patches: Final = chain.ordered_patches
if chain.terminal is not None:
return HookResult(fields=chain.fields, patches=patches, outcome=chain.terminal)
await self._run_awaited(hook, plan.awaited, chain.fields, call)
if plan.off_path:
self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, chain.fields, call))
if chain.reroute is not None:
self.reroute = chain.reroute
return HookResult(fields=chain.fields, patches=patches, outcome=chain.reroute)
async def run_plan(
self,
hook: Hook,
plan: HookPlan,
fields: Mapping[Field, object],
call: CallFacts | None,
binding: LegacyBinding,
between: Between | None = None,
) -> HookResult:
serial, parallel = _split(plan.stages)
chain: Final = await self._run_stages(hook, serial, _Chain(fields=fields), call, binding)
if chain.terminal is not None:
return HookResult(fields=chain.fields, patches=chain.ordered_patches, outcome=chain.terminal)
joined: Final = (
chain
if between is None
else _Chain(fields=await between(chain.fields), patches=chain.patches, reroute=chain.reroute)
)
done: Final = await self._run_stages(hook, parallel, joined, call, binding)
if done.terminal is not None:
return HookResult(fields=done.fields, patches=done.ordered_patches, outcome=done.terminal)
for entry in plan.inline:
_ = await _run_legacy(entry, binding, done.fields)
await self._run_awaited(hook, plan.awaited, done.fields, call)
if plan.off_path:
self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, done.fields, call))
if done.reroute is not None:
self.reroute = done.reroute
return HookResult(fields=done.fields, patches=done.ordered_patches, outcome=done.reroute)
def run_plan_sync(
self,
hook: Hook,
plan: HookPlan,
fields: Mapping[Field, object],
call: CallFacts | None,
binding: SyncBinding,
) -> Mapping[Field, object]:
v2_entries: Final = tuple(entry for entry in chain.from_iterable(plan.stages) if not entry.subscription.legacy)
if v2_entries or plan.awaited:
raise ContractViolation(
f"{hook.value} runs synchronously here, so only off-path V2 callbacks may subscribe"
)
result: Final = run_legacy_plan_sync(plan, fields, binding)
if plan.off_path:
self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, result, call))
return result
def _state(self, entry: Entry) -> object:
factory: Final = entry.manifest.request_state
if factory is None:
return None
key: Final = id(entry.manifest.logger)
if key not in self._states:
self._states[key] = factory()
return self._states[key]
async def _invoke(
self, hook: Hook, entry: Entry, fields: Mapping[Field, object], call: CallFacts | None, state: object
) -> object:
policy: Final = self._dispatcher.policy
handler: Final = _handler(entry.manifest.logger, entry.subscription.handler)
context: Final = CallContext[object](hook=hook, call=call, state=state)
view: Final = policy.view(entry.subscription, fields)
result: Final = (
await _awaited(handler(view, context))
if inspect.iscoroutinefunction(handler)
else await asyncio.to_thread(handler, view, context)
)
policy.check(entry.subscription, result)
return result
def _start(
self,
hook: Hook,
entry: Entry,
fields: Mapping[Field, object],
call: CallFacts | None,
live: bool,
binding: LegacyBinding | None,
) -> Awaitable[object] | None:
if entry.subscription.legacy:
return _legacy_start(entry, binding, fields)
return self._invoke(hook, entry, fields, call, self._state(entry) if live else None)
async def _gather(
self,
hook: Hook,
entries: Sequence[Entry],
fields: Mapping[Field, object],
call: CallFacts | None,
live: bool,
binding: LegacyBinding | None = None,
) -> tuple[object, ...]:
if len(entries) == 1 and entries[0].subscription.legacy:
return (await _run_legacy(entries[0], binding, fields),)
started: Final = tuple(self._start(hook, entry, fields, call, live, binding) for entry in entries)
return await _gather_started(started, _fail_fast(hook, tuple(entries)))
async def _run_stages(
self,
hook: Hook,
stages: tuple[tuple[Entry, ...], ...],
chain: _Chain,
call: CallFacts | None,
binding: LegacyBinding | None,
) -> _Chain:
if not stages:
return chain
stage: Final = stages[0]
results: Final = await self._gather(hook, stage, chain.fields, call, live=True, binding=binding)
folded: Final = self._fold(hook, stage, results, chain)
if folded.terminal is not None:
return folded
return await self._run_stages(hook, stages[1:], folded, call, binding)
def _fold(self, hook: Hook, stage: tuple[Entry, ...], results: tuple[object, ...], chain: _Chain) -> _Chain:
pairs: Final = tuple(zip(stage, results, strict=True))
decisive: Final = next((pair for pair in pairs if _decides(hook, pair[1])), None)
self._count_failures(hook, tuple(pair for pair in pairs if pair is not decisive))
match decisive:
case (_, BaseException() as error):
raise error
case (_, Abort() | Respond() as outcome):
return _Chain(fields=chain.fields, patches=chain.patches, reroute=chain.reroute, terminal=outcome)
case _:
return self._merge(pairs, chain)
def _merge(self, pairs: tuple[tuple[Entry, object], ...], chain: _Chain) -> _Chain:
stage_patches: Final = tuple((entry.position, result) for entry, result in pairs if isinstance(result, Patch))
reroutes: Final = tuple(result for _, result in pairs if isinstance(result, Reroute))
return _Chain(
fields=apply_patches(chain.fields, (patch for _, patch in stage_patches)),
patches=(*chain.patches, *stage_patches),
reroute=chain.reroute or next(iter(reroutes), None),
)
def _count_failures(self, hook: Hook, pairs: Sequence[tuple[Entry, object]]) -> None:
for entry, result in pairs:
if isinstance(result, BaseException) and not entry.subscription.legacy:
self._dispatcher.record_failure(entry, hook, result)
def _swallow(self, hook: Hook, entries: Sequence[Entry], results: tuple[object, ...]) -> None:
pairs: Final = tuple(zip(entries, results, strict=True))
fatal: Final = next(
(result for _, result in pairs if isinstance(result, BaseException) and _fatal(result)),
None,
)
self._count_failures(hook, tuple(pair for pair in pairs if pair[1] is not fatal))
if fatal is not None:
raise fatal
async def _run_awaited(
self, hook: Hook, entries: tuple[Entry, ...], fields: Mapping[Field, object], call: CallFacts | None
) -> None:
if entries:
self._swallow(hook, entries, await self._gather(hook, entries, fields, call, live=True))
async def _deliver_off_path(
self, hook: Hook, entries: tuple[Entry, ...], fields: Mapping[Field, object], call: CallFacts | None
) -> None:
self._swallow(hook, entries, await self._gather(hook, entries, fields, call, live=False))

View file

@ -0,0 +1,95 @@
from collections.abc import Mapping
from typing import Final, Protocol
from litellm.integrations.custom_logger_v2.enums import Field, Hook
from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS, Subscription
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch, Reroute, Respond
class ContractViolation(Exception):
pass
class UndeclaredReadError(ContractViolation, AttributeError):
pass
def _field_named(name: str) -> Field | None:
member: Final = Field._value2member_map_.get(name)
return member if isinstance(member, Field) else None
class FieldView:
__slots__ = ("_fields",)
def __init__(self, fields: Mapping[Field, object]) -> None:
self._fields = fields
def __getattr__(self, name: str) -> object:
field: Final = _field_named(name)
if field is None or field not in self._fields:
raise AttributeError(name)
return self._fields[field]
class DeclaredView:
__slots__ = ("_fields", "_reads")
def __init__(self, fields: Mapping[Field, object], reads: frozenset[Field]) -> None:
self._fields = fields
self._reads = reads
def __getattr__(self, name: str) -> object:
field: Final = _field_named(name)
if field is None or field not in self._reads:
raise UndeclaredReadError(f"{name} is not among the declared reads")
if field not in self._fields:
raise AttributeError(name)
return self._fields[field]
class Policy(Protocol):
def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object: ...
def check(self, subscription: Subscription, result: object) -> None: ...
class Unchecked:
def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object:
return FieldView(fields)
def check(self, subscription: Subscription, result: object) -> None:
return None
def _outcome_problem(hook: Hook, subscription: Subscription, outcome: Abort | Respond | Reroute) -> str | None:
if not subscription.may_abort:
return f"returned {type(outcome).__name__} without may_abort"
if type(outcome) not in HOOK_SPECS[hook].outcomes:
return f"returned {type(outcome).__name__}, which {hook.value} does not allow"
return None
def _result_problem(subscription: Subscription, result: object) -> str | None:
match result:
case None:
return None
case Patch():
undeclared: Final = result.fields - subscription.writes
if not undeclared:
return None
return f"patched {', '.join(sorted(field.value for field in undeclared))} outside its declared writes"
case Abort() | Respond() | Reroute():
return _outcome_problem(subscription.hook, subscription, result)
case _:
return f"returned {type(result).__name__}, which is neither None, a Patch nor an outcome"
class Enforced:
def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object:
return DeclaredView(fields, subscription.reads)
def check(self, subscription: Subscription, result: object) -> None:
problem: Final = _result_problem(subscription, result)
if problem is not None:
raise ContractViolation(f"{subscription.handler} on {subscription.hook.value} {problem}")

View file

@ -0,0 +1,97 @@
from enum import Enum, IntEnum
from typing import Final
class Hook(str, Enum):
prompt_resolve = "prompt_resolve"
pre_route = "pre_route"
filter_deployments = "filter_deployments"
pre_call = "pre_call"
during_call = "during_call"
pre_deployment_call = "pre_deployment_call"
post_deployment_success = "post_deployment_success"
post_deployment_failure = "post_deployment_failure"
post_call_success = "post_call_success"
post_call_failure = "post_call_failure"
response_headers = "response_headers"
stream_chunk = "stream_chunk"
list_models = "list_models"
mcp_tool_result = "mcp_tool_result"
log_prepare = "log_prepare"
provider_request_sent = "provider_request_sent"
provider_response_received = "provider_response_received"
attempt_ended = "attempt_ended"
call_ended = "call_ended"
router_event = "router_event"
audit_event = "audit_event"
class FieldTier(IntEnum):
small_record = 1
content = 2
wire = 3
compatibility = 4
class Field(str, Enum):
_value_: str
bit: int
tier: FieldTier
def __new__(cls, value: str, bit: int, tier: FieldTier) -> "Field":
member: Final = str.__new__(cls, value)
member._value_ = value
member.bit = bit
member.tier = tier
return member
model = ("model", 0, FieldTier.small_record)
key_identity = ("key_identity", 1, FieldTier.small_record)
deployment = ("deployment", 2, FieldTier.small_record)
candidate_deployments = ("candidate_deployments", 3, FieldTier.small_record)
request_tags = ("request_tags", 4, FieldTier.small_record)
dynamic_params = ("dynamic_params", 5, FieldTier.small_record)
guardrail_info = ("guardrail_info", 6, FieldTier.small_record)
outcome = ("outcome", 7, FieldTier.small_record)
error = ("error", 8, FieldTier.small_record)
usage = ("usage", 9, FieldTier.small_record)
cost = ("cost", 10, FieldTier.small_record)
metadata = ("metadata", 11, FieldTier.content)
messages = ("messages", 12, FieldTier.content)
input = ("input", 13, FieldTier.content)
tools = ("tools", 14, FieldTier.content)
optional_params = ("optional_params", 15, FieldTier.content)
request_headers = ("request_headers", 16, FieldTier.content)
response = ("response", 17, FieldTier.content)
response_headers = ("response_headers", 18, FieldTier.content)
stream_chunk = ("stream_chunk", 19, FieldTier.content)
assembled_stream = ("assembled_stream", 20, FieldTier.content)
tool_result = ("tool_result", 21, FieldTier.content)
model_list = ("model_list", 22, FieldTier.content)
request_body = ("request_body", 23, FieldTier.wire)
provider_request_body = ("provider_request_body", 24, FieldTier.wire)
provider_response_raw = ("provider_response_raw", 25, FieldTier.wire)
provider_response_headers = ("provider_response_headers", 26, FieldTier.wire)
standard_logging_payload = ("standard_logging_payload", 27, FieldTier.compatibility)
deprecated_kwargs = ("deprecated_kwargs", 28, FieldTier.compatibility)
deprecated_response_obj = ("deprecated_response_obj", 29, FieldTier.compatibility)
class Activation(str, Enum):
required = "required"
default_on = "default_on"
opt_in = "opt_in"
class OptOutBy(str, Enum):
admin = "admin"
caller = "caller"
ALL_FIELDS_MASK: Final = sum(1 << field.bit for field in Field)
def field_mask(fields: frozenset[Field]) -> int:
if Field.deprecated_kwargs in fields:
return ALL_FIELDS_MASK
return sum(1 << field.bit for field in fields)

View file

@ -0,0 +1,98 @@
from collections.abc import Iterator, Mapping
from functools import reduce
from typing import Final
from litellm.integrations.custom_logger_v2.enums import Field
from litellm.integrations.custom_logger_v2.legacy import KwargsFields
from litellm.integrations.custom_logger_v2.outcomes import InvalidPatchError, Patch, PatchOp, SetField
from litellm.integrations.custom_logger_v2.patches import LiveFields
_CANDIDATE_FIELDS: Final = (Field.deprecated_kwargs, Field.candidate_deployments, Field.model)
class CandidateFields(LiveFields):
__slots__ = ("_deployments", "_model")
def __init__(self, deployments: object, model: str) -> None:
self._deployments: Final = deployments
self._model: Final = model
@property
def deployments(self) -> object:
return self._deployments
def __getitem__(self, field: Field) -> object:
if field is Field.deprecated_kwargs:
return self._deployments
if field is Field.model:
return self._model
if field is Field.candidate_deployments and isinstance(self._deployments, list | tuple):
return tuple(self._deployments) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts
raise KeyError(field)
def __iter__(self) -> Iterator[Field]:
return iter(tuple(field for field in _CANDIDATE_FIELDS if self._has(field)))
def __len__(self) -> int:
return sum(1 for field in _CANDIDATE_FIELDS if self._has(field))
def _has(self, field: Field) -> bool:
return field is not Field.candidate_deployments or isinstance(self._deployments, list | tuple)
def apply(self, patch: Patch) -> Mapping[Field, object]:
return reduce(_narrow, patch.ops, self)
def _narrow(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]:
if not isinstance(fields, CandidateFields) or not isinstance(op, SetField):
raise InvalidPatchError("candidate deployments are replaced whole, never merged")
if op.field is Field.deprecated_kwargs:
return CandidateFields(op.value, str(fields[Field.model]))
if op.field is not Field.candidate_deployments:
raise InvalidPatchError(f"{op.field.value} has no place in filter_deployments")
return CandidateFields(_narrowed(fields.deployments, op.value), str(fields[Field.model]))
def _narrowed(current: object, proposed: object) -> list[object]: # mutable-ok: legacy filters take a list
if not isinstance(current, list | tuple) or not isinstance(proposed, list | tuple):
raise InvalidPatchError("candidate_deployments must stay a sequence of the current candidates")
allowed: Final = frozenset(map(id, current)) # pyright: ignore[reportUnknownArgumentType] # untyped deployment dicts
picked: Final = tuple(map(id, proposed)) # pyright: ignore[reportUnknownArgumentType] # see above
if not allowed.issuperset(picked) or len(frozenset(picked)) != len(picked):
raise InvalidPatchError("filter_deployments may only narrow the candidates, never add or repeat one")
return list(proposed) # pyright: ignore[reportUnknownArgumentType] # see above
class ResultFields(LiveFields):
__slots__ = ("_request", "_slot", "value")
def __init__(self, request: object, keys: Mapping[Field, str], slot: Field, value: object) -> None:
self._request: Final = KwargsFields(request, keys)
self._slot: Final = slot
self.value = value
def __getitem__(self, field: Field) -> object:
if field is self._slot or field is Field.deprecated_response_obj:
return self.value
return self._request[field]
def __iter__(self) -> Iterator[Field]:
return iter((*self._request, self._slot, Field.deprecated_response_obj))
def __len__(self) -> int:
return len(self._request) + 2
def apply(self, patch: Patch) -> Mapping[Field, object]:
values: Final = tuple(_slot_value(op, self._slot) for op in patch.ops)
if values:
self.value = values[-1]
return self
def _slot_value(op: PatchOp, slot: Field) -> object:
if not isinstance(op, SetField) or op.field not in (slot, Field.deprecated_response_obj):
raise InvalidPatchError(f"{op.field.value} cannot be patched here, only {slot.value}")
return op.value
__all__ = ["CandidateFields", "ResultFields"]

View file

@ -0,0 +1,211 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook
from litellm.integrations.custom_logger_v2.outcomes import Abort, Reroute, Respond
LegacyGuardrailEvent: TypeAlias = Literal["pre_call", "during_call", "post_call", "post_mcp_call"]
OutcomeType: TypeAlias = type[Abort] | type[Respond] | type[Reroute]
class InvalidDeclarationError(TypeError):
pass
@dataclass(frozen=True, slots=True)
class HookSpec:
legacy_methods: tuple[str, ...]
request_path: bool
writes: frozenset[Field] = frozenset()
outcomes: frozenset[OutcomeType] = frozenset()
legacy_guardrail_events: frozenset[LegacyGuardrailEvent] = frozenset()
has_call: bool = True
subscribable: bool = True
fail_fast_parallel: bool = False
_NO_OUTCOME: Final[frozenset[OutcomeType]] = frozenset()
_ABORT: Final[frozenset[OutcomeType]] = frozenset({Abort})
_ABORT_OR_RESPOND: Final[frozenset[OutcomeType]] = frozenset({Abort, Respond})
_ANY_OUTCOME: Final[frozenset[OutcomeType]] = frozenset({Abort, Respond, Reroute})
HOOK_SPECS: Final[Mapping[Hook, HookSpec]] = MappingProxyType(
{
Hook.prompt_resolve: HookSpec(
legacy_methods=("get_chat_completion_prompt", "async_get_chat_completion_prompt"),
request_path=True,
writes=frozenset({Field.messages, Field.optional_params, Field.model}),
outcomes=_ABORT_OR_RESPOND,
),
Hook.pre_route: HookSpec(
legacy_methods=("async_pre_routing_hook",),
request_path=True,
writes=frozenset({Field.model, Field.messages}),
outcomes=_ABORT_OR_RESPOND,
),
Hook.filter_deployments: HookSpec(
legacy_methods=("async_filter_deployments",),
request_path=True,
writes=frozenset({Field.candidate_deployments}),
outcomes=_ABORT,
),
Hook.pre_call: HookSpec(
legacy_methods=("async_pre_call_hook",),
request_path=True,
writes=frozenset(
{
Field.messages,
Field.optional_params,
Field.metadata,
Field.request_headers,
Field.guardrail_info,
Field.tools,
Field.model,
Field.input,
}
),
outcomes=_ANY_OUTCOME,
legacy_guardrail_events=frozenset({"pre_call"}),
),
Hook.during_call: HookSpec(
legacy_methods=("async_moderation_hook",),
request_path=True,
outcomes=_ABORT_OR_RESPOND,
legacy_guardrail_events=frozenset({"during_call"}),
fail_fast_parallel=True,
),
Hook.pre_deployment_call: HookSpec(
legacy_methods=("async_pre_call_deployment_hook", "async_pre_request_hook"),
request_path=True,
writes=frozenset({Field.provider_request_body, Field.request_headers}),
outcomes=_ABORT_OR_RESPOND,
),
Hook.post_deployment_success: HookSpec(
legacy_methods=("async_post_call_success_deployment_hook", "async_post_call_streaming_deployment_hook"),
request_path=True,
writes=frozenset({Field.response}),
outcomes=_ABORT_OR_RESPOND,
),
Hook.post_deployment_failure: HookSpec(
legacy_methods=("async_post_call_failure_deployment_hook",),
request_path=True,
writes=frozenset({Field.error}),
outcomes=_ABORT,
),
Hook.post_call_success: HookSpec(
legacy_methods=("async_post_call_success_hook",),
request_path=True,
writes=frozenset({Field.response, Field.guardrail_info}),
outcomes=_ABORT_OR_RESPOND,
legacy_guardrail_events=frozenset({"post_call"}),
),
Hook.post_call_failure: HookSpec(
legacy_methods=("async_post_call_failure_hook",),
request_path=True,
writes=frozenset({Field.error}),
outcomes=_ABORT,
),
Hook.response_headers: HookSpec(
legacy_methods=("async_post_call_response_headers_hook",),
request_path=True,
writes=frozenset({Field.response_headers}),
outcomes=_NO_OUTCOME,
),
Hook.stream_chunk: HookSpec(
legacy_methods=(
"async_post_call_streaming_hook",
"async_post_call_streaming_iterator_hook",
"log_stream_event",
),
request_path=True,
writes=frozenset({Field.stream_chunk}),
outcomes=_ABORT,
subscribable=False,
),
Hook.list_models: HookSpec(
legacy_methods=("async_filter_listed_models",),
request_path=True,
writes=frozenset({Field.model_list}),
outcomes=_ABORT,
has_call=False,
),
Hook.mcp_tool_result: HookSpec(
legacy_methods=("async_post_mcp_tool_call_hook",),
request_path=True,
writes=frozenset({Field.tool_result}),
outcomes=_ABORT_OR_RESPOND,
legacy_guardrail_events=frozenset({"post_mcp_call"}),
),
Hook.log_prepare: HookSpec(
legacy_methods=(
"logging_hook",
"async_logging_hook",
"redact_standard_logging_payload_from_model_call_details",
),
request_path=False,
writes=frozenset({Field.standard_logging_payload}),
),
Hook.provider_request_sent: HookSpec(legacy_methods=("log_pre_api_call",), request_path=False),
Hook.provider_response_received: HookSpec(legacy_methods=("log_post_api_call",), request_path=False),
Hook.attempt_ended: HookSpec(
legacy_methods=(
"log_success_event",
"log_failure_event",
"async_log_success_event",
"async_log_failure_event",
),
request_path=False,
),
Hook.call_ended: HookSpec(legacy_methods=(), request_path=False),
Hook.router_event: HookSpec(
legacy_methods=(
"log_success_fallback_event",
"log_failure_fallback_event",
"log_model_group_rate_limit_error",
),
request_path=False,
),
Hook.audit_event: HookSpec(
legacy_methods=("async_log_audit_log_event", "async_log_batch_line_items"),
request_path=False,
has_call=False,
),
}
)
@dataclass(frozen=True, slots=True)
class Subscription:
hook: Hook
handler: str
reads: frozenset[Field] = frozenset()
writes: frozenset[Field] = frozenset()
may_abort: bool = False
parallel: bool = False
awaited: bool = False
activation: Activation | None = None
legacy: bool = False
def __post_init__(self) -> None:
problem: Final = subscription_problem(self)
if problem is not None:
raise InvalidDeclarationError(f"{self.handler} on {self.hook.value}: {problem}")
def subscription_problem(subscription: Subscription) -> str | None:
spec: Final = HOOK_SPECS[subscription.hook]
if not spec.subscribable:
return "this hook point has no V2 subscription support yet"
legacy_writes: Final = frozenset({Field.deprecated_kwargs}) if subscription.legacy else frozenset[Field]()
forbidden: Final = subscription.writes - spec.writes - legacy_writes
if forbidden:
return f"writes {', '.join(sorted(field.value for field in forbidden))}, which this hook point cannot apply"
if subscription.may_abort and not spec.outcomes:
return "may_abort is set, but this hook point allows no outcome"
if subscription.awaited and (subscription.writes or subscription.may_abort):
return "an awaited subscription must neither write nor abort"
if subscription.parallel and (subscription.writes or not subscription.may_abort):
return "parallel is only for subscriptions that may abort and write nothing"
return None

View file

@ -0,0 +1,62 @@
from collections.abc import Generator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Final
from litellm.integrations.custom_logger_v2.dispatcher import Dispatcher, RequestScope
from litellm.integrations.custom_logger_v2.outcomes import Outcome, Patch
from litellm.integrations.custom_logger_v2.registry import CallbackRegistry
from litellm.integrations.custom_logger_v2.selection import RequestSelection
_NO_SELECTION: Final = RequestSelection()
_CURRENT_SCOPE: Final[ContextVar[RequestScope | None]] = ContextVar("custom_logger_v2_scope", default=None)
class V2Outcome(Exception):
def __init__(self, outcome: Outcome | None, patches: tuple[Patch, ...]) -> None:
super().__init__(f"a V2 callback ended the hook with {type(outcome).__name__}")
self.outcome: Final = outcome
self.patches: Final = patches
class _Host:
def __init__(self) -> None:
self._dispatcher: Dispatcher | None = None
def dispatcher(self) -> Dispatcher:
if self._dispatcher is None:
self._dispatcher = Dispatcher(CallbackRegistry())
return self._dispatcher
def reset(self) -> None:
self._dispatcher = None
_HOST: Final = _Host()
def get_dispatcher() -> Dispatcher:
return _HOST.dispatcher()
def reset_host_for_tests() -> None:
_HOST.reset()
def current_scope() -> RequestScope | None:
return _CURRENT_SCOPE.get()
@contextmanager
def bind_scope(scope: RequestScope) -> Generator[RequestScope]:
token: Final = _CURRENT_SCOPE.set(scope)
try:
yield scope
finally:
_CURRENT_SCOPE.reset(token)
@contextmanager
def request_scope(selection: RequestSelection = _NO_SELECTION) -> Generator[RequestScope]:
with _HOST.dispatcher().open_scope(selection) as scope, bind_scope(scope):
yield scope

View file

@ -0,0 +1,434 @@
import weakref
from collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from enum import Enum
from functools import lru_cache, reduce
from itertools import count
from types import MappingProxyType
from typing import Final, TypeAlias, TypeVar
from typing_extensions import TypeIs # noqa: TID251 # narrows the untyped legacy kwargs without copying them
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger, subclass_overrides_hook
from litellm.integrations.custom_logger_v2.call import CallFacts
from litellm.integrations.custom_logger_v2.dispatcher import (
Between,
HookResult,
RequestScope,
run_legacy_plan,
run_legacy_plan_sync,
)
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook
from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS, Subscription
from litellm.integrations.custom_logger_v2.host import V2Outcome, current_scope, get_dispatcher
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest, LegacyBinding, build_manifest
from litellm.integrations.custom_logger_v2.outcomes import Abort, InvalidPatchError, MergeField, Patch, PatchOp, Respond
from litellm.integrations.custom_logger_v2.patches import LiveFields
from litellm.integrations.custom_logger_v2.planner import Entry, HookPlan, plan_hook
from litellm.integrations.custom_logger_v2.selection import RequestSelection, Selection
ResultT: Final = TypeVar("ResultT")
SyncLegacyBinding: TypeAlias = Callable[[object, object], object]
# These hook points are the only ones where v1 custom logger subclasses are
# checked to see if they run in parallel
PARALLEL_GUARDRAIL_HOOKS: Final = frozenset({Hook.pre_call, Hook.post_call_success})
LEGACY_READS: Final = frozenset({Field.deprecated_kwargs, Field.deprecated_response_obj})
PROXY_REQUEST_KEYS: Final[Mapping[Field, str]] = MappingProxyType(
{
Field.model: "model",
Field.messages: "messages",
Field.input: "input",
Field.tools: "tools",
}
)
SUBSCRIBERS_CACHE_SIZE: Final = 128
PLAN_CACHE_SIZE: Final = 64
_SERIALS: Final = count()
class Admit(str, Enum):
overriders = "overriders"
every_logger = "every_logger"
every_entry = "every_entry"
_ABSENT: Final = object()
_NO_SELECTION: Final = RequestSelection()
@lru_cache(maxsize=None)
def legacy_subscription(hook: Hook, parallel: bool) -> Subscription:
spec: Final = HOOK_SPECS[hook]
may_abort: Final = bool(spec.outcomes)
gathered: Final = may_abort and (parallel or hook is Hook.during_call)
return Subscription(
hook=hook,
handler="call",
reads=LEGACY_READS,
writes=frozenset({Field.deprecated_kwargs}) if spec.writes and not gathered else frozenset(),
may_abort=may_abort,
parallel=gathered,
legacy=True,
)
def _subscribes(cls: type, guardrail: bool, hook: Hook) -> bool:
spec: Final = HOOK_SPECS[hook]
if not spec.subscribable:
return False
if guardrail and spec.legacy_guardrail_events:
return True
return any(subclass_overrides_hook(cls, method) for method in spec.legacy_methods)
@lru_cache(maxsize=512)
def _derived_subscriptions(cls: type, guardrail: bool, parallel: bool) -> Mapping[Hook, Subscription]:
return MappingProxyType(
{
hook: legacy_subscription(hook, parallel and hook in PARALLEL_GUARDRAIL_HOOKS)
for hook in Hook
if _subscribes(cls, guardrail, hook)
}
)
class LegacyLogger(CustomLoggerV2):
def __init__(self, inner: CustomLogger) -> None:
guardrail: Final = isinstance(inner, CustomGuardrail)
self.parallel = guardrail and bool(getattr(inner, "run_in_parallel", False))
self.serial: Final = next(_SERIALS)
self._inner: Final = weakref.ref(inner)
self.manifest: Final = CallbackManifest(
logger=self,
selection=Selection(name=type(inner).__name__, activation=Activation.required),
subscriptions=_derived_subscriptions(type(inner), guardrail, self.parallel),
request_state=None,
legacy_call=self.call,
)
@property
def inner(self) -> CustomLogger | None:
return self._inner()
def subscription(self, hook: Hook, admit: Admit) -> Subscription | None:
derived: Final = self.manifest.subscriptions.get(hook)
if derived is not None or admit is Admit.overriders or not HOOK_SPECS[hook].subscribable:
return derived
return legacy_subscription(hook, self.parallel and hook in PARALLEL_GUARDRAIL_HOOKS)
def call(self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object]) -> ResultT | None:
inner: Final = self._inner()
return None if inner is None else binding(inner, fields[Field.deprecated_kwargs])
_ADAPTERS: Final[dict[int, LegacyLogger]] = {} # mutable-ok: adapters live as long as the logger they wrap
def adapter_for(inner: CustomLogger) -> LegacyLogger:
key: Final = id(inner)
cached: Final = _ADAPTERS.get(key)
if cached is not None and cached.inner is inner:
return cached
adapter: Final = LegacyLogger(inner)
_ADAPTERS[key] = adapter
_ = weakref.finalize(inner, _ADAPTERS.pop, key, None)
return adapter
def _readable(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: legacy kwargs are an untyped dict
return isinstance(value, Mapping)
def _writable(
value: object,
) -> TypeIs[MutableMapping[str, object]]: # guard-ok: legacy kwargs are an untyped dict # mutable-ok: written in place
return isinstance(value, MutableMapping)
class OpaqueLegacy(CustomLoggerV2):
"""Holds a callback list entry that is not a CustomLogger (a string name or a plain function) at its list
position, for hook points that admit every entry. There is no class to derive subscriptions from, so it
subscribes to the one hook it was built for and runs serially."""
def __init__(self, inner: object, hook: Hook) -> None:
self.inner: Final = inner
self.serial: Final = next(_SERIALS)
self.manifest: Final = CallbackManifest(
logger=self,
selection=Selection(name=type(inner).__name__, activation=Activation.required),
subscriptions=MappingProxyType({hook: legacy_subscription(hook, False)}),
request_state=None,
legacy_call=self.call,
)
def call(self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object]) -> ResultT | None:
return binding(self.inner, fields[Field.deprecated_kwargs])
class KwargsFields(LiveFields):
"""A live Field view over a legacy hook's untyped kwargs dict, so the dispatcher can run over it without copying.
Two fields cover the whole call: deprecated_kwargs is the dict itself, and deprecated_response_obj is the
response, when the hook point passes one. Patching either one swaps in the new value.
Every other field maps to one key in the dict through the hook point's key table (PROXY_REQUEST_KEYS or
LOGGING_KEYS), and is present only when that key is. Patching one writes straight into the dict, so later
callbacks and the request see the change."""
__slots__ = ("_keys", "_kwargs", "_response")
def __init__(self, kwargs: object, keys: Mapping[Field, str], response: object = _ABSENT) -> None:
self._kwargs: Final = kwargs
self._keys: Final = keys
self._response: Final = response
def _key(self, field: Field) -> str | None:
key: Final = self._keys.get(field)
return key if key is not None and _readable(self._kwargs) and key in self._kwargs else None
def __getitem__(self, field: Field) -> object:
if field is Field.deprecated_kwargs:
return self._kwargs
if field is Field.deprecated_response_obj and self._response is not _ABSENT:
return self._response
key: Final = self._key(field)
if key is None or not _readable(self._kwargs):
raise KeyError(field)
return self._kwargs[key]
def __iter__(self) -> Iterator[Field]:
return iter(tuple(field for field in Field if self._has(field)))
def __len__(self) -> int:
return sum(1 for field in Field if self._has(field))
def _has(self, field: Field) -> bool:
if field is Field.deprecated_kwargs:
return True
if field is Field.deprecated_response_obj:
return self._response is not _ABSENT
return self._key(field) is not None
def apply(self, patch: Patch) -> Mapping[Field, object]:
return reduce(_apply_op, patch.ops, self)
def replaced(self, field: Field, value: object) -> "KwargsFields":
if field is Field.deprecated_kwargs:
return KwargsFields(value, self._keys, self._response)
return KwargsFields(self._kwargs, self._keys, value)
def write(self, field: Field, value: object) -> None:
target: Final = self._target(field)
target[self._keys[field]] = value
def merge(self, field: Field, keys: Mapping[str, object]) -> None:
target: Final = self._target(field)
current: Final = target.get(self._keys[field])
if current is None:
target[self._keys[field]] = dict(keys)
return
if not _writable(current):
raise InvalidPatchError(f"{field.value} holds no mapping here, so keys cannot be merged into it")
current.update(keys)
def _target(self, field: Field) -> MutableMapping[str, object]: # mutable-ok: legacy kwargs are the live request
if field not in self._keys:
raise InvalidPatchError(f"{field.value} has no place in the legacy kwargs of this hook")
if not _writable(self._kwargs):
raise InvalidPatchError(f"{field.value} cannot be written because the legacy kwargs are not a mapping")
return self._kwargs
def _apply_op(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]:
if not isinstance(fields, KwargsFields):
raise InvalidPatchError("legacy kwargs patches need KwargsFields")
if op.field in (Field.deprecated_kwargs, Field.deprecated_response_obj):
if isinstance(op, MergeField):
raise InvalidPatchError(f"{op.field.value} is replaced whole, never merged")
return fields.replaced(op.field, op.value)
if isinstance(op, MergeField):
fields.merge(op.field, op.keys)
return fields
fields.write(op.field, op.value)
return fields
def kwargs_patch(before: object, after: object) -> Patch | None:
return None if after is before else Patch().set(Field.deprecated_kwargs, after)
@dataclass(frozen=True, slots=True, eq=False)
class HookSubscribers:
hook: Hook
entries: tuple[Entry, ...]
plan: HookPlan
has_v2: bool
plans: Callable[[tuple[int, ...]], HookPlan]
@property
def legacy(self) -> tuple[object, ...]:
return tuple(inner for entry in self.entries if (inner := _wrapped(entry.manifest.logger)) is not None)
def plan_for(self, scope: RequestScope) -> HookPlan:
selected: Final = tuple(
index for index, entry in enumerate(self.entries) if entry.subscription.legacy or scope.selects(entry)
)
return self.plan if len(selected) == len(self.entries) else self.plans(selected)
def _wrapped(logger: CustomLoggerV2) -> object:
if isinstance(logger, LegacyLogger | OpaqueLegacy):
return logger.inner
return None
def _v2_entry(v2_callback: CustomLoggerV2, hook: Hook) -> tuple[CallbackManifest, Subscription] | None:
manifest: Final = build_manifest(v2_callback)
subscription: Final = manifest.subscriptions.get(hook)
return None if subscription is None else (manifest, subscription)
def _legacy_entry(
inner: CustomLogger, hook: Hook, admit: Admit, parallel: bool
) -> tuple[CallbackManifest, Subscription] | None:
adapter: Final = adapter_for(inner)
subscription: Final = adapter.subscription(hook, admit)
if subscription is None:
return None
if subscription.parallel and not parallel:
return adapter.manifest, legacy_subscription(hook, False)
return adapter.manifest, subscription
def _opaque_entry(item: object, hook: Hook) -> tuple[CallbackManifest, Subscription] | None:
if not HOOK_SPECS[hook].subscribable:
return None
opaque: Final = OpaqueLegacy(item, hook)
return opaque.manifest, opaque.manifest.subscriptions[hook]
def _subscriber(
item: object, hook: Hook, admit: Admit, include_v2: bool, parallel: bool
) -> tuple[CallbackManifest, Subscription] | None:
if isinstance(item, CustomLoggerV2):
return _v2_entry(item, hook) if include_v2 else None
if isinstance(item, CustomLogger):
return _legacy_entry(item, hook, admit, parallel)
return _opaque_entry(item, hook) if admit is Admit.every_entry else None
def _build_subscribers(
hook: Hook, callbacks: Sequence[object], admit: Admit, include_v2: bool, parallel: bool
) -> HookSubscribers:
members: Final = tuple(
member for item in callbacks if (member := _subscriber(item, hook, admit, include_v2, parallel)) is not None
)
entries: Final = tuple(
Entry(position, manifest, subscription) for position, (manifest, subscription) in enumerate(members)
)
def subset_plan(selected: tuple[int, ...]) -> HookPlan:
return plan_hook(tuple(entries[index] for index in selected))
return HookSubscribers(
hook=hook,
entries=entries,
plan=plan_hook(entries),
has_v2=any(not entry.subscription.legacy for entry in entries),
plans=lru_cache(maxsize=PLAN_CACHE_SIZE)(subset_plan),
)
@dataclass(frozen=True, slots=True, eq=False)
class _CachedSubscribers:
subscribers: HookSubscribers
pinned: tuple[object, ...]
_SUBSCRIBERS: Final[dict[tuple[object, ...], _CachedSubscribers]] = {} # mutable-ok: bounded subscribers cache
def _pinned(item: object) -> bool:
return not isinstance(item, CustomLogger) or isinstance(item, CustomLoggerV2)
def _cache_subscribers(key: tuple[object, ...], subscribers: HookSubscribers, callbacks: Sequence[object]) -> None:
if len(_SUBSCRIBERS) >= SUBSCRIBERS_CACHE_SIZE:
_SUBSCRIBERS.clear()
_SUBSCRIBERS[key] = _CachedSubscribers(
subscribers=subscribers, pinned=tuple(item for item in callbacks if _pinned(item))
)
for item in callbacks:
if not _pinned(item):
_ = weakref.finalize(item, _SUBSCRIBERS.pop, key, None)
def hook_subscribers(
hook: Hook,
callbacks: Sequence[object],
*,
admit: Admit = Admit.overriders,
include_v2: bool = True,
parallel: bool = True,
) -> HookSubscribers:
key: Final = (hook, admit, include_v2, parallel, *map(id, callbacks))
cached: Final = _SUBSCRIBERS.get(key)
if cached is not None:
return cached.subscribers
subscribers: Final = _build_subscribers(hook, callbacks, admit, include_v2, parallel)
_cache_subscribers(key, subscribers, callbacks)
return subscribers
def clear_subscribers_cache_for_tests() -> None:
_SUBSCRIBERS.clear()
async def run_subscribers(
subscribers: HookSubscribers,
fields: Mapping[Field, object],
binding: LegacyBinding,
*,
between: Between | None = None,
call: CallFacts | None = None,
scope: RequestScope | None = None,
) -> HookResult:
if not subscribers.has_v2:
return HookResult(fields=await run_legacy_plan(subscribers.hook, subscribers.plan, fields, binding, between))
bound: Final = scope or current_scope()
if bound is not None:
return await bound.run_plan(subscribers.hook, subscribers.plan_for(bound), fields, call, binding, between)
with get_dispatcher().open_scope(_NO_SELECTION) as transient:
return await transient.run_plan(
subscribers.hook, subscribers.plan_for(transient), fields, call, binding, between
)
def run_subscribers_sync(
subscribers: HookSubscribers,
fields: Mapping[Field, object],
binding: SyncLegacyBinding,
*,
call: CallFacts | None = None,
scope: RequestScope | None = None,
) -> Mapping[Field, object]:
if not subscribers.has_v2:
return run_legacy_plan_sync(subscribers.plan, fields, binding)
bound: Final = scope or current_scope()
if bound is not None:
return bound.run_plan_sync(subscribers.hook, subscribers.plan_for(bound), fields, call, binding)
with get_dispatcher().open_scope(_NO_SELECTION) as transient:
return transient.run_plan_sync(subscribers.hook, subscribers.plan_for(transient), fields, call, binding)
def kwargs_after(result: HookResult) -> object:
match result.outcome:
case Abort() | Respond() as terminal:
raise V2Outcome(terminal, result.patches)
case _:
return result.fields[Field.deprecated_kwargs]

View file

@ -0,0 +1,109 @@
import inspect
from collections.abc import Callable, Mapping, Sequence, Set
from dataclasses import replace
from types import MappingProxyType
from typing import ClassVar, Final, TypeVar
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook
from litellm.integrations.custom_logger_v2.hooks import InvalidDeclarationError, Subscription
from litellm.integrations.custom_logger_v2.selection import Selection
HandlerT: Final = TypeVar("HandlerT", bound=Callable[..., object])
_SUBSCRIPTION_ATTR: Final = "__custom_logger_v2_subscription__"
def on(
hook: Hook,
*,
reads: Set[Field] = frozenset(),
writes: Set[Field] = frozenset(),
may_abort: bool = False,
parallel: bool | None = None,
awaited: bool = False,
) -> Callable[[HandlerT], HandlerT]:
def decorate(handler: HandlerT) -> HandlerT:
subscription: Final = Subscription(
hook=hook,
handler=handler.__name__,
reads=frozenset(reads),
writes=frozenset(writes),
may_abort=may_abort,
parallel=(hook is Hook.during_call and may_abort) if parallel is None else parallel,
awaited=awaited,
)
setattr(handler, _SUBSCRIPTION_ATTR, subscription)
return handler
return decorate
def _declared_subscription(value: object) -> Subscription | None:
marker: Final[object] = getattr(value, _SUBSCRIPTION_ATTR, None)
return marker if isinstance(marker, Subscription) else None
def _namespace(cls: type) -> Mapping[str, object]:
return vars(cls)
def _declared_in(cls: type) -> Mapping[Hook, Subscription]:
declared: Final = tuple(
replace(subscription, handler=name)
for name, value in _namespace(cls).items()
if (subscription := _declared_subscription(value)) is not None
)
hooks: Final = tuple(subscription.hook for subscription in declared)
duplicated: Final = sorted({hook.value for hook in hooks if hooks.count(hook) > 1})
if duplicated:
raise InvalidDeclarationError(f"{cls.__name__} declares more than one handler for {', '.join(duplicated)}")
return MappingProxyType({subscription.hook: subscription for subscription in declared})
def _nearest(layers: Sequence[Mapping[Hook, Subscription]], hook: Hook) -> Subscription | None:
return next((layer[hook] for layer in layers if hook in layer), None)
def _handler_problem(cls: type, subscription: Subscription) -> str | None:
handler: Final[object] = getattr(cls, subscription.handler, None)
if not callable(handler):
return f"{subscription.handler} handles {subscription.hook.value} but is not callable"
redeclared: Final = _declared_subscription(handler)
if redeclared is not None and redeclared.hook is not subscription.hook:
return (
f"{subscription.handler} is decorated for {redeclared.hook.value} but also inherits "
f"the {subscription.hook.value} subscription"
)
blocking: Final = bool(subscription.writes) or subscription.may_abort or subscription.awaited
if blocking and not inspect.iscoroutinefunction(handler):
return f"{subscription.handler} must be async because it writes, may abort or is awaited"
return None
class CustomLoggerV2:
_own_subscriptions: ClassVar[Mapping[Hook, Subscription]] = MappingProxyType({})
subscriptions: ClassVar[Mapping[Hook, Subscription]] = MappingProxyType({})
request_state: ClassVar[type | None] = None
declared_hooks: ClassVar[frozenset[Hook]] = frozenset()
selection: Selection | None = None
enabled_hooks: frozenset[Hook] | None = None
parallel: bool = False
activation_overrides: Mapping[Hook, Activation] = MappingProxyType({})
def __init_subclass__(cls) -> None:
super().__init_subclass__()
cls._own_subscriptions = _declared_in(cls)
layers: Final = tuple(base._own_subscriptions for base in cls.__mro__ if issubclass(base, CustomLoggerV2))
resolved: Final = MappingProxyType(
{hook: subscription for hook in Hook if (subscription := _nearest(layers, hook)) is not None}
)
problems: Final = tuple(
problem
for subscription in resolved.values()
if (problem := _handler_problem(cls, subscription)) is not None
)
if problems:
raise InvalidDeclarationError(f"{cls.__name__}: {'; '.join(problems)}")
cls.subscriptions = resolved
CustomLoggerV2.declared_hooks = CustomLoggerV2.declared_hooks | resolved.keys()

View file

@ -0,0 +1,73 @@
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Final, Protocol, TypeAlias, TypeVar
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook, OptOutBy
from litellm.integrations.custom_logger_v2.hooks import InvalidDeclarationError, Subscription
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
from litellm.integrations.custom_logger_v2.selection import Selection
ResultT: Final = TypeVar("ResultT")
LegacyBinding: TypeAlias = Callable[[object, object], Awaitable[object] | None]
class LegacyCall(Protocol):
def __call__(
self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object], /
) -> ResultT | None: ...
@dataclass(frozen=True, slots=True)
class CallbackManifest:
logger: CustomLoggerV2
selection: Selection
subscriptions: Mapping[Hook, Subscription]
request_state: type | None
legacy_call: LegacyCall | None = None
def _instance_subscription(subscription: Subscription, parallel: bool, activation: Activation | None) -> Subscription:
return replace(
subscription,
parallel=subscription.parallel or (parallel and subscription.may_abort),
activation=activation,
)
def _caller_can_remove(selection: Selection) -> bool:
return selection.honors_no_log or OptOutBy.caller in selection.opt_out_by
def build_manifest(logger: CustomLoggerV2) -> CallbackManifest:
name: Final = type(logger).__name__
declared: Final = type(logger).subscriptions
enabled: Final = frozenset(declared) if logger.enabled_hooks is None else logger.enabled_hooks
not_implemented: Final = enabled - declared.keys()
if not_implemented:
hooks: Final = ", ".join(sorted(hook.value for hook in not_implemented))
raise InvalidDeclarationError(f"{name} enables {hooks}, which it does not implement")
stray_overrides: Final = logger.activation_overrides.keys() - enabled
if stray_overrides:
overridden: Final = ", ".join(sorted(hook.value for hook in stray_overrides))
raise InvalidDeclarationError(f"{name} overrides activation for {overridden}, which it does not enable")
subscriptions: Final = MappingProxyType(
{
hook: _instance_subscription(declared[hook], logger.parallel, logger.activation_overrides.get(hook))
for hook in Hook
if hook in enabled
}
)
selection: Final = logger.selection or Selection(name=name)
blocking: Final = any(subscription.writes or subscription.may_abort for subscription in subscriptions.values())
if blocking and _caller_can_remove(selection):
raise InvalidDeclarationError(
f"{selection.name} writes or may abort, so neither no-log nor a caller opt-out may remove it"
)
return CallbackManifest(
logger=logger,
selection=selection,
subscriptions=subscriptions,
request_state=type(logger).request_state,
)

View file

@ -0,0 +1,27 @@
import asyncio
from collections.abc import Coroutine
from concurrent.futures import ThreadPoolExecutor
from typing import Final, Protocol
class OffPathSink(Protocol):
def submit(self, delivery: Coroutine[object, object, None]) -> None: ...
class InProcessSink:
def __init__(self) -> None:
self._pending: set[asyncio.Task[None]] = set() # mutable-ok: strong references keep running tasks alive
self._loopless: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="custom-logger-v2-off-path")
def submit(self, delivery: Coroutine[object, object, None]) -> None:
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
_ = self._loopless.submit(asyncio.run, delivery)
return
task: Final[asyncio.Task[None]] = loop.create_task(delivery)
self._pending.add(task)
task.add_done_callback(self._pending.discard)
async def drain(self) -> None:
await asyncio.gather(*self._pending)

View file

@ -0,0 +1,106 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import ClassVar, Final, TypeAlias
from pydantic import BaseModel
from litellm.integrations.custom_logger_v2.enums import Field
JsonValue: TypeAlias = "str | int | float | bool | None | Sequence[JsonValue] | Mapping[str, JsonValue]"
MERGEABLE_FIELDS: Final = frozenset(
{
Field.dynamic_params,
Field.guardrail_info,
Field.metadata,
Field.optional_params,
Field.request_headers,
Field.response_headers,
Field.provider_request_body,
Field.standard_logging_payload,
}
)
OUTCOME_PATCH_FIELDS: Final = frozenset({Field.guardrail_info, Field.metadata})
class InvalidPatchError(ValueError):
pass
@dataclass(frozen=True, slots=True)
class SetField:
field: Field
value: object
@dataclass(frozen=True, slots=True)
class MergeField:
field: Field
keys: Mapping[str, object]
PatchOp: TypeAlias = SetField | MergeField
@dataclass(frozen=True, slots=True)
class Patch:
ops: tuple[PatchOp, ...] = ()
def set(self, field: Field, value: object) -> "Patch":
return Patch((*self.ops, SetField(field, value)))
def merge(self, field: Field, keys: Mapping[str, object]) -> "Patch":
if field not in MERGEABLE_FIELDS:
raise InvalidPatchError(f"{field.value} is not a mapping field, so it cannot be merged by key")
return Patch((*self.ops, MergeField(field, MappingProxyType(dict(keys)))))
@property
def fields(self) -> frozenset[Field]:
return frozenset(op.field for op in self.ops)
def _require_outcome_patch(patch: Patch | None) -> None:
if patch is None:
return
outside: Final = patch.fields - OUTCOME_PATCH_FIELDS
if outside:
names: Final = ", ".join(sorted(field.value for field in outside))
raise InvalidPatchError(f"an outcome may only patch guardrail_info and metadata, not {names}")
@dataclass(frozen=True, slots=True)
class Abort:
terminal: ClassVar[bool] = True
status: int
message: str
code: str | None = None
headers: Mapping[str, str] | None = None
body: JsonValue = None
patch: Patch | None = None
def __post_init__(self) -> None:
_require_outcome_patch(self.patch)
@dataclass(frozen=True, slots=True)
class Respond:
terminal: ClassVar[bool] = True
response: "JsonValue | BaseModel"
status: int = 200
patch: Patch | None = None
def __post_init__(self) -> None:
_require_outcome_patch(self.patch)
@dataclass(frozen=True, slots=True)
class Reroute:
terminal: ClassVar[bool] = False
model: str
Outcome: TypeAlias = Abort | Respond | Reroute

View file

@ -0,0 +1,70 @@
from abc import abstractmethod
from collections.abc import Iterable, Iterator, Mapping
from functools import reduce
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
from litellm.integrations.custom_logger_v2.enums import Field
from litellm.integrations.custom_logger_v2.outcomes import InvalidPatchError, MergeField, Patch, PatchOp, SetField
_STRING_KEYED: Final = TypeAdapter(Mapping[str, object])
class LiveFields(Mapping[Field, object]):
__slots__ = ()
@abstractmethod
def apply(self, patch: Patch) -> Mapping[Field, object]: ...
class PatchedFields(Mapping[Field, object]):
__slots__ = ("_base", "_changed")
def __init__(self, base: Mapping[Field, object], changed: Mapping[Field, object]) -> None:
self._base = base
self._changed = changed
def with_value(self, field: Field, value: object) -> "PatchedFields":
return PatchedFields(self._base, MappingProxyType({**self._changed, field: value}))
def __getitem__(self, field: Field) -> object:
return self._changed[field] if field in self._changed else self._base[field]
def __iter__(self) -> Iterator[Field]:
return iter(self._changed.keys() | self._base.keys())
def __len__(self) -> int:
return len(self._changed.keys() | self._base.keys())
def _with_value(fields: Mapping[Field, object], field: Field, value: object) -> PatchedFields:
patched: Final = fields if isinstance(fields, PatchedFields) else PatchedFields(fields, MappingProxyType({}))
return patched.with_value(field, value)
def _merged(current: object, field: Field, keys: Mapping[str, object]) -> Mapping[str, object]:
if current is None:
return MappingProxyType(dict(keys))
if not isinstance(current, Mapping):
raise InvalidPatchError(f"{field.value} holds no mapping, so keys cannot be merged into it")
return MappingProxyType({**_STRING_KEYED.validate_python(current), **keys})
def _apply_op(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]:
match op:
case SetField(field=field, value=value):
return _with_value(fields, field, value)
case MergeField(field=field, keys=keys):
return _with_value(fields, field, _merged(fields.get(field), field, keys))
def apply_patch(fields: Mapping[Field, object], patch: Patch) -> Mapping[Field, object]:
if isinstance(fields, LiveFields):
return fields.apply(patch)
return reduce(_apply_op, patch.ops, fields)
def apply_patches(fields: Mapping[Field, object], patches: Iterable[Patch]) -> Mapping[Field, object]:
return reduce(apply_patch, patches, fields)

View file

@ -0,0 +1,77 @@
from collections.abc import Sequence
from dataclasses import dataclass
from functools import reduce
from typing import Final
from litellm.integrations.custom_logger_v2.enums import Activation, Field, field_mask
from litellm.integrations.custom_logger_v2.hooks import Subscription
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest
@dataclass(frozen=True, slots=True, eq=False)
class Entry:
position: int
manifest: CallbackManifest
subscription: Subscription
@property
def activation(self) -> Activation:
return self.subscription.activation or self.manifest.selection.activation
@dataclass(frozen=True, slots=True)
class HookPlan:
stages: tuple[tuple[Entry, ...], ...] = ()
inline: tuple[Entry, ...] = ()
awaited: tuple[Entry, ...] = ()
off_path: tuple[Entry, ...] = ()
reads: frozenset[Field] = frozenset()
EMPTY_PLAN: Final = HookPlan()
def _blocks(subscription: Subscription) -> bool:
return bool(subscription.writes) or subscription.may_abort
def _level(placed: tuple[tuple[Subscription, int], ...], subscription: Subscription) -> int:
if subscription.may_abort:
return max((level for _, level in placed), default=-1) + 1
touched: Final = field_mask(subscription.reads | subscription.writes)
writes: Final = field_mask(subscription.writes)
after_writers: Final = tuple(level + 1 for prior, level in placed if field_mask(prior.writes) & touched)
not_before_readers: Final = tuple(level for prior, level in placed if field_mask(prior.reads) & writes)
return max((*after_writers, *not_before_readers), default=0)
def _place(
placed: tuple[tuple[Subscription, int], ...], subscription: Subscription
) -> tuple[tuple[Subscription, int], ...]:
return (*placed, (subscription, _level(placed, subscription)))
def _levels(entries: Sequence[Entry]) -> tuple[tuple[Entry, ...], ...]:
placed: Final = reduce(_place, (entry.subscription for entry in entries), ())
levels: Final = tuple(level for _, level in placed)
return tuple(_at_level(entries, levels, level) for level in range(max(levels, default=-1) + 1))
def _at_level(entries: Sequence[Entry], levels: tuple[int, ...], level: int) -> tuple[Entry, ...]:
return tuple(entry for entry, placed_at in zip(entries, levels, strict=True) if placed_at == level)
def plan_hook(entries: Sequence[Entry]) -> HookPlan:
if not entries:
return EMPTY_PLAN
blocking: Final = tuple(entry for entry in entries if _blocks(entry.subscription))
serial: Final = tuple(entry for entry in blocking if not entry.subscription.parallel)
parallel: Final = tuple(entry for entry in blocking if entry.subscription.parallel)
passive: Final = tuple(entry for entry in entries if not _blocks(entry.subscription))
return HookPlan(
stages=(*_levels(serial), *((parallel,) if parallel else ())),
inline=tuple(entry for entry in passive if entry.subscription.legacy),
awaited=tuple(entry for entry in passive if entry.subscription.awaited and not entry.subscription.legacy),
off_path=tuple(entry for entry in passive if not (entry.subscription.awaited or entry.subscription.legacy)),
reads=frozenset[Field]().union(*(entry.subscription.reads for entry in entries)),
)

View file

@ -0,0 +1,51 @@
from collections.abc import Callable, Mapping, Sequence
from functools import lru_cache
from types import MappingProxyType
from typing import Final
from litellm.integrations.custom_logger_v2.enums import Hook
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest
from litellm.integrations.custom_logger_v2.planner import Entry, HookPlan, plan_hook
DEFAULT_PLAN_CACHE_SIZE: Final = 256
def _hook_entries(manifests: tuple[CallbackManifest, ...], hook: Hook) -> tuple[Entry, ...]:
return tuple(
Entry(position, manifest, manifest.subscriptions[hook])
for position, manifest in enumerate(manifests)
if hook in manifest.subscriptions
)
class RegistrySnapshot:
__slots__ = ("entries", "manifests", "plan")
def __init__(self, manifests: tuple[CallbackManifest, ...], plan_cache_size: int) -> None:
self.manifests: Final = manifests
self.entries: Final[Mapping[Hook, tuple[Entry, ...]]] = MappingProxyType(
{hook: _hook_entries(manifests, hook) for hook in Hook}
)
self.plan: Final[Callable[[Hook, tuple[int, ...]], HookPlan]] = lru_cache(maxsize=plan_cache_size)(
self._build_plan
)
def _build_plan(self, hook: Hook, selected: tuple[int, ...]) -> HookPlan:
hook_entries: Final = self.entries[hook]
return plan_hook(tuple(hook_entries[index] for index in selected))
class CallbackRegistry:
def __init__(
self, manifests: Sequence[CallbackManifest] = (), plan_cache_size: int = DEFAULT_PLAN_CACHE_SIZE
) -> None:
self._plan_cache_size: Final = plan_cache_size
self.snapshot: RegistrySnapshot = RegistrySnapshot(tuple(manifests), plan_cache_size)
def register(self, manifest: CallbackManifest) -> None:
self.snapshot = RegistrySnapshot((*self.snapshot.manifests, manifest), self._plan_cache_size)
def unregister(self, logger: CustomLoggerV2) -> None:
kept: Final = tuple(manifest for manifest in self.snapshot.manifests if manifest.logger is not logger)
self.snapshot = RegistrySnapshot(kept, self._plan_cache_size)

View file

@ -0,0 +1,36 @@
from dataclasses import dataclass
from typing import Final
from litellm.integrations.custom_logger_v2.enums import Activation, OptOutBy
@dataclass(frozen=True, slots=True)
class Selection:
name: str
aliases: frozenset[str] = frozenset()
activation: Activation = Activation.default_on
opt_out_by: frozenset[OptOutBy] = frozenset({OptOutBy.admin})
honors_no_log: bool = False
@dataclass(frozen=True, slots=True)
class RequestSelection:
requested: frozenset[str] = frozenset()
admin_opt_outs: frozenset[str] = frozenset()
caller_opt_outs: frozenset[str] = frozenset()
no_log: bool = False
def _opted_out(selection: Selection, names: frozenset[str], request: RequestSelection) -> bool:
by_admin: Final = OptOutBy.admin in selection.opt_out_by and not names.isdisjoint(request.admin_opt_outs)
by_caller: Final = OptOutBy.caller in selection.opt_out_by and not names.isdisjoint(request.caller_opt_outs)
return by_admin or by_caller or (request.no_log and selection.honors_no_log)
def is_selected(selection: Selection, activation: Activation, request: RequestSelection) -> bool:
if activation is Activation.required:
return True
names: Final = selection.aliases | {selection.name}
if _opted_out(selection, names, request):
return False
return activation is Activation.default_on or not names.isdisjoint(request.requested)

View file

@ -0,0 +1,61 @@
import os
from collections.abc import Mapping
from typing import Final
from pydantic import BaseModel, ConfigDict, HttpUrl, TypeAdapter
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # Legacy params are untyped.
httpxSpecialProvider,
)
from litellm.types.integrations.customer_budget import CustomerBudgetEvent
class _Settings(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
endpoint: HttpUrl | None = None
headers: Mapping[str, str] | None = None
_URL: Final = TypeAdapter(HttpUrl)
_HEADERS: Final = TypeAdapter(Mapping[str, str])
class CustomerBudgetWebhook(CustomLogger):
def __init__(
self,
endpoint: str | None = None,
headers: Mapping[str, str] | None = None,
*,
async_httpx_client: AsyncHTTPHandler | None = None,
) -> None:
super().__init__() # pyright: ignore[reportUnknownMemberType] # Base logger kwargs are untyped.
self.endpoint: Final = str(_URL.validate_python(endpoint or os.environ.get("CUSTOMER_BUDGET_WEBHOOK_URL")))
self.headers: Final = {
"Content-Type": "application/json",
**_HEADERS.validate_json(os.environ.get("CUSTOMER_BUDGET_WEBHOOK_HEADERS", "{}")),
**(headers or {}),
}
self.async_httpx_client: Final = (
async_httpx_client
if async_httpx_client is not None
else get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
)
@staticmethod
def from_config(value: object) -> "CustomerBudgetWebhook":
settings: Final = _Settings.model_validate(value or {})
return CustomerBudgetWebhook(
endpoint=str(settings.endpoint) if settings.endpoint is not None else None,
headers=settings.headers,
)
async def async_on_customer_budget_event(self, event: CustomerBudgetEvent) -> None:
client: Final = self.async_httpx_client
response: Final = await client.post( # pyright: ignore[reportUnknownMemberType] # Legacy HTTP params are untyped.
url=self.endpoint, headers=dict(self.headers), json=event.model_dump(mode="json"), timeout=10
)
response.raise_for_status()

View file

@ -77,6 +77,11 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
prompt_data=prompt_data,
prompt_file=prompt_file,
prompt_id=registration_prompt_id,
ignore_prompt_manager_optional_params=(
litellm_params.ignore_prompt_manager_optional_params
if hasattr(litellm_params, "ignore_prompt_manager_optional_params")
else False
),
)
return dot_prompt_manager

View file

@ -54,9 +54,11 @@ class DotpromptManager(CustomPromptManagement):
prompt_file: str | None = None,
prompt_data: dict | str | None = None,
prompt_id: str | None = None,
ignore_prompt_manager_optional_params: bool | None = False,
):
import litellm
self.ignore_prompt_manager_optional_params = ignore_prompt_manager_optional_params
self.prompt_directory = prompt_directory or litellm.global_prompt_directory
# Support for JSON-based prompts stored in memory/database
if isinstance(prompt_data, str):

View file

@ -11,7 +11,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.constants import REDACTED_BY_LITELLM
from litellm.constants import FAILED_PROMPT_TOKEN_ESTIMATE_KEY, REDACTED_BY_LITELLM
from litellm.integrations._types.open_inference import ErrorAttributes
from litellm.integrations.arize import _utils
from litellm.integrations.langfuse.langfuse_otel_attributes import (
@ -19,8 +19,10 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
serialize_langfuse_datetime,
)
from litellm.integrations.opentelemetry import (
ESTIMATED_INPUT_TOKENS_ATTRIBUTE,
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
LITELLM_PROXY_REQUEST_SPAN_NAME,
USAGE_PROVIDER_STATUS_ATTRIBUTE,
OpenTelemetry,
OpenTelemetryConfig,
)
@ -378,9 +380,15 @@ class LangfuseOtelLogger(OpenTelemetry):
limit: Final = span_attribute_limit(span)
carried: Final = carried_keys(span)
new_keys: Final = tuple(key for key, _value in fields if key not in carried)
room: Final = (
len(new_keys) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0)
pending_usage: Final[frozenset[str]] = (
frozenset((USAGE_PROVIDER_STATUS_ATTRIBUTE, ESTIMATED_INPUT_TOKENS_ATTRIBUTE))
if isinstance(kwargs.get(FAILED_PROMPT_TOKEN_ESTIMATE_KEY), int)
else frozenset((USAGE_PROVIDER_STATUS_ATTRIBUTE,))
if FAILED_PROMPT_TOKEN_ESTIMATE_KEY in kwargs
else frozenset()
)
reserved: Final = _ERROR_ATTRIBUTES | pending_usage
room: Final = len(new_keys) if limit is None else max(limit - len(carried) - len(reserved - carried), 0)
written: Final = carried | frozenset(new_keys[:room])
for key, value in fields:
if key in written:

View file

@ -214,9 +214,13 @@ class ModelDeprecationEmailSender:
return True
async def run(self, llm_router: Router, *, now: datetime | None = None) -> int:
if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
return 0
if not await self.config.model_deprecations_enabled():
return 0
current: Final = now or datetime.now(timezone.utc)
settings: Final = await self.config.get_param("general_settings")
if settings is None or not _OptIn.model_validate(settings.param_value).enabled:
if not _OptIn.model_validate(settings.param_value if settings else {}).enabled:
return 0
base_url: Final = os.getenv("PROXY_BASE_URL", "")
if not base_url:
@ -253,7 +257,7 @@ class ModelDeprecationEmailSender:
async def run_scheduled_model_deprecation_emails() -> None:
"""Poll the email opt-in independently; a large owner mailing never delays Slack checks."""
"""Poll the email settings independently; a large owner mailing never delays Slack checks."""
from litellm._logging import verbose_proxy_logger
while True:

View file

@ -6,14 +6,14 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
from typing import TYPE_CHECKING, Any, ClassVar, Final, Protocol, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
from litellm.constants import FAILED_PROMPT_TOKEN_ESTIMATE_KEY, KUBERNETES_POD_ROUTING_KEY
from litellm.integrations._types.open_inference import (
OpenInferenceSpanKindValues,
)
@ -33,9 +33,11 @@ from litellm.integrations.otel.model.baggage import promoted_metadata
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.metadata import flatten_metadata
from litellm.integrations.otel.model.semconv import LiteLLM, Metric
from litellm.integrations.otel.model.utils import as_str
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
from litellm.integrations.otel.routing import routed_model_name, routing_decision_attributes
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
@ -140,6 +142,18 @@ def _response_field(response_obj: object, key: str) -> object:
return require_attributes(response_obj, _SupportsGet, "get").get(key)
def _semconv_response_model(
response_obj: object, response_model: object, model_group: str | None, deployment_model: str | None
) -> object:
hidden_params: Final = get_hidden_params(response_obj)
provider_model: Final = hidden_params.get("provider_response_model") if hidden_params is not None else None
if isinstance(provider_model, str) and provider_model:
return provider_model
if response_model and response_model == model_group and deployment_model:
return deployment_model
return response_model
def _billed_usage(response_obj: object, call_type: str | None, litellm_params: Mapping[str, object] | None) -> object:
if not response_obj or is_unbilled_non_inference_call_from_params(call_type, litellm_params, response_obj):
return None
@ -152,6 +166,7 @@ class _CacheHitView(TypedDict, total=False):
_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
_OPAQUE_VALUE: Final = TypeAdapter(object)
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
@ -166,6 +181,8 @@ HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE: Final = "http.response.status_code"
HTTP_ROUTE_ATTRIBUTE: Final = "http.route"
URL_PATH_ATTRIBUTE: Final = "url.path"
PREPROCESSING_DURATION_MS_ATTRIBUTE: Final = "litellm.preprocessing.duration_ms"
USAGE_PROVIDER_STATUS_ATTRIBUTE: Final = "litellm.usage.provider_status"
ESTIMATED_INPUT_TOKENS_ATTRIBUTE: Final = "litellm.usage.estimated_input_tokens"
TEAM_METADATA_ATTRIBUTE: Final = "litellm.team.metadata"
MODEL_GROUP_ATTRIBUTE: Final = "litellm.model_group"
PROVIDER_MODEL_ATTRIBUTE: Final = "litellm.provider.model"
@ -440,6 +457,8 @@ class OpenTelemetryConfig:
class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
wants_failed_prompt_token_estimate: ClassVar[bool] = True
def __init__(
self,
config: OpenTelemetryConfig | None = None,
@ -2225,6 +2244,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# Always create if no parent span exists (backward compatibility)
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
should_create_primary_span: Final = parent_otel_span is None or get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN")
failure_kwargs: Final = _JSON_OBJECT.validate_python(kwargs)
span = None
if should_create_primary_span:
@ -2239,6 +2259,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
span_kwargs["kind"] = self.span_kind.CLIENT
span = otel_tracer.start_span(**span_kwargs)
self.set_attributes(span, kwargs, response_obj)
self._set_failure_usage_attributes(span, failure_kwargs)
# Record exception information using OTEL standard method
span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs)
@ -2252,6 +2273,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# Note: parent_otel_span is guaranteed to be not None here
if parent_otel_span is not None and parent_otel_span.is_recording():
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._set_failure_usage_attributes(parent_otel_span, failure_kwargs)
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
@ -2271,6 +2293,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
):
parent_otel_span.end(end_time=self._to_ns(end_time))
def _set_failure_usage_attributes(self, span: Span, kwargs: Mapping[str, object]) -> None:
if FAILED_PROMPT_TOKEN_ESTIMATE_KEY not in kwargs:
return
self.safe_set_attribute(span=span, key=USAGE_PROVIDER_STATUS_ATTRIBUTE, value="unknown")
estimate: Final = kwargs[FAILED_PROMPT_TOKEN_ESTIMATE_KEY]
if isinstance(estimate, int):
self.safe_set_attribute(span=span, key=ESTIMATED_INPUT_TOKENS_ATTRIBUTE, value=estimate)
def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None:
"""
Record exception information on the span using OTEL standard methods.
@ -2521,11 +2551,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
#############################################
# The name of the LLM a request is being made to
if kwargs.get("model"):
request_model: Final = as_str(_OPAQUE_VALUE.validate_python(kwargs.get("model")))
if request_model:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_REQUEST_MODEL.value,
value=kwargs.get("model"),
value=request_model,
)
# The LLM request type
@ -2617,10 +2648,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# The model used to generate the response.
response_model: Final = routed_model_name(
_response_field(response_obj, "model"),
standard_logging_payload.get("model_group"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped
decision,
model_group: Final = as_str(standard_logging_payload.get("model_group")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped
routed_response_model: Final = routed_model_name(
_response_field(response_obj, "model"), model_group, decision
)
response_model: Final = (
_semconv_response_model(response_obj, routed_response_model, model_group, request_model)
if self._gen_ai_semconv_latest_experimental
else routed_response_model
)
if response_model:
self.safe_set_attribute(

View file

@ -89,23 +89,13 @@ shows as `redis.incr router_cooldowns` followed by `redis.ttl router_cooldowns`
way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue`,
`managed_files`, `mcp_servers`, ...), so a job tick renders `redis.set pod_lock`
rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every
module under `litellm/` and `enterprise/` that calls a shared cache or declares a
read or write on the request batch (`reserve_redis_batch_reads`,
`batch.mget`, `batch.set`, `batch.script`) and fails when
module under `litellm/` and `enterprise/` that calls a shared cache and fails when
one has no declared family, with the process-local `InMemoryCache` callers listed
as the only exemptions. A batch op carries the family that was active when it was
declared, so the routing prefetch armed before deployment selection
(`RoutingPrefetch.arm`) is `router_cooldowns` when only cooldown keys go out,
`router_usage` when only usage counters do, and `router_cooldowns_usage` when both
ride the same MGET, whichever pipeline or standalone read later settles it. A
per-request pipeline (`RedisBatch`) that carries ops of
one family is `"redis.pipeline auth_objects"`; one that carries several owners'
ops is `"redis.pipeline mixed"` with the sorted family list on
`litellm.redis.families` and the op count on `litellm.metadata.op_count` (an int,
never stringified). A cluster client cannot pipeline across slots, so there every
batch op settles on its own and one write-back of three auth objects shows as three
parallel `redis.set auth_objects` spans with the same caller, not one pipeline
span. Every `call_type` the Redis cache layer emits maps to a verb,
ride the same MGET, whichever pipeline or standalone read later settles it. Every `call_type` the Redis cache layer emits maps to a verb,
so the `{service} {call_type}` fallback is unreachable for Redis (a test asserts
it). Postgres helpers are `postgres.{verb} {table}`
(`"postgres.select LiteLLM_VerificationToken"`, `"postgres.update LiteLLM_TeamTable"`,
@ -130,12 +120,9 @@ Redis it is also `db.operation.name`), the target lands on
`litellm.service.target`, and the litellm call chain that issued the call
(`_retrieve_from_cache <- _async_get_cache`) travels as
`ServiceLoggerPayload.caller` onto `litellm.service.caller`, with the forwarding
frames (cache facades, circuit-breaker guards, batch retry wrappers, the native
execution's `lifecycle`/`streams` drivers) skipped so it names the code that wanted
the call. A call whose own frames are all forwarders
(a batch op settled in a task of its own, on a cluster client or a NOSCRIPT retry)
reports the chain its declaring code captured and threaded through
`service_caller(...)`, never the forwarders, and `unknown` when there is none.
frames (cache facades, circuit-breaker guards, the native execution's
`lifecycle`/`streams` drivers) skipped so it names the code that wanted the call,
and `unknown` when its own frames are all forwarders.
The cache key itself is never on the span: it is unbounded and carries key hashes
and session ids, and the span is already named by key family. Like every other
span they parent to the

View file

@ -20,14 +20,27 @@ from litellm.integrations.otel.plumbing.context import request_root_span
if TYPE_CHECKING:
from opentelemetry.trace import Span
from litellm.caching.dual_cache import DualCache
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.types.utils import ModelResponseStream
from litellm.types.utils import CallTypesLiteral, ModelResponseStream
class LangfuseOpenTelemetryV2(OpenTelemetryV2):
"""Stamps the caller's trace controls (name, user, session, tags) on the request. Langfuse reads them off
the root observation, and the proxy's root span is still recording when the LLM call starts."""
async def async_pre_call_hook(
self,
user_api_key_dict: "UserAPIKeyAuth",
cache: "DualCache",
data: dict,
call_type: "CallTypesLiteral",
) -> dict:
# Through v1.111.0 the proxy's leaf-only override check never ran OpenTelemetryV2's pre-call identity
# seeding for the Langfuse subclasses. The MRO-aware check that replaced it would, so this no-op keeps them
# as they were.
return data
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
root: Final = request_root_span()
if root is not None and root.is_recording():

View file

@ -10,7 +10,6 @@ table: one lambda per mapping operation, applied against the typed span data.
from collections.abc import Callable
from typing import Final
from litellm._internal_context import REDIS_FAMILIES_METADATA_KEY
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import (
MAX_TOOL_DEFINITION_ATTRS_PER_SPAN,
@ -200,12 +199,5 @@ class GenAIMapper:
# semconv naming the server it reached. Internal services (router, budget
# jobs, …) have no db.system, so they get only the litellm.service.* keys.
attrs.update(db_span_attributes(data.service_name, data.call_type, postgres_operation(data)))
attrs.update(
{
LiteLLM.REDIS_FAMILIES
if key == REDIS_FAMILIES_METADATA_KEY
else f"{LiteLLM.METADATA_PREFIX}{key}": value
for key, value in data.event_metadata.items()
}
)
attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()})
return attrs

View file

@ -334,8 +334,6 @@ class LiteLLM:
SERVICE_CALL_TYPE: Final = "litellm.service.call_type"
SERVICE_CALLER: Final = "litellm.service.caller"
SERVICE_TARGET: Final = "litellm.service.target"
# The sorted, comma-joined key families one Redis pipeline carried ops for; bounded, unlike the keys.
REDIS_FAMILIES: Final = "litellm.redis.families"
PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms"
# The logical name of the MCP server a tool call was routed to. There is no
# semconv key for an MCP server's *name* (the convention uses ``server.address``

View file

@ -263,6 +263,10 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_Alert",
"LiteLLM_PerformanceAlertState",
"LiteLLM_BudgetTable",
"LiteLLM_CustomerOrganizationTable",
"LiteLLM_CustomerTeamTable",
"LiteLLM_CustomerMemberTable",
"LiteLLM_CustomerBudgetIncreaseRequest",
"LiteLLM_CredentialsTable",
"LiteLLM_ProxyModelTable",
"LiteLLM_AgentsTable",
@ -305,9 +309,11 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_DailyUserSpend",
"LiteLLM_DailyGlobalSpend",
"LiteLLM_DailyOrganizationSpend",
"LiteLLM_DailyCustomerMemberSpend",
"LiteLLM_DailyEndUserSpend",
"LiteLLM_DailyAgentSpend",
"LiteLLM_DailyTeamSpend",
"LiteLLM_DailyCustomerMemberSpend",
"LiteLLM_DailyTagSpend",
"LiteLLM_ProxyWorkerHeartbeat",
"LiteLLM_CronJob",

View file

@ -0,0 +1,183 @@
from __future__ import annotations
import asyncio
import hashlib
import html
import os
from collections.abc import Awaitable, Callable, Sequence
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Final
from pydantic import AliasPath, BaseModel, ConfigDict, Field
from litellm._logging import verbose_proxy_logger
from litellm.models.alert import AlertSummary
from litellm.repositories.alert_repository import AlertRepository
from litellm.repositories.config_repository import ConfigRepository
if TYPE_CHECKING:
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
class _EmailSettings(BaseModel):
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
enabled: bool = Field(False, strict=True, validation_alias=AliasPath("email_settings", "Performance Alerts"))
class _Delivery(BaseModel):
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
revision: int = 1
cursor: int
first_window_start: datetime
next_send_at: datetime
lease_until: datetime
def _description(alert: AlertSummary) -> str:
labels: Final = {
"spend": "Spend spike",
"traffic": "Traffic spike",
"latency": "Slower responses",
"reliability": "More failed requests",
}
metric: Final = alert.evidence
def amount(value: float) -> str:
match metric.unit:
case "usd":
return f"${value:,.2f}"
case "ratio":
return f"{value:.1%}"
case "milliseconds":
return f"{value:,.0f} ms"
case "requests":
return f"{value:,.0f} requests"
return f"{labels[alert.type]} for {alert.resource_name}: {amount(metric.value)} vs {amount(metric.baseline)}"
def slack_text(alerts: Sequence[AlertSummary], base_url: str) -> str:
rows: Final = tuple(
f"<{base_url}/ui/alerts?alert={alert.id}|{html.escape(_description(alert), quote=False)}>"
for alert in alerts[:5]
)
return "New performance alerts\n\n" + "\n".join(rows) + f"\n\n<{base_url}/ui/alerts|See all alerts>"
def email_body(alerts: Sequence[AlertSummary], base_url: str) -> tuple[str, str]:
rows: Final = tuple(
f'<li><a href="{html.escape(base_url, quote=True)}/ui/alerts?alert={alert.id}">{html.escape(_description(alert))}</a></li>'
for alert in alerts[:5]
)
text: Final = (
"New performance alerts\n\n"
+ "\n".join(f"{_description(alert)}\n{base_url}/ui/alerts?alert={alert.id}" for alert in alerts[:5])
+ f"\n\nSee all alerts: {base_url}/ui/alerts"
)
return (
f'<h2>New performance alerts</h2><ul>{"".join(rows)}</ul><p><a href="{html.escape(base_url, quote=True)}/ui/alerts">See all alerts</a></p>',
text,
)
class PerformanceAlertNotifier:
def __init__(self, config: ConfigRepository, alerts: AlertRepository) -> None:
self.config: Final = config
self.alerts: Final = alerts
async def deliver(
self,
destination: str,
send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]],
now: datetime,
) -> None:
key: Final = "performance_alert_digest:" + hashlib.sha256(destination.encode()).hexdigest()
row: Final = await self.config.get_param(key)
previous: Final = (
_Delivery.model_validate(row.param_value)
if row
else _Delivery(
revision=0,
cursor=0,
first_window_start=now - timedelta(days=1),
next_send_at=now,
lease_until=now,
)
)
if previous.next_send_at > now or previous.lease_until > now:
return
through: Final = await self.alerts.latest_id()
found: Final = await self.alerts.summaries(previous.cursor, through)
alerts: Final = (
found
if previous.cursor
else tuple(alert for alert in found if alert.created_at >= previous.first_window_start)
)
claim: Final = previous.model_copy(
update={
"revision": previous.revision + 1,
"lease_until": now + timedelta(minutes=2),
"next_send_at": now + timedelta(days=1) if alerts else now,
}
)
if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), previous.revision):
return
if alerts and not await send(alerts, f"{key}-{through}"):
return
sent: Final = claim.model_copy(
update={
"revision": claim.revision + 1,
"cursor": through,
"next_send_at": now + timedelta(days=1) if alerts else now,
"lease_until": now,
}
)
if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision):
verbose_proxy_logger.warning("Could not record performance alert summary delivery")
async def run(self, slack: SlackAlerting | None, *, now: datetime | None = None) -> None:
from litellm.proxy.utils import send_email
base_url: Final = os.getenv("PROXY_BASE_URL", "").rstrip("/")
if not base_url:
return
if slack is not None:
for webhook in slack.performance_alert_webhooks():
async def send_slack(alerts: Sequence[AlertSummary], receipt: str, webhook: str = webhook) -> bool:
return await slack.send_performance_alert_summary(webhook, slack_text(alerts, base_url))
await self._try_deliver("slack:" + webhook, send_slack, now or datetime.now(timezone.utc))
if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
return
settings: Final = await self.config.get_param("general_settings")
if not _EmailSettings.model_validate(settings.param_value if settings else {}).enabled:
return
after: str = "" # rebind-ok: advances the bounded recipient page
while recipients := await self.alerts.recipients(after):
for recipient in recipients:
async def send_mail(
alerts: Sequence[AlertSummary], receipt: str, address: str = recipient.email
) -> bool:
body, text = email_body(alerts, base_url)
return await send_email(
receiver_email=address,
subject="New performance alerts",
html=body,
text=text,
message_id=f"<{receipt}@alerts.litellm.ai>",
)
await self._try_deliver("email:" + recipient.email, send_mail, now or datetime.now(timezone.utc))
after = recipients[-1].user_id
await asyncio.sleep(0.1)
async def _try_deliver(
self, destination: str, send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], now: datetime
) -> None:
try:
await asyncio.wait_for(self.deliver(destination, send, now), timeout=60)
except Exception as exc: # noqa: BLE001 # one unavailable destination must not stop the other recipients
verbose_proxy_logger.warning("Performance alert summary delivery failed (%s)", type(exc).__name__)

View file

@ -1061,18 +1061,39 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_deployment_latency_per_output_token"),
)
self.litellm_deployment_successful_fallbacks = self._counter_factory(
self.litellm_deployment_successful_fallbacks: _MetricLike = self._counter_factory(
"litellm_deployment_successful_fallbacks",
"LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model",
labelnames=self.get_labels_for_metric("litellm_deployment_successful_fallbacks"),
)
self.litellm_deployment_failed_fallbacks = self._counter_factory(
self.litellm_deployment_failed_fallbacks: _MetricLike = self._counter_factory(
"litellm_deployment_failed_fallbacks",
"LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model",
labelnames=self.get_labels_for_metric("litellm_deployment_failed_fallbacks"),
)
self.litellm_dynamic_limiter_share_tokens: _MetricLike = self._gauge_factory(
"litellm_dynamic_limiter_share_tokens",
"Dynamic rate limiter: the team's share of the model's TPM, as of the team's last settled request",
labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_share_tokens"),
multiprocess_mode="livemostrecent",
)
self.litellm_dynamic_limiter_borrowed_tokens: _MetricLike = self._gauge_factory(
"litellm_dynamic_limiter_borrowed_tokens",
"Dynamic rate limiter: tokens the team has used above its share this window, 0 while under its share",
labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_borrowed_tokens"),
multiprocess_mode="livemostrecent",
)
self.litellm_dynamic_limiter_throttled_total: _MetricLike = self._counter_factory(
"litellm_dynamic_limiter_throttled_total",
"Dynamic rate limiter: requests refused per team, model and reason, "
"the reason being the value of the response's rate_limited_by header",
labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_throttled_total"),
)
# Callback Logging Failure Metrics
self.litellm_callback_logging_failures_metric = self._counter_factory(
name="litellm_callback_logging_failures_metric",
@ -4172,6 +4193,66 @@ class PrometheusLogger(CustomLogger):
sanitize_prometheus_label_value(exception_status),
).inc()
def set_dynamic_limiter_share(
self,
model_group: str,
team: str,
team_alias: str | None,
priority: str,
share_tokens: int,
borrowed_tokens: int,
) -> None:
enum_values: Final = UserAPIKeyLabelValues(
model_group=model_group, team=team, team_alias=team_alias, priority=priority
)
self._set_labeled_gauge(
self.litellm_dynamic_limiter_share_tokens, "litellm_dynamic_limiter_share_tokens", enum_values, share_tokens
)
self._set_labeled_gauge(
self.litellm_dynamic_limiter_borrowed_tokens,
"litellm_dynamic_limiter_borrowed_tokens",
enum_values,
borrowed_tokens,
)
def _set_labeled_gauge(
self,
gauge: _MetricLike,
metric_name: DEFINED_PROMETHEUS_METRICS,
enum_values: UserAPIKeyLabelValues,
value: float,
) -> None:
from prometheus_client import Gauge
labels: Final = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name=metric_name), enum_values=enum_values
)
series: Final = gauge.labels(**labels)
if isinstance(series, Gauge):
series.set(value)
def increment_dynamic_limiter_throttled(
self,
model_group: str,
team: str,
team_alias: str | None,
priority: str,
rate_limit_type: str,
reason: str,
) -> None:
self._inc_labeled_counter(
self.litellm_dynamic_limiter_throttled_total,
"litellm_dynamic_limiter_throttled_total",
UserAPIKeyLabelValues(
model_group=model_group,
team=team,
team_alias=team_alias,
priority=priority,
rate_limit_type=rate_limit_type,
reason=reason,
),
)
def increment_callback_logging_failure(
self,
callback_name: str,
@ -5221,7 +5302,7 @@ def prometheus_label_factory(
tag: str | None = None,
*,
label_context: PrometheusLabelFactoryContext | None = None,
) -> dict:
) -> dict[str, str | None]:
"""
Returns a dictionary of label + values for prometheus.

View file

@ -5852,6 +5852,17 @@
"legendFormat": "self",
"range": true,
"refId": "M"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_daily_customer_member_spend_update_queue_latency_bucket[$__rate_interval])) by (le))",
"legendFormat": "redis_daily_customer_member_spend_update_queue",
"range": true,
"refId": "N"
}
],
"title": "Service latency p95 (litellm_<service>_latency)",
@ -6052,6 +6063,17 @@
"legendFormat": "self",
"range": true,
"refId": "N"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(litellm_redis_daily_customer_member_spend_update_queue_total_requests_total[$__rate_interval]))",
"legendFormat": "redis_daily_customer_member_spend_update_queue",
"range": true,
"refId": "O"
}
],
"title": "Service request rate (litellm_<service>_total_requests)",
@ -6252,6 +6274,17 @@
"legendFormat": "self / {{error_class}}",
"range": true,
"refId": "N"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(litellm_redis_daily_customer_member_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)",
"legendFormat": "redis_daily_customer_member_spend_update_queue / {{error_class}}",
"range": true,
"refId": "O"
}
],
"title": "Service failure rate (litellm_<service>_failed_requests)",
@ -6644,6 +6677,190 @@
],
"title": "litellm_lens_export_rows_written and litellm_lens_export_rows_dropped rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 456
},
"id": 117,
"panels": [],
"title": "Dynamic rate limiter (needs dynamic_rate_limiter_v3 and prometheus callbacks)",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "The team's token share on the model this window, per team and model group",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 457
},
"id": 118,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "litellm_dynamic_limiter_share_tokens",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "litellm_dynamic_limiter_share_tokens",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Tokens the team has used above its share this window, 0 while under it",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 457
},
"id": 119,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "litellm_dynamic_limiter_borrowed_tokens",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "litellm_dynamic_limiter_borrowed_tokens",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests the dynamic limiter refused per second, by team, model and the 429's rate_limited_by reason",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 465
},
"id": 120,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (team, team_alias, model_group, reason) (rate(litellm_dynamic_limiter_throttled_total[5m]))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "litellm_dynamic_limiter_throttled_total",
"type": "timeseries"
}
],
"preload": false,

View file

@ -33,6 +33,8 @@ def resolve_prompt_manager_ignore_flags(
class PromptManagementBase(ABC):
ignore_prompt_manager_optional_params: bool | None = False
@property
@abstractmethod
def integration_name(self) -> str:
@ -198,7 +200,8 @@ class PromptManagementBase(ABC):
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
prompt_spec=prompt_spec,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params
or self.ignore_prompt_manager_optional_params,
)
return self.post_compile_prompt_processing(
prompt_template=prompt_template,
@ -245,7 +248,8 @@ class PromptManagementBase(ABC):
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
prompt_spec=prompt_spec,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params
or self.ignore_prompt_manager_optional_params,
)
return self.post_compile_prompt_processing(
prompt_template=prompt_template,

View file

@ -300,6 +300,26 @@ def create(
)
else:
custom_llm_provider = custom_llm_provider or "gemini"
if agent and model and model != agent:
raise litellm.BadRequestError(
message=(
f"agent '{agent}' can only run on a deployment of that agent, but the model is '{model}'. "
"Configure a deployment whose model is the agent and send its model group."
),
model=model,
llm_provider=custom_llm_provider,
)
if extra_body and (
("agent" in extra_body and extra_body["agent"] != agent) or (agent and "model" in extra_body)
):
raise litellm.BadRequestError(
message=(
"extra_body cannot set a different agent or add a model to an agent interaction. "
"Send the agent and model as top-level fields."
),
model=model,
llm_provider=custom_llm_provider,
)
interactions_api_config: Final = get_provider_interactions_api_config(
provider=custom_llm_provider,
@ -340,7 +360,7 @@ def create(
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
model=agent or model,
optional_params=dict(optional_params),
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=custom_llm_provider,

View file

@ -1,3 +1,11 @@
## Compat names
- A change here that removes a name outside code can import keeps the name only when one of these holds
- It has no leading underscore and shipped in a stable release
- It is private, code outside this repo calls it, and the compat keeps that code doing what it did
- A private name whose old behavior cannot be kept stays removed. An `ImportError` names what changed, while a no-op that answers wrongly hides it
- A vendored copy of litellm in another repo is not a caller
## get_model_cost_map.py
- This module decides when to fetch and install. Fetching, parsing, integrity checks and alias expansion run in Rust
@ -9,6 +17,8 @@
- `utils._register_on_current_map` reruns the whole pass on the new table after `NativeTableSwapped`, and its last attempt passes `rows=None`, so no entry is lost
- Compat names exist only so code written against main keeps working. New code never calls them, and they never gain behavior of their own
- `_finalize_model_cost_map`, use `model_cost_inputs.install`
- `git_blob_id`, which Rust computes for `source_revision`
- `FALLBACK_GENERALIZATIONS_KEY`, `RESERVED_TOP_LEVEL_KEYS` and `RETRYABLE_FETCH_STATUS_CODES`, public constants. Rust owns the retry rule
- `GetModelCostMap.validate_model_cost_map` and `_check_*`, since Rust checks fetched documents
- `ModelCostMapReloaded.model_cost_map`, use `GetModelCostMap.loaded_model_cost_map()`
- the `sleep`, `rng` and `client` parameters, which are ignored with a `DeprecationWarning`
@ -25,3 +35,4 @@
- `utils._invalidate_model_cost_lowercase_map` forwards to `changed_in_place`
- `utils.get_model_cost_mutation_generation` forwards to `model_cost_version().writes`
- `utils._model_cost_lowercase_map` is always `None`
- `utils._runtime_registered_model_cost` is always empty, since kept registrations live in `model_cost_inputs`

View file

@ -141,6 +141,13 @@ def without(
return {key: value for key, value in mapping.items() if key not in keys}
def only(
mapping: Mapping[_Key, _Value], *keys: object
) -> dict[_Key, _Value]: # mutable-ok: a fresh copy the caller owns
"""``mapping`` with only ``keys``, as a new dict in the same order; ``mapping`` is unchanged."""
return {key: value for key, value in mapping.items() if key in keys}
def decoded_json_object(value: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(value)
if mapping is not None:

View file

@ -167,6 +167,15 @@ def _shift_months(moment: datetime, months: int) -> datetime:
return moment.replace(year=year, month=month, day=min(moment.day, get_last_day_of_month(year, month)))
def is_annual_budget_duration(duration: str) -> bool:
return _parse_duration(normalize_duration(duration)) == (12, "mo")
def get_annual_budget_window_start(reset_at: datetime, timezone_str: str) -> datetime:
boundary: Final = _setup_timezone(reset_at, timezone_str)[0]
return _shift_months(boundary, -12).astimezone(timezone.utc)
def get_budget_window_start(duration: str, reset_at: datetime) -> datetime:
"""Start of the budget period that ends at `reset_at`, under the same rules
`get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar
@ -220,8 +229,12 @@ def _parse_duration(duration: str) -> tuple[int | None, str | None]:
if not match:
return None, None
value, unit = match.groups()
return int(value), unit
value: Final = int(match.group(1))
spelled_unit: Final = match.group(2)
unit: Final = _extract_from_regex(duration)[1] or spelled_unit
if unit == "mo" and spelled_unit != "mo" and value not in (1, 12):
return value, spelled_unit
return value, unit
def _apply_time_of_day(dt: datetime, reset_time_of_day: time) -> datetime:
@ -390,14 +403,22 @@ def _handle_month_reset(
Args:
current_time: Current datetime
base_midnight: Midnight of current day
value: Number of months (currently only supports 1 month resets)
value: Number of months, 1 for monthly or 12 for annual resets
reset_time_of_day: Wall-clock time the reset lands on
Returns:
datetime: First day of the next reset month at `reset_time_of_day`
"""
if value == 12:
first_of_year: Final = base_midnight.replace(month=1, day=1)
annual_candidate: Final = _apply_time_of_day(first_of_year, reset_time_of_day)
return (
annual_candidate.replace(year=annual_candidate.year + 1)
if annual_candidate <= current_time
else annual_candidate
)
if value != 1:
raise ValueError("Monthly resets currently only support 1 month intervals")
raise ValueError("Calendar resets support 1 month or 12 month intervals")
first_of_this_month: Final = base_midnight.replace(day=1)
candidate: Final = _apply_time_of_day(first_of_this_month, reset_time_of_day)

View file

@ -8,9 +8,11 @@ Disable remote fetching entirely:
export LITELLM_LOCAL_BLOG_POSTS=True
"""
import asyncio
import os
import time
import xml.etree.ElementTree as ET
from collections.abc import Awaitable, Callable
from email.utils import parsedate_to_datetime
from importlib.resources import files
from typing import Final
@ -78,6 +80,7 @@ class GetBlogPosts:
_cached_posts: tuple[BlogPost, ...] | None = None
_last_fetch_time: float = 0.0
_refresh_lock: tuple[asyncio.AbstractEventLoop, asyncio.Lock] | None = None
@staticmethod
def load_local_blog_posts() -> tuple[BlogPost, ...]:
@ -98,7 +101,7 @@ class GetBlogPosts:
return response.text
@staticmethod
def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]:
def parse_rss_to_posts(xml_text: str | bytes, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]:
"""
Parse RSS XML into blog posts.
@ -147,14 +150,52 @@ class GetBlogPosts:
url,
str(e),
)
return cls.load_local_blog_posts()
return cls._cache_posts(cls.load_local_blog_posts())
if not cls.validate_blog_posts(posts):
return cls._cache_posts(cls.load_local_blog_posts())
return cls._cache_posts(posts)
@classmethod
def _cache_posts(cls, posts: tuple[BlogPost, ...]) -> tuple[BlogPost, ...]:
cls._cached_posts = posts
cls._last_fetch_time = time.time()
return posts
@classmethod
async def aget_blog_posts(cls, url: str, fetch: Callable[[str], Awaitable[bytes]]) -> tuple[BlogPost, ...]:
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
return cls.load_local_blog_posts()
cls._cached_posts = posts
cls._last_fetch_time = now
return posts
cached: Final = cls._cached_posts
if cached is not None and (time.time() - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS:
return cached
loop: Final = asyncio.get_running_loop()
state: Final = cls._refresh_lock
lock: Final = state[1] if state is not None and state[0] is loop else asyncio.Lock()
cls._refresh_lock = (loop, lock)
async with lock:
refreshed: Final = cls._cached_posts
if refreshed is not None and (time.time() - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS:
return refreshed
return await cls._refresh_blog_posts(url, fetch)
@classmethod
async def _refresh_blog_posts(cls, url: str, fetch: Callable[[str], Awaitable[bytes]]) -> tuple[BlogPost, ...]:
try:
xml_text: Final = await fetch(url)
posts: Final = cls.parse_rss_to_posts(xml_text)
except (httpx.HTTPError, ET.ParseError, ValueError) as e:
verbose_logger.warning(
"LiteLLM: Failed to fetch blog posts from %s: %s. Falling back to local backup.", url, str(e)
)
return cls._cache_posts(cls.load_local_blog_posts())
if not cls.validate_blog_posts(posts):
return cls._cache_posts(cls.load_local_blog_posts())
return cls._cache_posts(posts)
def get_blog_posts(url: str) -> tuple[BlogPost, ...]:

View file

@ -9,6 +9,7 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
```
"""
import hashlib
import json
import os
import sys
@ -51,6 +52,9 @@ from litellm.rust_bridge._native import (
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY})
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0
MODEL_COST_MAP_FETCH_MIN_DELAY_SECONDS: Final = 2.0
@ -456,18 +460,28 @@ def get_model_cost_map(
return _rows_with_rules_set(installed)
def registration_rows(body: bytes) -> dict[str, dict[str, object]]: # mutable-ok: register_model merges these rows
"""The rows of a fetched ``register_model`` document, without its fallback rules block. Raises ``ValueError`` on
a body that is not a JSON object of rows."""
document: Final = _CATALOG_ADAPTER.validate_python(json.loads(body))
return {key: entry for key, entry in document.items() if key != FALLBACK_GENERALIZATIONS_KEY}
def fetch_registration_document(url: str) -> dict[str, dict[str, object]] | None:
"""The document behind ``register_model(url)``, fetched in one attempt, or ``None`` when the fetch failed."""
"""The document behind ``register_model(url)``, fetched in one attempt. ``None`` when offline mode forbids the
fetch or it failed."""
if is_offline_mode():
verbose_logger.warning("LiteLLM: register_model did not fetch %s because LITELLM_OFFLINE_MODE is set", url)
return None
fetched: Final = cost_map_source(url, use_http_settings=False).fetch_blocking(fetch_policy(5, 1))
if isinstance(fetched, NativeFetchFailed):
verbose_logger.warning("LiteLLM: register_model could not fetch %s: %s", url, fetched.reason)
return None
try:
document: Final = _CATALOG_ADAPTER.validate_json(fetched.body)
return registration_rows(fetched.body)
except ValueError as e:
verbose_logger.warning("LiteLLM: register_model got an invalid cost map from %s: %s", url, e)
return None
return {key: entry for key, entry in document.items() if key != "fallback_generalizations"}
def _finalize_model_cost_map( # pyright: ignore[reportUnusedFunction] # main's private name, downstream code calls it
@ -478,15 +492,17 @@ def _finalize_model_cost_map( # pyright: ignore[reportUnusedFunction] # main's
case NativeRejected(reason=reason):
raise ValueError(reason)
case installed:
model_cost.pop("fallback_generalizations", None)
model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None)
return _rows_with_rules_set(installed)
_RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
def git_blob_id(body: bytes) -> str:
"""Compat for main's callers. ``git hash-object`` of the bytes, which ``source_revision`` reports."""
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
def _count_model_entries(model_cost: Mapping[str, object]) -> int:
return sum(1 for key in model_cost if key not in _RESERVED_TOP_LEVEL_KEYS)
return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS)
@lru_cache(maxsize=1)

View file

@ -44,16 +44,16 @@ BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
ITPM_COUNTER_KEY: Final = "_litellm_itpm_counter"
OTPM_COUNTER_KEY: Final = "_litellm_otpm_counter"
IO_TOKEN_ATTEMPT_KEY: Final = "_litellm_io_token_attempt"
IO_TOKEN_RESERVATION_ID_KEY: Final = "_litellm_io_token_reservation_id"
IO_TOKEN_RESERVATION_KEYS: Final = frozenset(
{
ITPM_RESERVED_KEY,
OTPM_RESERVED_KEY,
ITPM_CACHE_KEY,
OTPM_CACHE_KEY,
ITPM_COUNTER_KEY,
OTPM_COUNTER_KEY,
IO_TOKEN_ATTEMPT_KEY,
IO_TOKEN_RESERVATION_ID_KEY,
}

File diff suppressed because it is too large Load diff

View file

@ -428,6 +428,9 @@ class LoggingCallbackManager:
_get_all_callbacks = get_all_callbacks
def get_failure_callbacks(self) -> tuple[CustomLogger | Callable[..., object] | str, ...]:
return (*litellm.failure_callback, *litellm._async_failure_callback)
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
"""
Remove a callback object from every callback list it may have been
@ -489,7 +492,7 @@ class LoggingCallbackManager:
"""
# Get callback lists
success_callbacks: Final = set(litellm.success_callback + litellm._async_success_callback)
failure_callbacks: Final = set(litellm.failure_callback + litellm._async_failure_callback)
failure_callbacks: Final = set(self.get_failure_callbacks())
general_callbacks: Final = set(litellm.callbacks)
# Get all unique callbacks

View file

@ -318,6 +318,7 @@ class _BodyPrefixReader:
headers: Mapping[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
) -> Response:
with self.client.stream(
"GET",
@ -342,6 +343,7 @@ class _ImageBodyReader:
headers: Mapping[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
) -> Response:
with self.client.stream(
"GET",

View file

@ -14,11 +14,13 @@ from litellm.rust_bridge import diagnostics
REDACTED: Final = "REDACTED"
_PEM_PRIVATE_KEY_BLOCK: Final = r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----"
def _build_secret_patterns() -> "re.Pattern[str]":
patterns: Final[list[str]] = [
# PEM private key / certificate blocks
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
_PEM_PRIVATE_KEY_BLOCK,
# GCP OAuth2 access tokens (ya29.*)
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# Credential %s formatting (space separator, no key= prefix)
@ -100,6 +102,57 @@ def redact_string(value: str) -> str:
return diagnostics.run(lambda native: native.redact_text(value), lambda: python_redact_string(value))
_TOKEN_SHAPES: Final = (
r"ya29\.[A-Za-z0-9_.~+/-]{20,}",
r"(?:AKIA|ASIA)[0-9A-Z]{16}\b",
rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}",
r"(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{24,}",
r"AIza[0-9A-Za-z\-_]{35}",
r"dapi[0-9a-f]{32}\b",
r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}",
r"github_pat_[A-Za-z0-9_]{82}",
r"glpat-[A-Za-z0-9\-_]{20,}",
r"lin_api_[A-Za-z0-9]{40}\b",
r"xox[abposr]-[A-Za-z0-9-]{10,}",
r"hf_[A-Za-z0-9]{34,}",
r"npm_[A-Za-z0-9]{36}\b",
r"pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,}",
r"SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}(?![A-Za-z0-9_-])",
r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+",
)
# Tokens start at a word boundary or right after a JSON escape like "\n", since tool output is often serialized JSON
_TOKEN_START: Final = r"(?:(?<!\w)|(?<=\\[nrt])|(?<=\\u[0-9A-Fa-f]{4}))"
_TOKEN_PEM_PRIVATE_KEY_BLOCK: Final = (
r"-----BEGIN(?:[A-Z ]|-(?!----))*?PRIVATE KEY-----"
r"(?:[^-]|-(?!----))*?-----END(?:[A-Z ]|-(?!----))*?PRIVATE KEY-----"
)
_NON_JWT_TOKEN_RE: Final = re.compile(
f"{_TOKEN_PEM_PRIVATE_KEY_BLOCK}|{_TOKEN_START}(?:{'|'.join(_TOKEN_SHAPES[:-1])})"
)
_DOTTED_TOKEN: Final = f"{_TOKEN_START}(?:{_TOKEN_SHAPES[0]}|{_TOKEN_SHAPES[-2]})"
# Consume failed JWT headers once, including dotted keys and their boundary so the second pass sees them whole.
_UNMATCHED_JWT: Final = (
rf"(?P<unmatched_jwt>eyJ(?:(?!-----BEGIN|{_DOTTED_TOKEN})[A-Za-z0-9_-])*"
rf"(?:{_DOTTED_TOKEN})?[^A-Za-z0-9_-]?)"
)
_TOKEN_RE: Final = re.compile(f"{_NON_JWT_TOKEN_RE.pattern}|{_TOKEN_START}(?:{_TOKEN_SHAPES[-1]}|{_UNMATCHED_JWT})")
def redact_tokens(value: str, replacement: str = REDACTED) -> str:
"""Replace substrings shaped like a credential (provider key prefixes, JWTs, PEM private keys).
Unlike redact_string() this never matches on surrounding words such as ``password:`` or ``Basic``,
so it is safe to run over free-form prompt and tool content.
"""
def replace(match: re.Match[str]) -> str:
if match.lastgroup == "unmatched_jwt":
return _NON_JWT_TOKEN_RE.sub(replacement, match.group())
return match.expand(replacement)
return _TOKEN_RE.sub(replace, value)
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
_WINDOWS_DRIVE_PATH: Final = r"[A-Za-z]:\\[^\s'\"\)\]}>,]+"
_PRIVATE_OR_LOOPBACK_IPV4: Final = (

View file

@ -17,7 +17,7 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from pydantic import TypeAdapter
from pydantic import BaseModel, TypeAdapter
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
@ -31,6 +31,7 @@ from litellm.constants import (
DEFAULT_IMAGE_HEIGHT,
DEFAULT_IMAGE_TOKEN_COUNT,
DEFAULT_IMAGE_WIDTH,
DEFAULT_MAX_RECURSE_DEPTH,
MAX_IMAGE_URL_DOWNLOAD_SIZE_MB,
MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES,
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
@ -47,7 +48,7 @@ from litellm.constants import (
TOKEN_COUNTER_MAX_EXACT_CHARS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
ChatTemplateToolSection,
chat_template_tool_section,
@ -928,7 +929,6 @@ def token_counter(
if litellm.disable_token_counter is True:
return 0
verbose_logger.debug("messages in token_counter: %s, text in token_counter: %s", messages, text)
if text is not None and messages is not None:
raise ValueError("text and messages cannot both be set")
if use_default_image_token_count is None:
@ -1758,31 +1758,72 @@ def _count_file_tokens(
)
_ANTHROPIC_SERVER_TOOL_FIELDS: Final = MappingProxyType(
{
"server_tool_use": ("name", "input"),
"mcp_tool_use": ("name", "input"),
"web_search_tool_result": ("content",),
"web_fetch_tool_result": ("content",),
"code_execution_tool_result": ("content",),
"bash_code_execution_tool_result": ("content",),
"text_editor_code_execution_tool_result": ("content",),
"mcp_tool_result": ("content",),
"web_search_result": ("url", "title", "snippet"),
"web_fetch_result": ("url", "content"),
"code_execution_result": ("stdout", "stderr", "content"),
"bash_code_execution_result": ("stdout", "stderr", "content"),
"text_editor_code_execution_view_result": ("content",),
"text_editor_code_execution_str_replace_result": ("lines",),
"text_editor_code_execution_create_result": (),
"encrypted_code_execution_result": ("stderr", "content"),
"code_execution_output": (),
"bash_code_execution_output": (),
"web_search_tool_result_error": ("error_code",),
"web_fetch_tool_result_error": ("error_code",),
"code_execution_tool_result_error": ("error_code",),
"bash_code_execution_tool_result_error": ("error_code",),
"text_editor_code_execution_tool_result_error": ("error_code", "error_message"),
}
)
_ANTHROPIC_DOCUMENT: Final = TypeAdapter(AnthropicMessagesDocumentParam)
def _count_anthropic_content(
content: Mapping[str, object],
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
*,
_content_depth: int = 0,
) -> int:
"""
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
Uses TypedDict definitions from litellm.types.llms.anthropic to determine
what fields to count and how to handle nested structures.
Dynamically infers which fields to count based on the TypedDict definition,
avoiding hardcoded field names.
Uses readable server tool fields or the client tool TypedDict definitions.
"""
typeddict_cls: Final = _validate_anthropic_content(content)
type_hints: Final[Mapping[str, object]] = getattr(typeddict_cls, "__annotations__", {})
if _content_depth >= DEFAULT_MAX_RECURSE_DEPTH:
raise UnsupportedContentTypeError("Anthropic tool content nesting exceeds the token counter limit")
content_type: Final = content.get("type")
if content_type == "document":
return _count_document_tokens(
_ANTHROPIC_DOCUMENT.validate_python(content),
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
server_fields: Final = _ANTHROPIC_SERVER_TOOL_FIELDS.get(content_type) if isinstance(content_type, str) else None
fields: Final[Mapping[str, object]] = (
dict.fromkeys(server_fields)
if server_fields is not None
else getattr(_validate_anthropic_content(content), "__annotations__", {})
)
tokens = 0
# Fields to skip (metadata/identifiers that don't contribute to prompt tokens)
skip_fields: Final = {"type", "id", "tool_use_id", "cache_control", "is_error"}
# Iterate over all fields defined in the TypedDict
for field_name, field_type in type_hints.items():
for field_name in fields:
if field_name in skip_fields:
continue
@ -1801,7 +1842,21 @@ def _count_anthropic_content(
pdf_pricing,
)
elif isinstance(field_value, dict):
tokens += count_function(str(field_value))
if (
server_fields is not None
and field_name == "content"
and (field_mapping := as_str_mapping(content[field_name])) is not None
):
tokens += _count_anthropic_content(
field_mapping,
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
_content_depth=_content_depth + 1,
)
else:
tokens += count_function(str(field_value))
except Exception as e:
if default_token_count is not None:
return default_token_count
@ -1816,6 +1871,29 @@ _TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
)
def _is_text_block(block: object) -> bool:
mapping: Final = as_str_mapping(block)
block_type: Final = mapping.get("type") if mapping is not None else None
return isinstance(block, str) or (isinstance(block_type, str) and block_type in _TEXT_FIELD_BY_BLOCK_TYPE)
def _has_only_text_content(message: object) -> bool:
normalized: Final[object] = message.model_dump(exclude_none=True) if isinstance(message, BaseModel) else message
mapping: Final = as_str_mapping(normalized)
if mapping is None:
return False
content: Final = mapping.get("content")
if content is None or isinstance(content, str):
return True
blocks: Final = as_object_list(content)
return blocks is not None and all(_is_text_block(block) for block in blocks)
def has_only_text_content(messages: Sequence[object]) -> bool:
"""Whether counting ``messages`` stays on the tokenizer, never decoding images, files or documents."""
return all(_has_only_text_content(message) for message in messages)
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -1867,7 +1945,9 @@ def _count_content_list(
use_default_image_token_count,
pdf_pricing,
)
elif c["type"] in ("tool_use", "tool_result", "search_result"):
elif (
c["type"] in ("tool_use", "tool_result", "search_result") or c["type"] in _ANTHROPIC_SERVER_TOOL_FIELDS
):
num_tokens += _count_anthropic_content(
c,
count_function,
@ -1886,13 +1966,16 @@ def _count_content_list(
tool_name = str(c.get("tool_name") or "")
if tool_name:
num_tokens += count_function(tool_name)
elif (change := as_str_mapping(c)) is not None and change.get("type") in ("tool_addition", "tool_removal"):
num_tokens += _tool_change_tokens(count_function, as_str_mapping(change.get("tool")))
else:
content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
raise UnsupportedContentTypeError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"compaction, tool_reference, search_result)."
f"compaction, tool_reference, tool_addition, tool_removal, search_result, "
f"or Anthropic server tool blocks)."
)
return num_tokens
except Exception as e:
@ -1904,6 +1987,24 @@ def _count_content_list(
)
def _tool_change_tokens(count_function: TokenCounterFunction, tool: Mapping[str, object] | None) -> int:
match tool:
case {"type": "tool_definition", "definition": definition} if (
function := as_str_mapping(definition)
) is not None:
return count_function(_format_function_definitions((function,)))
case {"type": "tool_reference", "name": str() as name}:
return count_function(name)
case {"type": "mcp_tool_reference", "server_name": str() as server_name, "name": str() as name}:
return count_function(server_name) + count_function(name)
case {"type": "mcp_toolset_reference", "server_name": str() as server_name}:
return count_function(server_name)
case _:
raise UnsupportedContentTypeError(
f"Invalid tool change target type: {None if tool is None else tool.get('type')}"
)
def _format_function_definitions(tools: Sequence[object]) -> str:
"""Formats tool definitions in the format that OpenAI appears to use.
Based on https://github.com/forestwanglin/openai-java/blob/main/jtokkit/src/main/java/xyz/felh/openai/jtokkit/utils/TikTokenUtils.java

View file

@ -51,6 +51,7 @@ class _UrlFetcher(Protocol):
headers: dict[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
) -> httpx.Response: ...
@ -80,10 +81,11 @@ class _FetchOptions(TypedDict, total=False):
headers: ReadOnly[dict[str, str]]
follow_redirects: ReadOnly[bool]
timeout: ReadOnly[float | httpx.Timeout | None]
max_response_bytes: ReadOnly[int | None]
class _AsyncFetchOptions(_FetchOptions, total=False):
max_response_bytes: ReadOnly[int | None]
pass
# Cloud-internal IPs that ``ip.is_global`` can report as public. Everything

View file

@ -1,4 +1,3 @@
import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
@ -20,6 +19,8 @@ from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_
from litellm.llms.anthropic.messages.utils import (
is_reasoning_auto_summary_enabled,
json_copy_or_original,
object_mapping,
openai_strict_json_schema,
prompt_cache_key_from_user_id,
)
from litellm.responses.utils import auto_reasoning_summary
@ -1017,66 +1018,23 @@ class LiteLLMAnthropicMessagesAdapter:
Returns:
OpenAI-compatible response_format dict, or None if invalid
"""
if not isinstance(output_format, dict):
output_format_object: Final = object_mapping(output_format)
if output_format_object is None or output_format_object.get("type") != "json_schema":
return None
output_type: Final = output_format.get("type")
if output_type != "json_schema":
return None
schema = output_format.get("schema")
schema: Final = output_format_object.get("schema")
if not schema:
return None
# Deep copy to avoid mutating the original schema
schema = copy.deepcopy(schema)
# OpenAI strict mode requires additionalProperties: false on every object
self._add_additional_properties_false(schema)
# Convert to OpenAI response_format structure
return {
"type": "json_schema",
"json_schema": {
"name": "structured_output",
"schema": schema,
"schema": openai_strict_json_schema(schema),
"strict": True,
},
}
@staticmethod
def _add_additional_properties_false(schema: dict) -> None:
"""
Recursively ensure object schemas comply with OpenAI strict mode.
OpenAI's strict mode requires:
1. 'additionalProperties': false at every object nesting level
2. All property keys listed in 'required'
"""
if not isinstance(schema, dict):
return
if schema.get("type") == "object" and "properties" in schema:
schema["additionalProperties"] = False
schema["required"] = list(schema["properties"].keys())
for prop in schema["properties"].values():
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop)
# Handle array items
if "items" in schema:
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"])
# Handle anyOf/oneOf/allOf
for key in ("anyOf", "oneOf", "allOf"):
if key in schema:
for sub_schema in schema[key]:
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema)
# Handle $defs / definitions
for key in ("$defs", "definitions"):
if key in schema:
for def_schema in schema[key].values():
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema)
def _translate_midturn_system_message_to_openai(
self,
message: AnthropicMessagesSystemMessageParam,

View file

@ -19,6 +19,19 @@ from litellm._logging import verbose_logger
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2.enums import Hook
from litellm.integrations.custom_logger_v2.legacy import (
PROXY_REQUEST_KEYS,
Admit,
KwargsFields,
hook_subscribers,
kwargs_after,
kwargs_patch,
run_subscribers,
)
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
from litellm.integrations.custom_logger_v2.outcomes import Patch
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -374,7 +387,7 @@ async def _execute_pre_request_hooks(
pass
# Build complete request kwargs dict
request_kwargs = {
request_kwargs: dict[str, object] = { # mutable-ok: the hooks take and may replace the request dict
"tools": tools,
"stream": stream,
"litellm_params": {
@ -386,10 +399,21 @@ async def _execute_pre_request_hooks(
if not litellm.callbacks:
return request_kwargs
from litellm.integrations.custom_logger import CustomLogger as _CustomLogger
callbacks: Final = litellm.callbacks
if (
Hook.pre_deployment_call in CustomLoggerV2.declared_hooks
and (subscribers := hook_subscribers(Hook.pre_deployment_call, callbacks, admit=Admit.every_logger)).has_v2
):
return kwargs_after( # pyright: ignore[reportReturnType] # whatever the last hook returned, as before
await run_subscribers(
subscribers,
KwargsFields(request_kwargs, PROXY_REQUEST_KEYS),
partial(_start_pre_request_hook, model, messages),
)
)
for callback in litellm.callbacks:
if not isinstance(callback, _CustomLogger):
for callback in callbacks:
if not isinstance(callback, CustomLogger):
continue
# Call the pre-request hook
@ -402,6 +426,27 @@ async def _execute_pre_request_hooks(
return request_kwargs
def _start_pre_request_hook(
model: str,
messages: list[dict[str, object]], # mutable-ok: the hook takes the messages list
callback: object,
request_kwargs: object,
) -> Awaitable[Patch | None] | None:
if not isinstance(callback, CustomLogger):
return None
return _pre_request_hook(callback, model, messages, request_kwargs)
async def _pre_request_hook(
callback: CustomLogger,
model: str,
messages: list[dict[str, object]], # mutable-ok: see above
request_kwargs: object,
) -> Patch | None:
modified_kwargs: Final = await callback.async_pre_request_hook(model, messages, request_kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownVariableType] # the request dict as the previous hook left it
return None if modified_kwargs is None else kwargs_patch(request_kwargs, modified_kwargs) # pyright: ignore[reportUnknownArgumentType] # see above
async def _try_websearch_short_circuit(
model: str,
messages: list[dict],

View file

@ -35,6 +35,8 @@ from litellm.llms.anthropic.messages.native.utils import (
from litellm.llms.anthropic.messages.utils import (
is_reasoning_auto_summary_enabled,
json_copy_or_original,
object_mapping,
openai_strict_json_schema,
prompt_cache_key_from_user_id,
)
from litellm.types.llms.anthropic import (
@ -588,10 +590,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
)
@staticmethod
def _resolve_output_format_strict(strict: bool | None) -> bool:
if strict is None:
return True
return strict
def _resolve_output_format_strict(strict: object) -> bool:
return True if strict is None else bool(strict)
def translate_request(
self,
@ -690,19 +690,23 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
# output_format / output_config.format -> text format
# output_format: {"type": "json_schema", "schema": {...}}
# output_config: {"format": {"type": "json_schema", "schema": {...}}}
output_format: object = anthropic_request.get("output_format")
output_config = anthropic_request.get("output_config")
if not isinstance(output_format, dict) and isinstance(output_config, dict):
output_format = output_config.get("format")
if isinstance(output_format, dict) and output_format.get("type") == "json_schema":
output_config_object: Final = object_mapping(anthropic_request.get("output_config"))
top_level_output_format: Final = object_mapping(anthropic_request.get("output_format"))
output_format: Final = (
top_level_output_format
if top_level_output_format is not None or output_config_object is None
else object_mapping(output_config_object.get("format"))
)
if output_format is not None and output_format.get("type") == "json_schema":
schema: Final = output_format.get("schema")
if schema:
output_strict: Final = self._resolve_output_format_strict(output_format.get("strict"))
responses_kwargs["text"] = {
"format": {
"type": "json_schema",
"name": "structured_output",
"schema": schema,
"strict": self._resolve_output_format_strict(output_format.get("strict")),
"schema": openai_strict_json_schema(schema) if output_strict else schema,
"strict": output_strict,
}
}

View file

@ -1,6 +1,7 @@
import copy
import json
import os
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeVar
@ -37,6 +38,61 @@ def json_copy_or_original(value: _JsonValue) -> _JsonValue:
return value
_SUBSCHEMA_LIST_KEYWORDS: Final = ("anyOf", "oneOf", "allOf")
_SUBSCHEMA_MAP_KEYWORDS: Final = ("$defs", "definitions")
_JSON_ARRAY: Final = TypeAdapter(list[object])
def _json_array(value: object) -> Sequence[object] | None:
try:
return _JSON_ARRAY.validate_python(value, strict=True)
except ValidationError:
return None
def openai_strict_json_schema(schema: object) -> object:
"""New copy of a JSON schema where every object with properties forbids extra keys and requires all of them.
OpenAI strict structured outputs rejects any schema that leaves an object open or a property optional.
"""
source: Final = object_mapping(schema)
if source is None:
return copy.deepcopy(schema)
properties: Final = object_mapping(source.get("properties"))
closing: Final[Mapping[str, object]] = (
{"required": list(properties), "additionalProperties": False}
if properties is not None and _is_object_type(source.get("type"))
else {}
)
return {**{key: _strict_subschema(key, value) for key, value in source.items()}, **closing}
def _is_object_type(schema_type: object) -> bool:
"""Whether a ``type`` keyword names an object, alone or in a union such as ``["object", "null"]``."""
types: Final = _json_array(schema_type)
return schema_type == "object" if types is None else "object" in types
def _strict_subschema(key: str, value: object) -> object:
if key == "items":
return openai_strict_json_schema(value)
if key == "properties" or key in _SUBSCHEMA_MAP_KEYWORDS:
named: Final = object_mapping(value)
return (
copy.deepcopy(value)
if named is None
else {name: openai_strict_json_schema(subschema) for name, subschema in named.items()}
)
if key in _SUBSCHEMA_LIST_KEYWORDS:
variants: Final = _json_array(value)
return (
copy.deepcopy(value) if variants is None else [openai_strict_json_schema(variant) for variant in variants]
)
return copy.deepcopy(value)
class _ClaudeCodeUserId(LiteLLMBaseModel):
"""The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation."""
@ -136,7 +192,7 @@ def proxy_sub_call_logging_kwargs(parent_kwargs: Mapping[str, object]) -> ProxyS
if disable_header is None
else {**parent_params, "proxy_server_request": {"headers": {X_LITELLM_DISABLE_CALLBACKS: disable_header}}}
)
trusted_vars: Final = _object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD))
trusted_vars: Final = object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD))
if trusted_vars is None:
return dynamic_params
return {**dynamic_params, "litellm_trusted_callback_vars": trusted_vars}
@ -154,13 +210,13 @@ def proxy_sub_call_trace_id(parent_kwargs: Mapping[str, object]) -> str | None:
def _disable_callbacks_header(parent_kwargs: Mapping[str, object]) -> str | None:
parent_request: Final = _object_mapping(parent_kwargs.get("proxy_server_request"))
headers: Final = _object_mapping(parent_request.get("headers")) if parent_request is not None else None
parent_request: Final = object_mapping(parent_kwargs.get("proxy_server_request"))
headers: Final = object_mapping(parent_request.get("headers")) if parent_request is not None else None
header: Final = headers.get(X_LITELLM_DISABLE_CALLBACKS) if headers is not None else None
return header if isinstance(header, str) else None
def _object_mapping(value: object) -> Mapping[str, object] | None:
def object_mapping(value: object) -> Mapping[str, object] | None:
try:
return _OBJECT_MAPPING.validate_python(value)
except ValidationError:

View file

@ -145,7 +145,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
def get_stripped_model_name(self, model: str) -> str:
return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
return (
model.replace("responses/", "")
.replace("o_series/", "")
.replace("azure_ai/", "")
.replace(AzureOpenAIGPT5Config.GPT5_SERIES_ROUTE, "")
)
def _validate_input_param(
self, input: str | ResponseInputParam, *, drop_params: bool = False

View file

@ -1,7 +1,9 @@
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
@ -16,6 +18,24 @@ else:
LiteLLMLoggingObj = Any
_BODY_FIELDS: Final = TypeAdapter(dict[str, object])
def extra_body_fields(optional_params: Mapping[str, object] | None) -> Mapping[str, object]:
"""The caller's ``extra_body`` fields, merged under the provider body after the transform so they add fields but never
replace the input or a mapped param."""
extra_body: Final = (optional_params or {}).get("extra_body")
return _BODY_FIELDS.validate_python(extra_body) if isinstance(extra_body, Mapping) else {}
def split_extra_body(
optional_params: Mapping[str, object] | None,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""The params without ``extra_body``, and the ``extra_body`` fields to merge into the provider body."""
params: Final = _BODY_FIELDS.validate_python(optional_params or {})
return {k: v for k, v in params.items() if k != "extra_body"}, extra_body_fields(params)
class BaseEmbeddingConfig(BaseConfig, ABC):
@abstractmethod
def transform_embedding_request(

View file

@ -78,7 +78,7 @@ class BaseImageGenerationConfig(ABC):
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
) -> dict[str, object]: # mutable-ok: inherited contract
raise NotImplementedError(
"ImageVariationConfig implementa 'transform_request_image_variation' for image variation models"
)
@ -100,6 +100,32 @@ class BaseImageGenerationConfig(ABC):
"ImageVariationConfig implements 'transform_response_image_variation' for image variation models"
)
async def async_transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict[str, object], # mutable-ok: inherited contract
optional_params: dict[str, object], # mutable-ok: inherited contract
litellm_params: dict[str, object], # mutable-ok: inherited contract
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:
return self.transform_image_generation_response( # pyright: ignore[reportUnknownMemberType] # legacy sync hook has untyped dict params
model=model,
raw_response=raw_response,
model_response=model_response,
logging_obj=logging_obj,
request_data=request_data,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=encoding,
api_key=api_key,
json_mode=json_mode,
)
def use_multipart_form_data(self) -> bool:
"""
Returns True if this provider requires multipart/form-data instead of JSON.

View file

@ -6,6 +6,7 @@ import copy
import json
import urllib.parse
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, get_args, overload
import httpx
@ -15,6 +16,7 @@ import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
from litellm.litellm_core_utils.hidden_params import set_hidden_param
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.embedding.transformation import extra_body_fields
from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -103,7 +105,7 @@ class BedrockEmbedding(BaseAWSLLM):
region_from_model: str | None = None,
) -> tuple[Credentials | None, str]:
auth_params: Final = pop_aws_auth_params(optional_params)
explicit_region_name: Final = optional_params.pop("aws_region_name", None)
explicit_region_name: Final = _OPTIONAL_STR.validate_python(optional_params.pop("aws_region_name", None))
aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name
### SET REGION NAME ###
@ -291,9 +293,10 @@ class BedrockEmbedding(BaseAWSLLM):
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
api_key: str | None = None,
is_async_invoke: bool | None = False,
extra_body: Mapping[str, object] = MappingProxyType({}),
):
responses: Final[list[dict]] = []
for data in batch_data:
for data in ({**extra_body, **item} for item in batch_data):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
@ -360,9 +363,10 @@ class BedrockEmbedding(BaseAWSLLM):
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
api_key: str | None = None,
is_async_invoke: bool | None = False,
extra_body: Mapping[str, object] = MappingProxyType({}),
):
responses: Final[list[dict]] = []
for data in batch_data:
for data in ({**extra_body, **item} for item in batch_data):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
@ -462,9 +466,12 @@ class BedrockEmbedding(BaseAWSLLM):
)
inference_params = copy.deepcopy(optional_params)
inference_params = {
k: v for k, v in inference_params.items() if k.lower() not in self.aws_authentication_params
k: v
for k, v in inference_params.items()
if k.lower() not in self.aws_authentication_params and k != "extra_body"
}
inference_params.pop("user", None) # make sure user is not passed in for bedrock call
extra_body: Final = extra_body_fields(optional_params)
data: CohereEmbeddingRequest | None = None
batch_data: list | None = None
@ -535,6 +542,8 @@ class BedrockEmbedding(BaseAWSLLM):
)
batch_data.append(nova_request)
request_data: Final = None if data is None else {**extra_body, **data}
### SET RUNTIME ENDPOINT ###
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
api_base=api_base,
@ -552,6 +561,7 @@ class BedrockEmbedding(BaseAWSLLM):
client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None),
timeout=timeout,
batch_data=batch_data,
extra_body=extra_body,
credentials=credentials,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
@ -566,6 +576,7 @@ class BedrockEmbedding(BaseAWSLLM):
client=(client if client is not None and isinstance(client, HTTPHandler) else None),
timeout=timeout,
batch_data=batch_data,
extra_body=extra_body,
credentials=credentials,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
@ -579,7 +590,7 @@ class BedrockEmbedding(BaseAWSLLM):
if returned_response is None:
raise Exception("Unable to map Bedrock request to provider")
return returned_response
elif data is None:
elif request_data is None:
raise Exception("Unable to map Bedrock request to provider")
headers = {"Content-Type": "application/json"}
@ -591,7 +602,7 @@ class BedrockEmbedding(BaseAWSLLM):
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=json.dumps(data),
data=json.dumps(request_data),
headers=headers,
api_key=api_key,
)
@ -607,7 +618,7 @@ class BedrockEmbedding(BaseAWSLLM):
logging_obj=logging_obj,
optional_params=optional_params,
encoding=encoding,
data=data,
data=request_data,
complete_api_base=prepped.url,
api_key=None,
aembedding=aembedding,

View file

@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Optional, TypeVar, cast
import httpx
from httpx import Response
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.constants import AWS_ERROR_TYPE_HEADER
@ -23,7 +23,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
bedrock_invoke_guardrail_headers,
)
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
from litellm.types.utils import ModelResponseStream
from litellm.types.utils import Embedding, EmbeddingResponse, ModelResponseStream, PromptTokensDetailsWrapper, Usage
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
@ -376,6 +376,53 @@ def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]:
return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)}
class _TitanEmbeddingsByType(BaseModel):
model_config = ConfigDict(frozen=True)
binary: tuple[int, ...] | None = None
floats: tuple[float, ...] | None = Field(default=None, alias="float")
class _TitanEmbeddingReply(BaseModel):
model_config = ConfigDict(frozen=True)
embedding: tuple[float, ...] | None = None
embeddings_by_type: _TitanEmbeddingsByType = Field(default=_TitanEmbeddingsByType(), alias="embeddingsByType")
input_text_token_count: int = Field(alias="inputTextTokenCount")
@property
def vector(self) -> tuple[float, ...] | None:
if self.embedding is not None:
return self.embedding
if self.embeddings_by_type.floats is not None:
return self.embeddings_by_type.floats
return self.embeddings_by_type.binary
def _invoke_embedding_response(model: str, httpx_response: Response, has_input_image: bool) -> EmbeddingResponse | None:
reply: Final = relayed_json_object(httpx_response)
if reply is None or reply.keys().isdisjoint(("embedding", "embeddingsByType")):
return None
try:
parsed: Final = _TitanEmbeddingReply.model_validate(reply)
except ValidationError:
return None
if parsed.vector is None:
return None
return EmbeddingResponse(
model=model,
usage=Usage(
prompt_tokens=parsed.input_text_token_count,
completion_tokens=0,
total_tokens=parsed.input_text_token_count,
prompt_tokens_details=PromptTokensDetailsWrapper(image_count=1, text_tokens=parsed.input_text_token_count)
if has_input_image
else None,
),
data=[Embedding(embedding=list(parsed.vector), index=0, object="embedding")],
)
def _served_service_tier(response_headers: Mapping[str, str]) -> str | None:
headers: Final = httpx.Headers(response_headers)
if _BEDROCK_SERVICE_TIER_HEADER not in headers:
@ -574,6 +621,14 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
from litellm.types.utils import LlmProviders, ModelResponse
from litellm.utils import ProviderConfigManager
embedding_response: Final = (
_invoke_embedding_response(model, httpx_response, has_input_image="inputImage" in request_data)
if "invoke" in endpoint
else None
)
if embedding_response is not None:
return embedding_response
if "invoke" in endpoint:
chat_config_model = "invoke/" + model
elif "converse" in endpoint:

View file

@ -1671,6 +1671,7 @@ class HTTPHandler:
follow_redirects: bool | None = None,
timeout: float | httpx.Timeout | None = None,
stream: bool = False,
max_response_bytes: int | None = None,
):
# Set follow_redirects to UseClientDefault if None
_follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
@ -1681,6 +1682,16 @@ class HTTPHandler:
url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout
)
if max_response_bytes is not None:
return self._get_with_response_limit(
url,
params=query,
headers=headers,
max_bytes=max_response_bytes,
follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects,
timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout),
)
response: Final = self.client.get(
url,
params=query,
@ -1691,6 +1702,67 @@ class HTTPHandler:
return response
def _get_with_response_limit(
self,
url: str,
*,
params: httpx.QueryParams,
headers: Mapping[str, str] | None,
timeout: httpx.Timeout,
max_bytes: int,
follow_redirects: bool,
) -> httpx.Response:
request: Final = self.client.build_request(
"GET",
url,
headers=MappingProxyType({**(headers or {}), "accept-encoding": "identity"}),
params=params,
timeout=timeout,
)
response: Final = self.client.send(request, stream=True, follow_redirects=False)
return self._read_with_response_limit(response, max_bytes=max_bytes, follow_redirects=follow_redirects)
def _next_redirect_hop(self, hop: httpx.Response) -> httpx.Response:
if hop.next_request is None:
return hop
hop.close()
return self.client.send(hop.next_request, auth=None, stream=True, follow_redirects=False)
def _last_redirect_hop(self, response: httpx.Response, *, follow_redirects: bool) -> httpx.Response:
hop = response # rebind-ok: walks the redirect chain without recursion
for _ in range(10):
if hop.next_request is None or not follow_redirects:
return hop
hop = self._next_redirect_hop(hop)
if hop.next_request is None or not follow_redirects:
return hop
hop.close()
raise ValueError("Too many redirects")
def _read_with_response_limit(
self, response: httpx.Response, *, max_bytes: int, follow_redirects: bool
) -> httpx.Response:
final: Final = self._last_redirect_hop(response, follow_redirects=follow_redirects)
try:
if final.is_redirect or final.is_error:
return httpx.Response(final.status_code, headers=final.headers, request=final.request)
content_encoding: Final = header_value(final.headers, "content-encoding")
if ("identity" if content_encoding is None else content_encoding).lower() != "identity":
raise HTTPResponseLimitError("Response size limits require an uncompressed response")
content_length: Final = header_value(final.headers, "content-length")
if int("0" if content_length is None else content_length) > max_bytes:
raise HTTPResponseLimitError("Response exceeds the configured size limit")
with BytesIO() as body:
for chunk in final.iter_bytes(chunk_size=65536):
if body.tell() + len(chunk) > max_bytes:
raise HTTPResponseLimitError("Response exceeds the configured size limit")
body.write(chunk)
return httpx.Response(
final.status_code, headers=final.headers, content=body.getvalue(), request=final.request
)
finally:
final.close()
def _send_streaming_get(
self,
url: str,

Some files were not shown because too many files have changed in this diff Show more