merge: sync main into bedrock s3 media branch
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
39
.github/actions/cache-go-build/action.yml
vendored
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
name: "Cache the Go build"
|
||||
description: >-
|
||||
Restore the Go module and build caches on every run and save them only from
|
||||
main, so pull requests reuse main's cache instead of evicting it with their
|
||||
own copies.
|
||||
|
||||
actions/setup-go's built-in cache saves from every ref, and a cache saved by a
|
||||
pull request is readable only by that pull request. Callers must pass
|
||||
`cache: false` to setup-go and run this action after it.
|
||||
|
||||
inputs:
|
||||
dependency-path:
|
||||
description: "Path to the go.sum that keys the cache"
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Restore and save the Go module and build caches
|
||||
if: github.ref == 'refs/heads/main'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-
|
||||
|
||||
- name: Restore the Go module and build caches
|
||||
if: github.ref != 'refs/heads/main'
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-
|
||||
37
.github/actions/restore-rust-bridge/action.yml
vendored
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
name: "Restore the Rust bridge"
|
||||
description: >-
|
||||
Restore the prebuilt Rust bridge that the unit workflow's rust-bridge job saves
|
||||
on main, keyed on every input of the Rust build.
|
||||
|
||||
The bridge is abi3, so one build serves every supported Python version. A job
|
||||
whose Rust inputs match main installs the restored wheel and native library
|
||||
instead of compiling them inside `uv sync`. On a miss the caller builds from
|
||||
source as before.
|
||||
|
||||
The restored `rust-bridge-dist` directory holds the editable litellm wheel, the
|
||||
litellm-proxy gateway wheel, and `litellm/rust_bridge/_native.abi3.so`.
|
||||
|
||||
outputs:
|
||||
key:
|
||||
description: "Cache key for the bridge built from this checkout"
|
||||
value: ${{ steps.key.outputs.key }}
|
||||
cache-hit:
|
||||
description: "'true' when the bridge for this exact key was restored"
|
||||
value: ${{ steps.restore.outputs.cache-hit }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Define the Rust bridge cache key
|
||||
id: key
|
||||
shell: bash
|
||||
env:
|
||||
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }}
|
||||
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Restore the Rust bridge
|
||||
id: restore
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: rust-bridge-dist
|
||||
key: ${{ steps.key.outputs.key }}
|
||||
33
.github/osv-scanner.toml
vendored
|
|
@ -11,22 +11,27 @@ reason = "filelock WindowsFileLock ignores mode in every release from 3.10.0 bef
|
|||
[[IgnoredVulns]]
|
||||
id = "GHSA-pcg2-g6cj-q92c"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped"
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-pw9p-v7r6-gx4p"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-r4j5-9xpp-3985"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-45c3-73f7-pv4m"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
|
||||
reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.2; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-mmhr-3c5m-f64f"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "soupsieve 4.0 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup in tests/code_coverage_tests/bedrock_pricing.py to parse AWS pricing HTML with html.parser and find table rows and cells, not CSS selectors; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5"
|
||||
|
||||
[[PackageOverrides]]
|
||||
name = "pillow"
|
||||
|
|
@ -35,3 +40,23 @@ ecosystem = "PyPI"
|
|||
vulnerability.ignore = true
|
||||
effectiveUntil = 2027-02-01
|
||||
reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups and of the mlflow extra stays on 12.3.0; 3.11+ is on 13.0.0; pillow has never shipped an older-major release after the next major (no 10.x after 11.0.0, no 11.x after 12.0.0), so 13.x is the only fix line; extended to 2027-02-01 with maintainer approval, same date as every other Python 3.10 entry; remove once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-vvf4-9x63-8xvv"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "dnspython 2.9.2 is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.2; the advisory is memory and CPU use when dns.rdata.from_text parses a WKS record from attacker-controlled zone file text, and litellm only reaches dnspython through pydantic EmailStr, which never parses zone files or does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-5cjw-f85j-cv6r"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "tornado 6.6.1 is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-6655-6qjp-jx7x"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "soupsieve 3.0.2 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. The advisory is exponential regex backtracking when compiling an untrusted CSS selector. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup only in tests/code_coverage_tests/bedrock_pricing.py, which calls find and find_all and never compiles a CSS selector; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-7g44-g9rc-x53w"
|
||||
ignoreUntil = 2027-02-01
|
||||
reason = "soupsieve 3.0.2 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. The advisory is unbounded recursion when parsing an untrusted CSS selector with deeply nested pseudo-classes. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup only in tests/code_coverage_tests/bedrock_pricing.py, which calls find and find_all and never parses a CSS selector; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5"
|
||||
|
|
|
|||
210
.github/pr-assets/customer-hierarchy-settings/api-proof.md
vendored
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
## Captured API proof
|
||||
|
||||
These are actual curl requests to a local proxy and a controlled HTTP provider, using synthetic records and keys. Addresses, header files and IDs below are placeholders. Response excerpts omit unrelated fields and credentials.
|
||||
|
||||
### Permission change: verified before and after
|
||||
|
||||
The baseline process ran clean relevant source at `6685525fa2a809e0d4bc2c13cb07aef8eec242c3`. The requests used the same bound key and org policy; the after request was rerun after the schema update: the org allowed only `gpt-4.1-mini`; the key allowed `usage-proof-model` and was bound with `metadata.customer_org_id`. No team/member was needed for this org-level comparison.
|
||||
|
||||
```sh
|
||||
curl --silent --show-error --request POST "$PROXY_BASE/v1/chat/completions" \
|
||||
--header "@$SCOPED_KEY_HEADERS" --header 'Content-Type: application/json' \
|
||||
--data-binary @request.json
|
||||
```
|
||||
|
||||
`request.json`:
|
||||
```json
|
||||
{
|
||||
"model": "usage-proof-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Synthetic before-after permission check"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Baseline: **HTTP 200**.
|
||||
```json
|
||||
{
|
||||
"object": "chat.completion",
|
||||
"model": "usage-proof-model",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "Controlled usage proof response.",
|
||||
"role": "assistant"
|
||||
},
|
||||
"provider_specific_fields": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Current implementation: **HTTP 403**, with **zero controlled-provider calls**.
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "The requested model 'usage-proof-model' is not in the allowed models for this customer. Check the models this customer can use and try again.",
|
||||
"type": "customer_model_access_denied",
|
||||
"param": "model",
|
||||
"code": "403"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Current hierarchy and key intersection
|
||||
|
||||
Admin requests created the records below; each returned **201**. Member `budget_id` selects its template and `budget_override_id` identifies its optional override:
|
||||
|
||||
`POST /management/v1/customer_organizations`
|
||||
```json
|
||||
{
|
||||
"customer_org_id": "<org-id>",
|
||||
"alias": "Permissions and deletion proof",
|
||||
"budget": {
|
||||
"max_budget": 500,
|
||||
"allowed_models": [
|
||||
"gpt-4.1-mini",
|
||||
"gpt-5.5"
|
||||
]
|
||||
},
|
||||
"default_budgets": [
|
||||
{
|
||||
"name": "Standard",
|
||||
"is_default": true,
|
||||
"max_budget": 100,
|
||||
"allowed_models": [
|
||||
"gpt-4.1-mini"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`POST /management/v1/customer_teams`
|
||||
```json
|
||||
{
|
||||
"customer_team_id": "<team-id>",
|
||||
"parent_org_id": "<org-id>",
|
||||
"alias": "Proof customer team",
|
||||
"budget": {
|
||||
"max_budget": 250,
|
||||
"allowed_models": [
|
||||
"gpt-4.1-mini",
|
||||
"gpt-5.5"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`POST /management/v1/customer_members`
|
||||
```json
|
||||
{
|
||||
"customer_member_id": "<member-id>",
|
||||
"parent_team_id": "<team-id>",
|
||||
"alias": "Proof customer member",
|
||||
"budget_override": {
|
||||
"max_budget": 100,
|
||||
"allowed_models": [
|
||||
"gpt-4.1-mini"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
A synthetic `internal_user` was created with `auto_create_key: false`. The admin then created these keys through `POST /key/generate` (each **200**; returned credentials omitted):
|
||||
|
||||
| Key | models | metadata / budget binding |
|
||||
| --- | --- | --- |
|
||||
| A | `["gpt-4.1-mini", "gpt-5.5"]` | `{"customer_org_id":"<org-id>","customer_member_id":"<member-id>"}` |
|
||||
| B | `["gpt-4.1-mini"]` | Same org/member binding |
|
||||
| C | `["gpt-4.1-mini"]` | `budget_id: "<shared-budget-id>"`; no customer binding |
|
||||
|
||||
Inference curl uses the appropriate key header file. Captured request bodies:
|
||||
```json
|
||||
{
|
||||
"model": "gpt-4.1-mini",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Synthetic permission check"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
```json
|
||||
{
|
||||
"model": "gpt-4.1-mini",
|
||||
"input": "Synthetic permission check"
|
||||
}
|
||||
```
|
||||
|
||||
| State / key | Requested model | Chat HTTP | Responses HTTP | Provider calls per request |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Member restriction / A | gpt-4.1-mini | 200 | 200 | 1 |
|
||||
| Member restriction / A | gpt-5.5 | 403 | 403 | 0 |
|
||||
| Member widened; key B remains narrow | gpt-5.5 | 403 | 403 | 0 |
|
||||
| Member widened; org narrowed / A | gpt-5.5 | 403 | 403 | 0 |
|
||||
|
||||
The widening PATCH was `{"budget_override":{"allowed_models":["gpt-4.1-mini","gpt-5.5"]}}` on the member. The org narrowing PATCH was `{"budget":{"allowed_models":["gpt-4.1-mini"]}}`. Key B still denied the model after the member widened.
|
||||
|
||||
### Cascade, retained history and shared key budget
|
||||
|
||||
**Same-revision comparison:** deleting the populated org without `cascade=true` returned **409**:
|
||||
```json
|
||||
{
|
||||
"type": "urn:litellm:error:customer-hierarchy-has-children",
|
||||
"title": "Has children",
|
||||
"status": 409,
|
||||
"detail": "Delete the customer's children before deleting the parent."
|
||||
}
|
||||
```
|
||||
|
||||
The confirmed cascade request returned **204**, with an empty body:
|
||||
|
||||
```sh
|
||||
curl --silent --show-error --request DELETE \
|
||||
"$PROXY_BASE/management/v1/customer_organizations/$ORG_ID?cascade=true" \
|
||||
--header "@$ADMIN_HEADERS"
|
||||
```
|
||||
|
||||
Subsequent GETs for that org, its team and its member each returned **404**. The existing demo organization still returned **200**.
|
||||
|
||||
Key C referenced the org-owned budget. After PATCHing that budget to `max_budget: 0`, its inference returned **422 `budget_exceeded` before and after cascade**, with zero provider calls. Its authenticated `GET /key/info` retained the same relation and limit (selected fields):
|
||||
```json
|
||||
{
|
||||
"info": {
|
||||
"budget_id": "<shared-budget-id>",
|
||||
"litellm_budget_table": {
|
||||
"max_budget": 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The same admin daily query returned **200** before and after cascade:
|
||||
|
||||
```sh
|
||||
curl --silent --show-error \
|
||||
"$PROXY_BASE/management/v1/customer_organizations/daily/activity?customer_org_ids=$ORG_ID&start_date=$UTC_DATE&end_date=$UTC_DATE" \
|
||||
--header "@$ADMIN_HEADERS"
|
||||
```
|
||||
|
||||
Selected captured metadata was identical before and after; API-key/entity breakdowns are omitted:
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"total_spend": 0.28,
|
||||
"total_prompt_tokens": 200,
|
||||
"total_completion_tokens": 40,
|
||||
"total_tokens": 240,
|
||||
"total_api_requests": 4,
|
||||
"total_successful_requests": 2,
|
||||
"total_failed_requests": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
BIN
.github/pr-assets/customer-hierarchy-settings/before.png
vendored
Normal file
|
After Width: | Height: | Size: 8.3 KiB |
BIN
.github/pr-assets/customer-hierarchy-settings/delete-confirmation.png
vendored
Normal file
|
After Width: | Height: | Size: 51 KiB |
BIN
.github/pr-assets/customer-hierarchy-settings/member-settings.png
vendored
Normal file
|
After Width: | Height: | Size: 93 KiB |
BIN
.github/pr-assets/customer-hierarchy-settings/members.png
vendored
Normal file
|
After Width: | Height: | Size: 90 KiB |
69
.github/pr-assets/customer-hierarchy-settings/passthrough-api-proof.md
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
The OpenAI passthrough route now resolves customer organization, team and effective member model restrictions before forwarding the request. It uses the existing customer resolver and model matcher, then checks the model again after hooks. Gateway key permissions still apply.
|
||||
|
||||
Actual curl proof used a controlled HTTP upstream. Before: PR head `43d55ab2b91e4cccccf014c5f813a2b70a4e89f9` merged with target `eb811913f1`, before the fix. After: the implementation linked from the current PR description. Counts below come from the upstream request log.
|
||||
|
||||
Setup requests, using synthetic identifiers:
|
||||
|
||||
```http
|
||||
POST /management/v1/customer_organizations
|
||||
{"customer_org_id":"proof-org","alias":"Synthetic passthrough permission proof","budget":{"max_budget":100,"allowed_models":["gpt-4.1-mini"]}}
|
||||
|
||||
POST /user/new
|
||||
{"user_id":"proof-user","user_role":"internal_user","auto_create_key":false}
|
||||
|
||||
POST /key/generate
|
||||
{"user_id":"proof-user","models":["gpt-4.1-mini","gpt-5.5"],"metadata":{"customer_org_id":"proof-org"}}
|
||||
|
||||
POST /key/generate
|
||||
{"user_id":"proof-user","models":["gpt-4.1-mini"],"metadata":{"customer_org_id":"proof-org"}}
|
||||
```
|
||||
|
||||
The first key permits both models; the second permits only mini. Credentials from both responses are omitted.
|
||||
|
||||
```bash
|
||||
curl "$GATEWAY_BASE_URL/openai/v1/chat/completions" \
|
||||
--header "@$CUSTOMER_KEY_AB_HEADERS" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{"model":"gpt-5.5","messages":[{"role":"user","content":"Synthetic passthrough policy check"}]}'
|
||||
```
|
||||
|
||||
Before: HTTP 200, one upstream call. Captured response excerpt:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "gpt-5.5",
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 120
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
After: HTTP 403, zero upstream calls. Captured response:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "The requested model 'gpt-5.5' is not in the allowed models for this customer. Check the models this customer can use and try again.",
|
||||
"type": "customer_model_access_denied",
|
||||
"param": "model",
|
||||
"code": "403"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The same key still receives HTTP 200 for `gpt-4.1-mini`, with one upstream call. After widening the organization to permit both models, the mini-only key still receives HTTP 403 for `gpt-5.5`, with zero upstream calls:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "The requested model 'gpt-5.5' is not available for this API key, or the model name is invalid. Check the models available to you and try again.",
|
||||
"type": "key_model_access_denied",
|
||||
"param": "model",
|
||||
"code": "403"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The proof deletes its synthetic organization afterward. Registered-route regressions also cover team restrictions, member templates and overrides, hook model rewrites, model-less file retrieval, and the existing configured passthrough model contract.
|
||||
BIN
.github/pr-assets/customer-hierarchy-settings/teams.png
vendored
Normal file
|
After Width: | Height: | Size: 73 KiB |
BIN
.github/pr-assets/customer-usage-detail/before-main-6685525.png
vendored
Normal file
|
After Width: | Height: | Size: 8.3 KiB |
BIN
.github/pr-assets/customer-usage-detail/final-overview.png
vendored
Normal file
|
After Width: | Height: | Size: 66 KiB |
BIN
.github/pr-assets/customer-usage-detail/final-report.png
vendored
Normal file
|
After Width: | Height: | Size: 61 KiB |
8
.github/workflows/test-linting.yml
vendored
|
|
@ -73,14 +73,14 @@ jobs:
|
|||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-lint-
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Clean Python cache
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
|
|
@ -396,7 +396,7 @@ jobs:
|
|||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
|
|
|
|||
|
|
@ -38,6 +38,16 @@ concurrency:
|
|||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# These jobs check packaging, installation and imports, so they build the Rust
|
||||
# workspace unoptimized. The release profile's fat LTO and opt-level 3 pushed
|
||||
# every job to its timeout. The overrides go through Cargo's environment
|
||||
# because a maturin `--profile ci` argument also reaches dependencies that
|
||||
# build from source, such as jiter on Python 3.15, which have no ci profile
|
||||
env:
|
||||
CARGO_PROFILE_RELEASE_OPT_LEVEL: "0"
|
||||
CARGO_PROFILE_RELEASE_LTO: "off"
|
||||
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "256"
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
62
.github/workflows/test-merge-smoke.yml
vendored
|
|
@ -3,6 +3,8 @@ name: smoke
|
|||
on:
|
||||
pull_request:
|
||||
branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"]
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
|
@ -52,8 +54,26 @@ jobs:
|
|||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Restore the Rust bridge
|
||||
id: rust-bridge
|
||||
uses: ./.github/actions/restore-rust-bridge
|
||||
|
||||
- name: Restore the Cargo build cache
|
||||
if: steps.rust-bridge.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
|
||||
env:
|
||||
RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }}
|
||||
PYTHON_VERSION: ${{ matrix.python-version }}
|
||||
run: |
|
||||
if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION"
|
||||
exit 0
|
||||
fi
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/litellm-*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
|
||||
- name: Create the loopback-only network namespace
|
||||
run: |
|
||||
|
|
@ -87,6 +107,7 @@ jobs:
|
|||
|
||||
- name: Upload smoke diagnostics
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: merge-smoke-diagnostics-py${{ matrix.python-version }}
|
||||
|
|
@ -97,6 +118,10 @@ jobs:
|
|||
if: always()
|
||||
run: sudo ip netns delete smoke
|
||||
|
||||
- name: Keep only source-built wheels in the uv cache
|
||||
if: always() && github.ref == 'refs/heads/main'
|
||||
run: uv cache prune --ci
|
||||
|
||||
all-extras:
|
||||
name: all-extras-py${{ matrix.python-version }}
|
||||
runs-on: ubuntu-24.04
|
||||
|
|
@ -123,12 +148,34 @@ jobs:
|
|||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Restore the Rust bridge
|
||||
id: rust-bridge
|
||||
uses: ./.github/actions/restore-rust-bridge
|
||||
|
||||
- name: Restore the Cargo build cache
|
||||
if: steps.rust-bridge.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install every extra and dependency group
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }}
|
||||
env:
|
||||
RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }}
|
||||
PYTHON_VERSION: ${{ matrix.python-version }}
|
||||
run: |
|
||||
if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION"
|
||||
exit 0
|
||||
fi
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" --no-install-project --no-install-package litellm-proxy
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
|
||||
- name: Import the SDK, the proxy server and the enterprise package
|
||||
run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise"
|
||||
|
||||
- name: Keep only source-built wheels in the uv cache
|
||||
if: always() && github.ref == 'refs/heads/main'
|
||||
run: uv cache prune --ci
|
||||
|
||||
windows-sdk:
|
||||
name: windows-sdk
|
||||
runs-on: windows-2025
|
||||
|
|
@ -154,15 +201,16 @@ jobs:
|
|||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Set up the Rust bridge
|
||||
uses: ./.github/actions/rust-bridge
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
|
||||
|
||||
- name: Run the Windows tests
|
||||
run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests
|
||||
|
||||
- name: Keep only source-built wheels in the uv cache
|
||||
if: always() && github.ref == 'refs/heads/main'
|
||||
run: uv cache prune --ci
|
||||
|
||||
proxy-build-from-pip:
|
||||
name: proxy-build-from-pip
|
||||
runs-on: ubuntu-24.04
|
||||
|
|
@ -298,6 +346,10 @@ jobs:
|
|||
if: always()
|
||||
run: cat /tmp/litellm-pip.log || true
|
||||
|
||||
- name: Keep only source-built wheels in the uv cache
|
||||
if: always() && github.ref == 'refs/heads/main'
|
||||
run: uv cache prune --ci
|
||||
|
||||
smoke-passed:
|
||||
name: smoke passed
|
||||
needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]
|
||||
|
|
|
|||
10
.github/workflows/test-rust-db.yml
vendored
|
|
@ -42,10 +42,6 @@ concurrency:
|
|||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
SCCACHE_GHA_VERSION: litellm-ci-v1
|
||||
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
|
||||
jobs:
|
||||
|
|
@ -91,11 +87,7 @@ jobs:
|
|||
with:
|
||||
workspaces: litellm/rust
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Set up sccache
|
||||
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
|
||||
with:
|
||||
version: "v0.18.0"
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Install Python dependencies
|
||||
working-directory: .
|
||||
|
|
|
|||
19
.github/workflows/test-rust.yml
vendored
|
|
@ -78,10 +78,6 @@ concurrency:
|
|||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
SCCACHE_GHA_VERSION: litellm-ci-v1
|
||||
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
|
||||
jobs:
|
||||
|
|
@ -108,11 +104,6 @@ jobs:
|
|||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Set up sccache
|
||||
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
|
||||
with:
|
||||
version: "v0.18.0"
|
||||
|
||||
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
|
||||
rust-test:
|
||||
|
|
@ -155,11 +146,6 @@ jobs:
|
|||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Set up sccache
|
||||
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
|
||||
with:
|
||||
version: "v0.18.0"
|
||||
|
||||
- name: Check generated trace contracts
|
||||
working-directory: .
|
||||
run: uv run litellm/scripts/generate_trace_types.py --check
|
||||
|
|
@ -206,11 +192,6 @@ jobs:
|
|||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Set up sccache
|
||||
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
|
||||
with:
|
||||
version: "v0.18.0"
|
||||
|
||||
- run: uv build --wheel --out-dir dist
|
||||
|
||||
- run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
|
||||
|
|
|
|||
18
.github/workflows/test-terraform-provider.yml
vendored
|
|
@ -4,6 +4,7 @@ on:
|
|||
push:
|
||||
paths:
|
||||
- "litellm/deploy/terraform/provider/**"
|
||||
- ".github/actions/cache-go-build/**"
|
||||
- ".github/workflows/test-terraform-provider.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
|
|
@ -13,6 +14,7 @@ on:
|
|||
- "litellm/deploy/terraform/provider/**"
|
||||
- "litellm/proxy/**"
|
||||
- "litellm/enterprise/**"
|
||||
- ".github/actions/cache-go-build/**"
|
||||
- ".github/workflows/test-terraform-provider.yml"
|
||||
|
||||
permissions:
|
||||
|
|
@ -38,8 +40,12 @@ jobs:
|
|||
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
|
||||
with:
|
||||
go-version-file: litellm/deploy/terraform/provider/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: litellm/deploy/terraform/provider/go.sum
|
||||
cache: false
|
||||
|
||||
- name: Cache the Go build
|
||||
uses: ./.github/actions/cache-go-build
|
||||
with:
|
||||
dependency-path: litellm/deploy/terraform/provider/go.sum
|
||||
|
||||
- name: gofmt
|
||||
run: |
|
||||
|
|
@ -120,8 +126,12 @@ jobs:
|
|||
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
|
||||
with:
|
||||
go-version-file: litellm/deploy/terraform/provider/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: litellm/deploy/terraform/provider/go.sum
|
||||
cache: false
|
||||
|
||||
- name: Cache the Go build
|
||||
uses: ./.github/actions/cache-go-build
|
||||
with:
|
||||
dependency-path: litellm/deploy/terraform/provider/go.sum
|
||||
|
||||
- name: Audit provider endpoints against the schema
|
||||
working-directory: litellm/deploy/terraform/provider
|
||||
|
|
|
|||
28
.github/workflows/test-unit.yml
vendored
|
|
@ -41,13 +41,6 @@ jobs:
|
|||
timeout-minutes: 2
|
||||
uses: ./.github/actions/detect-changes
|
||||
|
||||
- name: Define editable Rust bridge cache key
|
||||
id: rust-bridge-key
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }}
|
||||
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
|
|
@ -66,30 +59,17 @@ jobs:
|
|||
id: rust-bridge-cache
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830
|
||||
with:
|
||||
path: rust-bridge-dist
|
||||
key: ${{ steps.rust-bridge-key.outputs.key }}
|
||||
uses: ./.github/actions/restore-rust-bridge
|
||||
|
||||
- name: Cache the Rust build
|
||||
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Set up sccache
|
||||
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
|
||||
uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a
|
||||
with:
|
||||
version: "v0.18.0"
|
||||
|
||||
- name: Build editable Rust bridge
|
||||
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
SCCACHE_GHA_VERSION: litellm-ci-v1
|
||||
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
run: |
|
||||
mkdir -p rust-bridge-dist
|
||||
|
|
@ -103,10 +83,6 @@ jobs:
|
|||
timeout-minutes: 15
|
||||
working-directory: litellm/packaging/litellm-proxy
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
SCCACHE_GHA_VERSION: litellm-ci-v1
|
||||
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../../rust-bridge-dist
|
||||
|
||||
|
|
@ -125,7 +101,7 @@ jobs:
|
|||
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830
|
||||
with:
|
||||
path: rust-bridge-dist
|
||||
key: ${{ steps.rust-bridge-key.outputs.key }}
|
||||
key: ${{ steps.rust-bridge-cache.outputs.key }}
|
||||
|
||||
- name: Expose the Rust bridge artifact
|
||||
id: rust-bridge-artifact
|
||||
|
|
|
|||
|
|
@ -36,11 +36,6 @@ _billing_time: Final[ContextVar[datetime | None]] = ContextVar("billing_time", d
|
|||
_post_response: Final[ContextVar[bool]] = ContextVar("post_response", default=False)
|
||||
|
||||
_service_target: Final[ContextVar[str | None]] = ContextVar("service_target", default=None)
|
||||
# Event-metadata key under which a Redis pipeline reports the sorted, comma-joined families its ops were
|
||||
# declared under when they span more than one.
|
||||
REDIS_FAMILIES_METADATA_KEY: Final = "families"
|
||||
|
||||
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
|
||||
|
||||
_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
|
||||
|
||||
|
|
@ -140,22 +135,6 @@ def with_service_target(
|
|||
return decorate
|
||||
|
||||
|
||||
@contextmanager
|
||||
def service_caller(caller: str | None) -> Generator[None]:
|
||||
"""Name the litellm code a datastore call was issued for when its own frames cannot: an operation
|
||||
declared in one task and run in another (a batch op retried on the flush) carries the chain captured
|
||||
where it was declared."""
|
||||
token: Final = _service_caller.set(caller)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_restore(_service_caller, token)
|
||||
|
||||
|
||||
def current_service_caller() -> str | None:
|
||||
return _service_caller.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def mirrored_call() -> Generator[None]:
|
||||
"""A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included."""
|
||||
|
|
@ -197,6 +176,20 @@ def current_request_parent_otel_span() -> Span | None:
|
|||
return _request_parent_otel_span.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def request_parent_otel_span(span: Span | None) -> Generator[None]:
|
||||
"""Anchor the datastore service spans made inside this block, and by the tasks it starts, under the span an SDK
|
||||
caller passed. A request that already anchored its own, as the proxy does at auth, keeps it."""
|
||||
if span is None or _request_parent_otel_span.get() is not None:
|
||||
yield
|
||||
return
|
||||
token: Final = _request_parent_otel_span.set(span)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_restore(_request_parent_otel_span, token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def outside_mirrored_call() -> Generator[None]:
|
||||
"""Logging a mirrored call: the calls its callbacks start are not mirrors themselves."""
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@ def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpda
|
|||
"model_access_group_list_transactions": dict(
|
||||
_entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable")
|
||||
),
|
||||
"customer_org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerOrganizationTable")),
|
||||
"customer_team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerTeamTable")),
|
||||
"customer_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerMemberTable")),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -14,9 +14,10 @@ from litellm.constants import (
|
|||
MODEL_INSIGHTS_TASK_TAG_PREFIX,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry
|
||||
from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution
|
||||
from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids
|
||||
from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks
|
||||
from litellm.proxy.db.model_usage_rollup import date_from_start_time
|
||||
from litellm.proxy.db.model_usage_rollup import date_from_start_time, utc_date_from_start_time
|
||||
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
|
||||
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
|
||||
from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens
|
||||
|
|
@ -70,9 +71,32 @@ def _spend(table: str, key: tuple[tuple[str, str], ...], spend: float) -> Contri
|
|||
return Contribution(table, key, (("spend", spend),))
|
||||
|
||||
|
||||
_CUSTOMER_ID_KEYS: Final = ("customer_org_id", "customer_team_id", "customer_member_id")
|
||||
|
||||
|
||||
def _customer_ids(metadata: Mapping[str, object]) -> Mapping[str, str | None]:
|
||||
"""The customer hierarchy a request is charged to: the member the request named, else the key's attribution."""
|
||||
entities: Final = _fields(metadata.get("accounting_entities"))
|
||||
if isinstance(entities.get("customer_org_id"), str):
|
||||
return {key: value if isinstance(value := entities.get(key), str) else None for key in _CUSTOMER_ID_KEYS}
|
||||
serialized: Final = metadata.get(CUSTOMER_USAGE_METADATA_KEY)
|
||||
if not isinstance(serialized, str):
|
||||
return dict.fromkeys(_CUSTOMER_ID_KEYS)
|
||||
try:
|
||||
attribution: Final = CustomerUsageAttribution.model_validate_json(serialized)
|
||||
except ValidationError:
|
||||
return dict.fromkeys(_CUSTOMER_ID_KEYS)
|
||||
return {
|
||||
"customer_org_id": attribution.customer_org_id,
|
||||
"customer_team_id": attribution.customer_team_id,
|
||||
"customer_member_id": attribution.customer_member_id,
|
||||
}
|
||||
|
||||
|
||||
def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
|
||||
spend: Final = _number(row.get("spend"))
|
||||
entities: Final = _fields(metadata.get("accounting_entities"))
|
||||
customer: Final = _customer_ids(metadata)
|
||||
identities: Final = (
|
||||
("LiteLLM_VerificationToken", "token", entities.get("api_key")),
|
||||
("LiteLLM_UserTable", "user_id", entities.get("user")),
|
||||
|
|
@ -80,6 +104,9 @@ def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, obje
|
|||
("LiteLLM_TeamTable", "team_id", entities.get("team_id")),
|
||||
("LiteLLM_OrganizationTable", "organization_id", entities.get("organization_id")),
|
||||
("LiteLLM_EndUserTable", "user_id", entities.get("end_user")),
|
||||
("LiteLLM_CustomerOrganizationTable", "customer_org_id", customer["customer_org_id"]),
|
||||
("LiteLLM_CustomerTeamTable", "customer_team_id", customer["customer_team_id"]),
|
||||
("LiteLLM_CustomerMemberTable", "customer_member_id", customer["customer_member_id"]),
|
||||
("LiteLLM_AgentsTable", "agent_id", entities.get("agent_id")),
|
||||
("LiteLLM_ProjectTable", "project_id", entities.get("project_id")),
|
||||
)
|
||||
|
|
@ -159,7 +186,14 @@ def _daily_contributions(row: Mapping[str, object], metadata: Mapping[str, objec
|
|||
)
|
||||
# The legacy daily organization writer uses the charged org argument, not
|
||||
# SpendLogs' normalized empty-string identity for a request with no org.
|
||||
daily_row: Final = {**row, "organization_id": _fields(metadata.get("accounting_entities")).get("organization_id")}
|
||||
entities: Final = _fields(metadata.get("accounting_entities"))
|
||||
customer: Final = _customer_ids(metadata)
|
||||
daily_row: Final = {
|
||||
**row,
|
||||
"organization_id": entities.get("organization_id"),
|
||||
"customer_org_id": customer["customer_org_id"],
|
||||
**{key: customer[key] or "" for key in ("customer_team_id", "customer_member_id")},
|
||||
}
|
||||
return tuple(_daily_rows(daily_row, shared_key, counters))
|
||||
|
||||
|
||||
|
|
@ -169,13 +203,26 @@ def _daily_rows(
|
|||
counters: tuple[tuple[str, int | float], ...],
|
||||
) -> Iterator[Contribution]:
|
||||
for entity, table in DAILY_SPEND_TABLES.items():
|
||||
customer_date = _utc_date(row) if entity == "customer_member" else None
|
||||
if entity == "customer_member" and customer_date is None:
|
||||
continue
|
||||
daily_key = tuple(
|
||||
(column, customer_date if column == "date" and customer_date is not None else value)
|
||||
for column, value in shared_key
|
||||
)
|
||||
model_group = (("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),)
|
||||
for identity in daily_spend_entity_ids(row, entity):
|
||||
yield Contribution(
|
||||
table.name,
|
||||
((table.entity_id_column, _text(identity)), *shared_key),
|
||||
(
|
||||
(table.entity_id_column, _text(identity)),
|
||||
*((column, _text(row.get(column))) for column in table.parent_id_columns),
|
||||
*daily_key,
|
||||
*((("model_group", _text(row.get("model_group"))),) if table.hashed_identity else ()),
|
||||
),
|
||||
counters,
|
||||
(
|
||||
("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),
|
||||
*(() if table.hashed_identity else model_group),
|
||||
*((("request_id", _text(row.get("request_id"))),) if table.carries_request_id else ()),
|
||||
),
|
||||
)
|
||||
|
|
@ -259,18 +306,7 @@ def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, objec
|
|||
|
||||
def _utc_date(row: Mapping[str, object]) -> str | None:
|
||||
start: Final = row.get("startTime")
|
||||
try:
|
||||
parsed: Final = datetime.fromisoformat(start.replace("Z", "+00:00")) if isinstance(start, str) else start
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(parsed, datetime):
|
||||
return None
|
||||
return (
|
||||
(parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed)
|
||||
.astimezone(timezone.utc)
|
||||
.date()
|
||||
.isoformat()
|
||||
)
|
||||
return utc_date_from_start_time(start) if isinstance(start, (datetime, str)) else None
|
||||
|
||||
|
||||
def _guardrail_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ _METADATA_FIELDS: Final = frozenset(
|
|||
"compression_savings",
|
||||
"internal_call_origin",
|
||||
"billed_retry_attempt",
|
||||
"user_api_key_customer_usage",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -51,6 +52,9 @@ _ACCOUNTING_FIELDS: Final = frozenset(
|
|||
"team_id",
|
||||
"organization_id",
|
||||
"end_user",
|
||||
"customer_member_id",
|
||||
"customer_team_id",
|
||||
"customer_org_id",
|
||||
"agent_id",
|
||||
"request_tags",
|
||||
"status",
|
||||
|
|
|
|||
|
|
@ -10,8 +10,10 @@ from typing import (
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.accounting.application import apply_deliveries
|
||||
from litellm.accounting.delivery_projection import entity_transactions
|
||||
from litellm.accounting.metrics import record_backlog, record_storage
|
||||
from litellm.accounting.projection import Contribution
|
||||
from litellm.proxy.spend_tracking.customer_budget_alerts import CustomerBudgetAlerting
|
||||
from litellm.repositories.accounting_delivery_repository import (
|
||||
AccountingDeliveryRepository,
|
||||
DeliveryBatchResult,
|
||||
|
|
@ -65,6 +67,9 @@ async def flush_accounting_deliveries(
|
|||
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
|
||||
if result.applied:
|
||||
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
|
||||
await CustomerBudgetAlerting.detect_committed_batch(
|
||||
entity_transactions(applied), prisma_client, proxy_logging_obj.internal_usage_cache.dual_cache
|
||||
)
|
||||
if result.parked:
|
||||
verbose_proxy_logger.error(
|
||||
"Accounting delivery parked %d requests after isolated failures", len(result.parked)
|
||||
|
|
|
|||
|
|
@ -1,151 +1,42 @@
|
|||
"""Atomic affinity claims shared by deployment and tier-model selection."""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import (
|
||||
Final,
|
||||
cast, # noqa: TID251 # Heterogeneous cache results are narrowed only to object, then validated
|
||||
)
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.redis_batch import active_request_redis_batch
|
||||
from litellm.caching.redis_batch import Batch
|
||||
from litellm.redis.stores.router_session_pins import ROUTER_SESSION_PINS_TARGET, PinKey, RouterSessionPinStore
|
||||
|
||||
ROUTER_SESSION_PINS_TARGET: Final = "router_session_pins"
|
||||
|
||||
_PIN_JSON_ADAPTER: Final = TypeAdapter[JsonValue](JsonValue)
|
||||
|
||||
_READ_PIN_SCRIPT: Final = """
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if current ~= false then
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[1])
|
||||
end
|
||||
return current
|
||||
"""
|
||||
|
||||
_CLAIM_PIN_SCRIPT: Final = """
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if current == false then
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
|
||||
return ARGV[1]
|
||||
end
|
||||
if ARGV[3] then
|
||||
local decoded, stored = pcall(cjson.decode, current)
|
||||
if ARGV[4] then
|
||||
local legacy = current
|
||||
if decoded and type(stored) == 'string' then legacy = stored end
|
||||
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
|
||||
if eligible['model_id'] == legacy then
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return current
|
||||
end
|
||||
end
|
||||
end
|
||||
if decoded and type(stored) == 'table' then
|
||||
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
|
||||
local matches = true
|
||||
for key, value in pairs(eligible) do
|
||||
if stored[key] ~= value then matches = false; break end
|
||||
end
|
||||
for key, _ in pairs(stored) do
|
||||
if eligible[key] == nil then matches = false; break end
|
||||
end
|
||||
if matches then
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return current
|
||||
end
|
||||
end
|
||||
end
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
|
||||
return ARGV[1]
|
||||
end
|
||||
if current == ARGV[1] then
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
end
|
||||
return current
|
||||
"""
|
||||
__all__ = [
|
||||
"ROUTER_SESSION_PINS_TARGET",
|
||||
"claim_affinity_pin",
|
||||
"read_affinity_pin",
|
||||
"set_local_affinity_pin",
|
||||
]
|
||||
|
||||
|
||||
def set_local_affinity_pin(cache: DualCache, cache_key: str, value: object, ttl_seconds: int) -> None:
|
||||
"""Replace the entry because InMemoryCache.set_cache preserves a live key's expiry."""
|
||||
cache.in_memory_cache.delete_cache(cache_key)
|
||||
cache.in_memory_cache.set_cache(cache_key, value, ttl=ttl_seconds)
|
||||
def set_local_affinity_pin(pins: RouterSessionPinStore, cache_key: PinKey, value: object, ttl_seconds: int) -> None:
|
||||
pins.set_local(cache_key, value, ttl_seconds)
|
||||
|
||||
|
||||
def _legacy_pin_matches(stored: object, pin_value: Mapping[str, str]) -> bool:
|
||||
if isinstance(stored, dict):
|
||||
return all(stored.get(key) is not None and str(stored[key]) == value for key, value in pin_value.items())
|
||||
return isinstance(stored, str) and len(pin_value) == 1 and stored in pin_value.values()
|
||||
|
||||
|
||||
def claim_affinity_pin_in_memory(
|
||||
cache: DualCache,
|
||||
cache_key: str,
|
||||
pin_value: Mapping[str, str],
|
||||
ttl_seconds: int,
|
||||
*,
|
||||
eligible_values: tuple[Mapping[str, str], ...] | None = None,
|
||||
preserve_legacy_model_ids: bool = False,
|
||||
) -> object:
|
||||
"""No await between read and write, so same-loop claims agree during a Redis outage."""
|
||||
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
|
||||
if existing is not None and eligible_values is None:
|
||||
if _legacy_pin_matches(existing, pin_value):
|
||||
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
|
||||
return existing
|
||||
eligible_existing: Final = existing in (eligible_values or ()) or (
|
||||
preserve_legacy_model_ids
|
||||
and isinstance(existing, str)
|
||||
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
|
||||
)
|
||||
winner: Final = existing if existing is not None and eligible_existing else pin_value
|
||||
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
|
||||
return winner
|
||||
|
||||
|
||||
def _decode_pin(value: str) -> object:
|
||||
try:
|
||||
return _PIN_JSON_ADAPTER.validate_json(value)
|
||||
except ValidationError:
|
||||
return value
|
||||
|
||||
|
||||
async def read_affinity_pin(cache: DualCache, cache_key: str, ttl_seconds: int) -> object:
|
||||
async def read_affinity_pin(pins: RouterSessionPinStore, cache_key: PinKey, ttl_seconds: int) -> object:
|
||||
"""Refresh the authoritative pin without writing a previously observed value back."""
|
||||
redis_cache: Final = cache.redis_cache
|
||||
if redis_cache is not None:
|
||||
if pins.has_redis:
|
||||
try:
|
||||
read_script: Final = redis_cache.async_register_script(_READ_PIN_SCRIPT)
|
||||
batch: Final = active_request_redis_batch(redis_cache)
|
||||
raw: Final = cast( # cast-ok: Redis script returns a serialized pin or a miss; validate before use
|
||||
object,
|
||||
await batch.script(_READ_PIN_SCRIPT, read_script, (cache_key,), (int(ttl_seconds),))
|
||||
if batch is not None and batch.pending
|
||||
else await read_script(keys=(cache_key,), args=(int(ttl_seconds),)),
|
||||
)
|
||||
decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
|
||||
value: Final = _decode_pin(decoded) if isinstance(decoded, str) else None
|
||||
if value is None:
|
||||
cache.in_memory_cache.delete_cache(cache_key)
|
||||
else:
|
||||
set_local_affinity_pin(cache, cache_key, value, ttl_seconds)
|
||||
value: Final = await pins.read(cache_key, ttl_seconds, batch=Batch.PRE_CALL)
|
||||
except Exception as error: # noqa: BLE001 # Redis faults retain pod-local continuity, unlike a successful miss
|
||||
verbose_router_logger.debug("Affinity cache: Redis read failed, using pod-local pin. error=%s", error)
|
||||
else:
|
||||
return value
|
||||
local_value: Final = cast( # cast-ok: the heterogeneous local cache is parsed by the affinity caller
|
||||
object, cache.in_memory_cache.get_cache(cache_key)
|
||||
)
|
||||
local_value: Final = pins.get_local(cache_key)
|
||||
if local_value is not None:
|
||||
set_local_affinity_pin(cache, cache_key, local_value, ttl_seconds)
|
||||
set_local_affinity_pin(pins, cache_key, local_value, ttl_seconds)
|
||||
return local_value
|
||||
|
||||
|
||||
async def claim_affinity_pin(
|
||||
cache: DualCache,
|
||||
cache_key: str,
|
||||
pins: RouterSessionPinStore,
|
||||
cache_key: PinKey,
|
||||
pin_value: Mapping[str, str],
|
||||
ttl_seconds: int,
|
||||
*,
|
||||
|
|
@ -157,33 +48,20 @@ async def claim_affinity_pin(
|
|||
Eligible claims refresh the returned winner. Legacy deployment claims only refresh
|
||||
a matching candidate. Resolve Redis per call because the proxy attaches it lazily.
|
||||
"""
|
||||
redis_cache: Final = cache.redis_cache
|
||||
if redis_cache is not None:
|
||||
if pins.has_redis:
|
||||
try:
|
||||
claim_script: Final = redis_cache.async_register_script(_CLAIM_PIN_SCRIPT)
|
||||
args: Final = (
|
||||
json.dumps(dict(pin_value)),
|
||||
int(ttl_seconds),
|
||||
*(
|
||||
(json.dumps(tuple(dict(value) for value in eligible_values)),)
|
||||
if eligible_values is not None
|
||||
else ()
|
||||
),
|
||||
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
|
||||
winner: Final = await pins.claim(
|
||||
cache_key,
|
||||
pin_value,
|
||||
ttl_seconds,
|
||||
eligible_values=eligible_values,
|
||||
preserve_legacy_model_ids=preserve_legacy_model_ids,
|
||||
)
|
||||
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
|
||||
object, await claim_script(keys=(cache_key,), args=args)
|
||||
)
|
||||
decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
|
||||
if not isinstance(decoded, str):
|
||||
return pin_value
|
||||
winner: Final = _decode_pin(decoded)
|
||||
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
|
||||
return winner
|
||||
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
|
||||
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
|
||||
return claim_affinity_pin_in_memory(
|
||||
cache,
|
||||
else:
|
||||
return pin_value if winner is None else winner
|
||||
return pins.claim_local(
|
||||
cache_key,
|
||||
pin_value,
|
||||
ttl_seconds,
|
||||
|
|
|
|||
|
|
@ -8,11 +8,10 @@ Has 4 primary methods:
|
|||
- async_get_cache
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Mapping, Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
|
@ -24,7 +23,6 @@ from litellm.types.caching import RedisPipelineIncrementOperation
|
|||
|
||||
from .base_cache import BaseCache
|
||||
from .in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache
|
||||
from .redis_batch import BatchResult, RedisBatch, active_post_call_redis_batch, active_request_redis_batch
|
||||
from .redis_cache import RedisCache, RedisCircuitBreakerOpenError, log_redis_failure
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -59,14 +57,6 @@ class PendingBatchRead:
|
|||
previous_access_times: dict[str, float | None]
|
||||
|
||||
|
||||
def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
|
||||
if future.cancelled():
|
||||
return
|
||||
failure: Final = future.exception()
|
||||
if failure is not None:
|
||||
log_redis_failure(verbose_logger, logging.WARNING, "post-call Redis increment failed", failure)
|
||||
|
||||
|
||||
class DualCache(BaseCache):
|
||||
"""
|
||||
DualCache is a cache implementation that updates both Redis and an in-memory cache simultaneously.
|
||||
|
|
@ -269,9 +259,6 @@ class DualCache(BaseCache):
|
|||
result = in_memory_result
|
||||
|
||||
if result is None and self.redis_cache is not None and local_only is False:
|
||||
request_batch: Final = active_request_redis_batch(self.redis_cache)
|
||||
if request_batch is not None and request_batch.read_as_missing(key):
|
||||
return None
|
||||
# If not found in in-memory cache, try fetching from Redis
|
||||
redis_result: Final = await self.redis_cache.async_get_cache(key, parent_otel_span=parent_otel_span)
|
||||
|
||||
|
|
@ -316,20 +303,6 @@ class DualCache(BaseCache):
|
|||
|
||||
return sublist_keys, previous_access_times
|
||||
|
||||
def reserve_redis_batch_reads(self, keys: Sequence[str]) -> tuple[list[str], dict[str, float | None]]:
|
||||
"""Reserve the memory-missed keys whose throttled Redis reads are due, as a batch read would."""
|
||||
if self.redis_cache is None:
|
||||
return [], {}
|
||||
key_list: Final = list(keys)
|
||||
memory: Final = self.in_memory_cache
|
||||
in_memory_result: Final = (
|
||||
None
|
||||
if memory is None # pyright: ignore[reportUnnecessaryComparison] # handle an absent in-memory tier
|
||||
else memory.batch_get_cache(key_list)
|
||||
)
|
||||
result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in key_list)
|
||||
return self._reserve_redis_batch_keys(time.time(), key_list, result)
|
||||
|
||||
def _rollback_redis_batch_key_reservations(self, previous_access_times: dict[str, float | None]) -> None:
|
||||
with self._last_redis_batch_access_time_lock:
|
||||
for key, previous_time in previous_access_times.items():
|
||||
|
|
@ -493,28 +466,6 @@ class DualCache(BaseCache):
|
|||
verbose_logger, logging.ERROR, "LiteLLM Cache: exception in async add_cache", e, with_traceback=True
|
||||
)
|
||||
|
||||
async def async_set_cache_pre_call(self, key: str, value: object, ttl: float | None) -> BatchResult[None] | None:
|
||||
"""Memory now, the Redis SET on the request's pipeline, sent with the next read any caller awaits; None
|
||||
when no pipeline is open, so the caller takes its direct path."""
|
||||
batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache)
|
||||
return None if batch is None else await self._set_on_batch(batch, key, value, ttl)
|
||||
|
||||
async def async_delete_cache_pre_call(self, key: str) -> Awaitable[None] | None:
|
||||
"""Memory now, the Redis DEL on the request's pipeline; None when no pipeline is open, so the caller
|
||||
takes its direct path."""
|
||||
batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache)
|
||||
if batch is None:
|
||||
return None
|
||||
if self.in_memory_cache is not None:
|
||||
self.in_memory_cache.delete_cache(key)
|
||||
return batch.delete(key)
|
||||
|
||||
async def _set_on_batch(self, batch: RedisBatch, key: str, value: object, ttl: float | None) -> BatchResult[None]:
|
||||
effective_ttl: Final = self.default_in_memory_ttl if ttl is None else ttl
|
||||
if self.in_memory_cache is not None:
|
||||
await self.in_memory_cache.async_set_cache(key, value, ttl=effective_ttl)
|
||||
return batch.set(key, value, effective_ttl)
|
||||
|
||||
# async_batch_set_cache
|
||||
async def async_set_cache_pipeline(
|
||||
self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs
|
||||
|
|
@ -582,41 +533,6 @@ class DualCache(BaseCache):
|
|||
)
|
||||
return result
|
||||
|
||||
async def async_increment_cache_post_call(
|
||||
self,
|
||||
key: str,
|
||||
value: float,
|
||||
ttl: int | None,
|
||||
parent_otel_span: Span | None = None,
|
||||
) -> None:
|
||||
"""Memory is incremented now; the Redis increment rides the request's post-call pipeline when one is
|
||||
open, and runs on its own as ``async_increment_cache`` otherwise."""
|
||||
await self.async_increment_cache_pipeline_post_call(
|
||||
(RedisPipelineIncrementOperation(key=key, increment_value=value, ttl=ttl),), parent_otel_span
|
||||
)
|
||||
|
||||
async def async_increment_cache_pipeline_post_call(
|
||||
self,
|
||||
increment_list: Sequence["RedisPipelineIncrementOperation"],
|
||||
parent_otel_span: Span | None = None,
|
||||
) -> None:
|
||||
batch: Final = None if self.redis_cache is None else active_post_call_redis_batch(self.redis_cache)
|
||||
operations: Final = list(increment_list)
|
||||
if batch is None:
|
||||
await self.async_increment_cache_pipeline(operations, parent_otel_span=parent_otel_span)
|
||||
return
|
||||
try:
|
||||
if self.in_memory_cache is not None:
|
||||
await self.in_memory_cache.async_increment_pipeline(
|
||||
increment_list=operations, parent_otel_span=parent_otel_span
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # same tolerance as async_increment_cache_pipeline
|
||||
log_redis_failure(verbose_logger, logging.WARNING, "in-memory increment failed", e)
|
||||
for operation in increment_list:
|
||||
batch.increment(operation["key"], operation["increment_value"], operation["ttl"]).on_settled(
|
||||
_log_deferred_increment_failure
|
||||
)
|
||||
|
||||
async def async_increment_cache_pipeline(
|
||||
self,
|
||||
increment_list: list["RedisPipelineIncrementOperation"],
|
||||
|
|
|
|||
|
|
@ -1,453 +1,24 @@
|
|||
"""One Redis pipeline for several independent operations, each with its own result and its own failure.
|
||||
|
||||
A ``RedisBatch`` collects MGETs, Lua scripts and increments declared by unrelated callers and sends them
|
||||
in one ``pipeline(transaction=False)`` round trip. Every declaration returns an awaitable; awaiting one
|
||||
flushes whatever has been declared so far, so callers keep their existing ``await`` shape and their own
|
||||
error handling while sharing the wire. Redis Cluster clients run each operation on its own, as before:
|
||||
a cluster pipeline is per node anyway and the existing per-operation paths already group by slot.
|
||||
"""
|
||||
"""The current request's native Redis batches: a pre-call batch for the reads it needs before answering and a
|
||||
post-call batch for the writes nobody waits on, both owned by the native ``NativeRequestRedis``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
|
||||
from contextvars import ContextVar, Token, copy_context
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from collections.abc import Awaitable, Callable, Generator, Sequence
|
||||
from contextvars import ContextVar, Token
|
||||
from enum import Enum, auto
|
||||
from types import MappingProxyType, TracebackType
|
||||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
|
||||
from litellm._internal_context import (
|
||||
REDIS_FAMILIES_METADATA_KEY,
|
||||
current_service_target,
|
||||
service_caller,
|
||||
service_target,
|
||||
)
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.redis_cache import (
|
||||
RedisCache,
|
||||
_get_call_stack_info, # pyright: ignore[reportPrivateUsage] # same caller chain every RedisCache method reports
|
||||
_run_under_circuit_breaker, # pyright: ignore[reportPrivateUsage] # same health signal as every RedisCache method
|
||||
log_redis_failure,
|
||||
)
|
||||
from litellm.caching.redis_cluster_cache import RedisClusterCache
|
||||
from litellm.types.services import ServiceTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.rust_bridge._native import NativeRequestRedis, NativeSend
|
||||
|
||||
_T = TypeVar("_T")
|
||||
_T_co: Final = TypeVar("_T_co", covariant=True)
|
||||
_ScriptArg = str | bytes | int | float
|
||||
SettledHook = Callable[[asyncio.Future[_T]], Awaitable[None] | None]
|
||||
POST_CALL_FLUSH_DEADLINE_SECONDS: Final = 1.0
|
||||
|
||||
|
||||
class RegisteredScript(Protocol):
|
||||
def __call__(self, keys: Sequence[str], args: Sequence[_ScriptArg]) -> Awaitable[object]: ...
|
||||
|
||||
|
||||
class _RedisPipeline(Protocol):
|
||||
def mget(self, keys: Sequence[str]) -> object: ...
|
||||
def evalsha(self, sha: str, numkeys: int, *keys_and_args: _ScriptArg) -> object: ...
|
||||
def incrbyfloat(self, name: str, amount: float) -> object: ...
|
||||
def expire(self, name: str, time: timedelta) -> object: ...
|
||||
def set(self, name: str, value: str, ex: timedelta | None = None) -> object: ...
|
||||
def delete(self, *names: str) -> object: ...
|
||||
async def execute(self, raise_on_error: bool = True) -> list[object]: ...
|
||||
|
||||
|
||||
class _Op(Generic[_T]):
|
||||
"""One declared operation: how many pipeline replies it consumes, how to turn them into a result, and
|
||||
how to run on its own when the batch cannot pipeline (cluster client, or a reply the pipeline cannot
|
||||
settle, like NOSCRIPT)."""
|
||||
|
||||
__slots__ = ("caller", "future", "settled_hooks", "target")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.future: Final[asyncio.Future[_T]] = asyncio.get_running_loop().create_future()
|
||||
self.future.add_done_callback(_mark_retrieved)
|
||||
self.settled_hooks: Final[list[SettledHook[_T]]] = [] # mutable-ok: append-only registry
|
||||
self.target: Final = current_service_target()
|
||||
self.caller: Final = _get_call_stack_info()
|
||||
|
||||
async def run_settled_hooks(self) -> None:
|
||||
for hook in self.settled_hooks:
|
||||
await self._run_settled_hook(hook)
|
||||
|
||||
async def _run_settled_hook(self, hook: SettledHook[_T]) -> None:
|
||||
try:
|
||||
follow_up: Final = hook(self.future)
|
||||
if follow_up is not None:
|
||||
await follow_up
|
||||
except Exception as e: # noqa: BLE001 # one owner's follow-up must not stop the others
|
||||
verbose_logger.warning("redis batch settled hook failed: %s", e)
|
||||
|
||||
def enqueue(self, pipe: _RedisPipeline) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
def resolve(self, replies: Sequence[object]) -> _T:
|
||||
raise NotImplementedError
|
||||
|
||||
async def run_alone(self) -> _T:
|
||||
raise NotImplementedError
|
||||
|
||||
def settle(self, replies: Sequence[object]) -> Awaitable[None] | None:
|
||||
"""Resolve from pipeline replies; return a coroutine when the op has to be retried on its own."""
|
||||
failure: Final = next((reply for reply in replies if isinstance(reply, Exception)), None)
|
||||
if failure is None:
|
||||
try:
|
||||
self.future.set_result(self.resolve(replies))
|
||||
except Exception as e: # noqa: BLE001 # a reply this op cannot decode fails this op alone
|
||||
self.future.set_exception(e)
|
||||
return None
|
||||
if _is_missing_script(failure):
|
||||
return self._settle_alone()
|
||||
self.future.set_exception(failure)
|
||||
return None
|
||||
|
||||
async def _settle_alone(self) -> None:
|
||||
try:
|
||||
with service_target(self.target), service_caller(self.caller):
|
||||
self.future.set_result(await self.run_alone())
|
||||
except Exception as e: # noqa: BLE001 # the declaring caller owns the failure of its own operation
|
||||
self.future.set_exception(e)
|
||||
|
||||
|
||||
def _is_missing_script(failure: Exception) -> bool:
|
||||
"""Imported lazily: this module is reachable from a base ``import litellm`` while redis is not a base dependency."""
|
||||
from redis.exceptions import NoScriptError
|
||||
|
||||
return isinstance(failure, NoScriptError)
|
||||
|
||||
|
||||
def _mark_retrieved(future: asyncio.Future[object]) -> None:
|
||||
"""A caller that stops awaiting (cancelled request) must not leave an 'exception never retrieved' log."""
|
||||
if not future.cancelled():
|
||||
future.exception()
|
||||
|
||||
|
||||
class _MGet(_Op[Mapping[str, object]]):
|
||||
__slots__ = ("_keys", "_redis_cache")
|
||||
|
||||
def __init__(self, redis_cache: RedisCache, keys: Sequence[str]) -> None:
|
||||
super().__init__()
|
||||
self._redis_cache: Final = redis_cache
|
||||
self._keys: Final[tuple[str, ...]] = tuple(dict.fromkeys(keys))
|
||||
|
||||
def enqueue(self, pipe: _RedisPipeline) -> int:
|
||||
pipe.mget(tuple(self._redis_cache.check_and_fix_namespace(key=key) for key in self._keys))
|
||||
return 1
|
||||
|
||||
def resolve(self, replies: Sequence[object]) -> Mapping[str, object]:
|
||||
values: Final = replies[0]
|
||||
if not isinstance(values, (list, tuple)):
|
||||
raise TypeError(f"MGET reply is not a list: {type(values).__name__}")
|
||||
return MappingProxyType(
|
||||
{key: self._redis_cache._get_cache_logic(value) for key, value in zip(self._keys, values)} # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # shared decode with async_batch_get_cache
|
||||
)
|
||||
|
||||
async def run_alone(self) -> Mapping[str, object]:
|
||||
found: Mapping[str, object] = await self._redis_cache.async_batch_get_cache(key_list=list(self._keys)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped cache API
|
||||
if any(key not in found for key in self._keys):
|
||||
raise ConnectionError("batch get did not return every key")
|
||||
return found
|
||||
|
||||
|
||||
class _Script(_Op[object]):
|
||||
__slots__ = ("_args", "_keys", "_redis_cache", "_run", "_sha")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redis_cache: RedisCache,
|
||||
source: str,
|
||||
run: RegisteredScript,
|
||||
keys: Sequence[str],
|
||||
args: Sequence[_ScriptArg],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._redis_cache: Final = redis_cache
|
||||
self._sha: Final = hashlib.sha1(source.encode()).hexdigest() # noqa: S324 # EVALSHA identifies scripts by SHA-1
|
||||
self._run: Final = run
|
||||
self._keys: Final[tuple[str, ...]] = tuple(keys)
|
||||
self._args: Final[tuple[_ScriptArg, ...]] = tuple(args)
|
||||
|
||||
def enqueue(self, pipe: _RedisPipeline) -> int:
|
||||
namespaced: Final = tuple(self._redis_cache.check_and_fix_namespace(key=key) for key in self._keys)
|
||||
pipe.evalsha(self._sha, len(namespaced), *namespaced, *self._args)
|
||||
return 1
|
||||
|
||||
def resolve(self, replies: Sequence[object]) -> object:
|
||||
return replies[0]
|
||||
|
||||
async def run_alone(self) -> object:
|
||||
return await self._run(keys=self._keys, args=self._args)
|
||||
|
||||
|
||||
class _Increment(_Op[float]):
|
||||
__slots__ = ("_key", "_redis_cache", "_ttl", "_value")
|
||||
|
||||
def __init__(self, redis_cache: RedisCache, key: str, value: float, ttl: int | None) -> None:
|
||||
super().__init__()
|
||||
self._redis_cache: Final = redis_cache
|
||||
self._key: Final = key
|
||||
self._value: Final = value
|
||||
self._ttl: Final = ttl
|
||||
|
||||
def enqueue(self, pipe: _RedisPipeline) -> int:
|
||||
name: Final = self._redis_cache.check_and_fix_namespace(key=self._key)
|
||||
pipe.incrbyfloat(name, self._value)
|
||||
if self._ttl is None:
|
||||
return 1
|
||||
pipe.expire(name, timedelta(seconds=self._ttl))
|
||||
return 2
|
||||
|
||||
def resolve(self, replies: Sequence[object]) -> float:
|
||||
reply: Final = replies[0]
|
||||
if not isinstance(reply, (int, float, str, bytes)):
|
||||
raise TypeError(f"INCRBYFLOAT reply is not numeric: {type(reply).__name__}")
|
||||
return float(reply)
|
||||
|
||||
async def run_alone(self) -> float:
|
||||
value: object = await self._redis_cache.async_increment(key=self._key, value=self._value, ttl=self._ttl) # pyright: ignore[reportUnknownMemberType] # untyped cache API
|
||||
if not isinstance(value, (int, float)):
|
||||
raise TypeError(f"increment did not return a number: {type(value).__name__}")
|
||||
return float(value)
|
||||
|
||||
|
||||
class _Set(_Op[None]):
|
||||
"""SET with the cache's TTL rules, same encoding as ``async_set_cache_pipeline_with_ttls``."""
|
||||
|
||||
__slots__ = ("_key", "_redis_cache", "_ttl", "_value")
|
||||
|
||||
def __init__(self, redis_cache: RedisCache, key: str, value: object, ttl: float | None) -> None:
|
||||
super().__init__()
|
||||
self._redis_cache: Final = redis_cache
|
||||
self._key: Final = key
|
||||
self._value: Final = value
|
||||
self._ttl: Final = ttl
|
||||
|
||||
def enqueue(self, pipe: _RedisPipeline) -> int:
|
||||
ttl: Final = self._redis_cache.get_ttl(ttl=self._ttl) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped cache API
|
||||
pipe.set(
|
||||
self._redis_cache.check_and_fix_namespace(key=self._key),
|
||||
json.dumps(self._value),
|
||||
ex=None if ttl is None else timedelta(seconds=ttl),
|
||||
)
|
||||
return 1
|
||||
|
||||
def resolve(self, replies: Sequence[object]) -> None:
|
||||
return None
|
||||
|
||||
async def run_alone(self) -> None:
|
||||
await self._redis_cache.async_set_cache_pipeline_with_ttls(((self._key, self._value, self._ttl),))
|
||||
|
||||
|
||||
class _Delete(_Op[None]):
|
||||
"""DEL of one key, the pipelined twin of ``async_delete_cache``."""
|
||||
|
||||
__slots__ = ("_key", "_redis_cache")
|
||||
|
||||
def __init__(self, redis_cache: RedisCache, key: str) -> None:
|
||||
super().__init__()
|
||||
self._redis_cache: Final = redis_cache
|
||||
self._key: Final = key
|
||||
|
||||
def enqueue(self, pipe: _RedisPipeline) -> int:
|
||||
pipe.delete(self._redis_cache.check_and_fix_namespace(key=self._key))
|
||||
return 1
|
||||
|
||||
def resolve(self, replies: Sequence[object]) -> None:
|
||||
return None
|
||||
|
||||
async def run_alone(self) -> None:
|
||||
await self._redis_cache.async_delete_cache(self._key)
|
||||
|
||||
|
||||
class BatchResult(Generic[_T]):
|
||||
"""Awaitable handle for one declared operation; awaiting it flushes the batch it belongs to."""
|
||||
|
||||
__slots__ = ("_batch", "_op")
|
||||
|
||||
def __init__(self, batch: RedisBatch, op: _Op[_T]) -> None:
|
||||
self._batch: Final = batch
|
||||
self._op: Final = op
|
||||
|
||||
def __await__(self) -> Generator[object, None, _T]:
|
||||
return self._wait().__await__()
|
||||
|
||||
async def _wait(self) -> _T:
|
||||
if not self._op.future.done():
|
||||
await self._batch.flush()
|
||||
return self._op.future.result()
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
return self._op.future.done()
|
||||
|
||||
def on_settled(self, hook: SettledHook[_T]) -> None:
|
||||
"""For an owner that does not await: runs inside the flush once this operation has its result or
|
||||
failure (or was cancelled with the pipeline), so the flush completes with the follow-up done."""
|
||||
self._op.settled_hooks.append(hook)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RedisBatch:
|
||||
"""Operations declared here go out in one pipeline the next time any of them is awaited or ``flush`` runs."""
|
||||
|
||||
redis_cache: RedisCache
|
||||
name: str = "redis_batch"
|
||||
_pending: list[_Op[object]] = field(default_factory=list) # mutable-ok: drained by flush
|
||||
_flush_hooks: list[Callable[[], None]] = field(default_factory=list) # mutable-ok: append-only registry
|
||||
_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
_misses: set[str] = field(default_factory=set) # mutable-ok: keys an MGET of this request read as absent
|
||||
flushes: int = 0
|
||||
|
||||
def mget(self, keys: Sequence[str]) -> BatchResult[Mapping[str, object]]:
|
||||
op: Final = _MGet(self.redis_cache, keys)
|
||||
op.future.add_done_callback(self._note_misses)
|
||||
return self._declare(op)
|
||||
|
||||
def _note_misses(self, future: asyncio.Future[Mapping[str, object]]) -> None:
|
||||
if future.cancelled() or future.exception() is not None:
|
||||
return
|
||||
self._misses.update(key for key, value in future.result().items() if value is None)
|
||||
|
||||
def read_as_missing(self, key: str) -> bool:
|
||||
"""True when an MGET on this batch already found no value under ``key`` and nothing has set it since,
|
||||
so a per-key GET later in the same request can be answered without another round trip."""
|
||||
return key in self._misses
|
||||
|
||||
def script(
|
||||
self, source: str, run: RegisteredScript, keys: Sequence[str], args: Sequence[_ScriptArg]
|
||||
) -> BatchResult[object]:
|
||||
return self._declare(_Script(self.redis_cache, source, run, keys, args))
|
||||
|
||||
def increment(self, key: str, value: float, ttl: int | None = None) -> BatchResult[float]:
|
||||
return self._declare(_Increment(self.redis_cache, key, value, ttl))
|
||||
|
||||
def set(self, key: str, value: object, ttl: float | None = None) -> BatchResult[None]:
|
||||
self._misses.discard(key)
|
||||
return self._declare(_Set(self.redis_cache, key, value, ttl))
|
||||
|
||||
def delete(self, key: str) -> BatchResult[None]:
|
||||
self._misses.add(key)
|
||||
return self._declare(_Delete(self.redis_cache, key))
|
||||
|
||||
def add_flush_hook(self, hook: Callable[[], None]) -> None:
|
||||
"""Called at the start of every flush so lazily bound readers can declare their keys into the same trip."""
|
||||
self._flush_hooks.append(hook)
|
||||
|
||||
@property
|
||||
def pending(self) -> int:
|
||||
return len(self._pending)
|
||||
|
||||
def _declare(self, op: _Op[_T]) -> BatchResult[_T]:
|
||||
self._pending.append(op) # pyright: ignore[reportArgumentType] # heterogeneous ops share the flush loop
|
||||
return BatchResult(self, op)
|
||||
|
||||
async def flush(self) -> None:
|
||||
async with self._lock:
|
||||
for hook in self._flush_hooks:
|
||||
hook()
|
||||
ops: Final = tuple(self._pending)
|
||||
self._pending.clear()
|
||||
if not ops:
|
||||
return
|
||||
self.flushes += 1
|
||||
try:
|
||||
if isinstance(self.redis_cache, RedisClusterCache):
|
||||
await asyncio.gather(*(op._settle_alone() for op in ops)) # pyright: ignore[reportPrivateUsage] # batch owns its ops
|
||||
else:
|
||||
await self._flush_pipeline(ops)
|
||||
finally:
|
||||
for op in ops:
|
||||
if not op.future.done():
|
||||
op.future.cancel()
|
||||
await asyncio.gather(*(op.run_settled_hooks() for op in ops))
|
||||
|
||||
async def _flush_pipeline(self, ops: Sequence[_Op[object]]) -> None:
|
||||
start_time: Final = time.time()
|
||||
target, metadata = _pipeline_service_event(ops)
|
||||
widths: list[int] = [] # mutable-ok: filled while enqueuing
|
||||
|
||||
async def run() -> list[object]:
|
||||
client: Final = self.redis_cache.init_async_client()
|
||||
async with client.pipeline(transaction=False) as pipe:
|
||||
widths.extend(op.enqueue(pipe) for op in ops)
|
||||
return await pipe.execute(raise_on_error=False)
|
||||
|
||||
try:
|
||||
replies: Final = await _run_under_circuit_breaker(self.redis_cache._health, self.name, run) # pyright: ignore[reportPrivateUsage] # same breaker as the cache's own methods
|
||||
except Exception as e: # noqa: BLE001 # each declaring caller applies its own Redis fallback
|
||||
log_redis_failure(verbose_logger, logging.WARNING, f"{self.name}: pipeline of {len(ops)} ops failed", e)
|
||||
with service_target(target):
|
||||
asyncio.create_task(
|
||||
self.redis_cache.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=time.time() - start_time,
|
||||
error=e,
|
||||
call_type=self.name,
|
||||
start_time=start_time,
|
||||
end_time=time.time(),
|
||||
event_metadata=metadata,
|
||||
)
|
||||
)
|
||||
for op in ops:
|
||||
op.future.set_exception(e)
|
||||
return
|
||||
with service_target(target):
|
||||
asyncio.create_task(
|
||||
self.redis_cache.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=time.time() - start_time,
|
||||
call_type=self.name,
|
||||
start_time=start_time,
|
||||
end_time=time.time(),
|
||||
event_metadata=metadata,
|
||||
)
|
||||
)
|
||||
retries: list[Awaitable[None]] = [] # mutable-ok: collected while slicing replies
|
||||
offset = 0
|
||||
for op, width in zip(ops, widths):
|
||||
retry = op.settle(replies[offset : offset + width])
|
||||
offset += width
|
||||
if retry is not None:
|
||||
retries.append(retry)
|
||||
if retries:
|
||||
await asyncio.gather(*retries)
|
||||
|
||||
|
||||
MIXED_PIPELINE_TARGET: Final = "mixed"
|
||||
|
||||
|
||||
def _pipeline_service_event(ops: Sequence[_Op[object]]) -> tuple[str | None, dict[str, int | str]]:
|
||||
"""The target and metadata of one pipeline flush: the one key family every op was declared under, or
|
||||
``"mixed"`` plus the sorted families when owners of several families share the trip."""
|
||||
families: Final = sorted({op.target for op in ops if op.target is not None})
|
||||
if len(families) > 1:
|
||||
return MIXED_PIPELINE_TARGET, {"op_count": len(ops), REDIS_FAMILIES_METADATA_KEY: ",".join(families)}
|
||||
return next(iter(families), None), {"op_count": len(ops)}
|
||||
|
||||
|
||||
def _backend_key(redis_cache: RedisCache) -> object:
|
||||
"""Two ``RedisCache`` instances built from the same connection settings and namespace talk to the same server
|
||||
under the same key prefix, so the proxy's cache and the router's cache share one pipeline (the router gets its
|
||||
port as a string, hence the ``str`` comparison); a cache whose settings cannot be compared (a test double) gets
|
||||
its own."""
|
||||
try:
|
||||
settings: Final = tuple(sorted((str(k), str(v)) for k, v in redis_cache.redis_kwargs.items() if v is not None)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportUnknownArgumentType] # untyped cache API
|
||||
except AttributeError:
|
||||
return ("instance", id(redis_cache))
|
||||
return (type(redis_cache), redis_cache.namespace, settings)
|
||||
|
||||
|
||||
class NativePending(Protocol[_T_co]):
|
||||
def resolve(self) -> Awaitable[_T_co]: ...
|
||||
|
||||
|
|
@ -511,30 +82,22 @@ def _native_send(phase: RequestPhase, request: NativeRequestRedis) -> NativeSend
|
|||
|
||||
|
||||
_open_post_call: Final[weakref.WeakSet[RequestRedisBatches]] = weakref.WeakSet()
|
||||
"""Requests whose post-call batch still holds declared ops, so a shutdown can send them before Redis goes away."""
|
||||
"""Requests whose post-call batch may still hold declared ops, so a shutdown can send them before Redis goes away."""
|
||||
|
||||
|
||||
class RequestRedisBatches:
|
||||
"""One ``RedisBatch`` per Redis backend for the current request, so readers of different caches that
|
||||
share a server (the proxy's and the router's) share the pipeline.
|
||||
|
||||
The post-call batches hold the writes nothing waits on (counters, token scripts, the response cache).
|
||||
They flush once, when the success or failure callbacks have all run, or at ``post_call_deadline``
|
||||
seconds after the first declaration when no callback phase closes them."""
|
||||
"""The request's native batches. The post-call batch goes out once, when the success or failure callbacks have
|
||||
all run, or ``post_call_deadline`` seconds after its first declaration when no callback phase closes it."""
|
||||
|
||||
__slots__ = (
|
||||
"__weakref__",
|
||||
"_batches",
|
||||
"_deadline",
|
||||
"_deadline_flush",
|
||||
"_native",
|
||||
"_native_context",
|
||||
"_native_factory",
|
||||
"_native_flush_hooks",
|
||||
"_native_misses",
|
||||
"_native_send_factory",
|
||||
"_post_call",
|
||||
"_post_call_settlements",
|
||||
"_post_call_unsent",
|
||||
"post_call_deadline",
|
||||
"prefetched",
|
||||
)
|
||||
|
|
@ -545,54 +108,27 @@ class RequestRedisBatches:
|
|||
native_factory: NativeRequestFactory = _native_request,
|
||||
native_send_factory: NativeSendFactory = _native_send,
|
||||
) -> None:
|
||||
self._batches: Final[dict[object, RedisBatch]] = {} # mutable-ok: lazily filled per backend
|
||||
self._post_call: Final[dict[object, RedisBatch]] = {} # mutable-ok: lazily filled per backend
|
||||
self._post_call_settlements: tuple[PostCallSettlement, ...] = ()
|
||||
self._post_call_unsent = False
|
||||
self.post_call_deadline: Final = post_call_deadline
|
||||
self._native_factory: Final = native_factory
|
||||
self._native_send_factory: Final = native_send_factory
|
||||
self._native: NativeRequestRedis | None = None
|
||||
self._native_context: Final = copy_context()
|
||||
self._native_flush_hooks: Final[list[Callable[[], None]]] = [] # mutable-ok: append-only registry
|
||||
self._native_misses: frozenset[str] = frozenset()
|
||||
self._deadline: asyncio.TimerHandle | None = None
|
||||
self._deadline_flush: asyncio.Task[None] | None = None
|
||||
# Reads declared early for a consumer that runs later in the request, keyed by consumer name.
|
||||
self.prefetched: Final[dict[str, object]] = {} # mutable-ok: armed pre-admission, taken at use
|
||||
|
||||
def batch(self, redis_cache: RedisCache) -> RedisBatch:
|
||||
key: Final = _backend_key(redis_cache)
|
||||
batch = self._batches.get(key)
|
||||
if batch is None:
|
||||
batch = RedisBatch(redis_cache, name="request_redis_batch")
|
||||
self._batches[key] = batch
|
||||
return batch
|
||||
|
||||
def post_call(self, redis_cache: RedisCache) -> RedisBatch:
|
||||
key: Final = _backend_key(redis_cache)
|
||||
existing: Final = self._post_call.get(key)
|
||||
batch: Final = (
|
||||
existing
|
||||
if existing is not None
|
||||
else self._post_call.setdefault(key, RedisBatch(redis_cache, name="post_call_redis_batch"))
|
||||
)
|
||||
if self._deadline is None:
|
||||
self._deadline = asyncio.get_running_loop().call_later(self.post_call_deadline, self._flush_on_deadline)
|
||||
_open_post_call.add(self)
|
||||
return batch
|
||||
|
||||
def native(self) -> NativeRequestRedis:
|
||||
"""The request's native batches, built on first use so requests that touch no native store pay nothing.
|
||||
|
||||
They keep a copy of the context they are built in, and the cycle collector cannot see through native
|
||||
objects, so they are built in the context this scope was opened from, which cannot reach the scope."""
|
||||
"""The request's native batches, built on first use so requests that touch no native store pay nothing."""
|
||||
if self._native is None:
|
||||
self._native = self._native_context.run(self._native_factory, self.post_call_deadline)
|
||||
self._native = self._native_factory(self.post_call_deadline)
|
||||
return self._native
|
||||
|
||||
def native_post_call(self) -> NativeRequestRedis:
|
||||
"""``native`` for a post-call declaration, so a shutdown drain also flushes it."""
|
||||
_open_post_call.add(self)
|
||||
self._post_call_unsent = True
|
||||
return self.native()
|
||||
|
||||
def native_send(self, phase: RequestPhase) -> NativeSend:
|
||||
|
|
@ -612,7 +148,7 @@ class RequestRedisBatches:
|
|||
self._native_flush_hooks.append(hook)
|
||||
|
||||
def note_native_misses(self, keys: Sequence[str]) -> None:
|
||||
"""Keys a native pre-call read found absent (or a native DEL removed), like ``RedisBatch.read_as_missing``."""
|
||||
"""Keys a native pre-call read found absent, or a native DEL removed, so a later GET can skip Redis."""
|
||||
self._native_misses = self._native_misses | frozenset(keys)
|
||||
|
||||
def forget_native_miss(self, key: str) -> None:
|
||||
|
|
@ -628,31 +164,21 @@ class RequestRedisBatches:
|
|||
hook()
|
||||
finally:
|
||||
_active_request_batches.reset(token)
|
||||
await self.flush_pre_call()
|
||||
|
||||
async def flush_pre_call(self) -> None:
|
||||
"""Send the pre-call ops declared so far, such as write-backs nobody awaits before the request scope closes.
|
||||
The pre-call flush hooks are skipped: they declare reads for a reader about to await, and none is waiting."""
|
||||
if self._native is not None:
|
||||
await self._native.flush_pre()
|
||||
|
||||
def _flush_on_deadline(self) -> None:
|
||||
self._deadline = None
|
||||
self._deadline_flush = asyncio.ensure_future(self.flush_post_call())
|
||||
|
||||
async def flush_all(self) -> None:
|
||||
"""Batch whatever is still declared (write-backs nobody awaits) before the request scope closes. The pre-call
|
||||
flush hooks are skipped: they declare reads for later readers, and none are left."""
|
||||
await asyncio.gather(
|
||||
*(batch.flush() for batch in self._batches.values() if batch.pending),
|
||||
*(() if self._native is None else (self._native.flush_pre(),)),
|
||||
)
|
||||
|
||||
async def flush_post_call(self) -> None:
|
||||
"""One pipeline per backend for the post-call writes; the deadline is disarmed since this is that flush."""
|
||||
if self._deadline is not None:
|
||||
self._deadline.cancel()
|
||||
self._deadline = None
|
||||
await asyncio.gather(
|
||||
*(batch.flush() for batch in self._post_call.values() if batch.pending),
|
||||
*(() if self._native is None else (self._native.flush_post(),)),
|
||||
)
|
||||
if not any(batch.pending for batch in self._post_call.values()):
|
||||
"""Send the post-call writes. The request stays registered with the shutdown drain until they are back, so a
|
||||
drain that overlaps this flush waits for it, and an op declared while it runs keeps the request registered."""
|
||||
self._post_call_unsent = False
|
||||
if self._native is not None:
|
||||
await self._native.flush_post()
|
||||
if not self._post_call_unsent:
|
||||
_open_post_call.discard(self)
|
||||
|
||||
def track_post_call_settlement(self, settlement: PostCallSettlement) -> None:
|
||||
|
|
@ -663,36 +189,16 @@ class RequestRedisBatches:
|
|||
def post_call_settlements(self) -> tuple[PostCallSettlement, ...]:
|
||||
return self._post_call_settlements
|
||||
|
||||
@property
|
||||
def batches(self) -> tuple[RedisBatch, ...]:
|
||||
return tuple(self._batches.values())
|
||||
|
||||
|
||||
_active_request_batches: Final[ContextVar[RequestRedisBatches | None]] = ContextVar(
|
||||
"request_redis_batches", default=None
|
||||
)
|
||||
|
||||
|
||||
def active_request_redis_batch(redis_cache: RedisCache) -> RedisBatch | None:
|
||||
"""The request's batch for this backend, or None outside a ``request_redis_batch_scope``."""
|
||||
batches: Final = _active_request_batches.get()
|
||||
if batches is None:
|
||||
return None
|
||||
return batches.batch(redis_cache)
|
||||
|
||||
|
||||
def active_request_redis_batches() -> RequestRedisBatches | None:
|
||||
return _active_request_batches.get()
|
||||
|
||||
|
||||
def active_post_call_redis_batch(redis_cache: RedisCache) -> RedisBatch | None:
|
||||
"""The request's post-call batch for this backend, or None outside a ``request_redis_batch_scope``."""
|
||||
batches: Final = _active_request_batches.get()
|
||||
if batches is None:
|
||||
return None
|
||||
return batches.post_call(redis_cache)
|
||||
|
||||
|
||||
async def flush_post_call_redis_batches() -> None:
|
||||
"""Called where the success and failure callbacks of a request have all run."""
|
||||
batches: Final = _active_request_batches.get()
|
||||
|
|
@ -706,7 +212,7 @@ async def drain_post_call_redis_batches() -> None:
|
|||
|
||||
|
||||
class request_redis_batch_scope:
|
||||
"""Redis reads declared inside share one pipeline per backend; nested scopes join the outer one."""
|
||||
"""Redis ops declared inside share the request's native batches; nested scopes join the outer one."""
|
||||
|
||||
__slots__ = ("_native_factory", "_native_send_factory", "_post_call_deadline", "_token")
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import current_service_caller
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
from litellm.constants import (
|
||||
DEFAULT_REDIS_MAJOR_VERSION,
|
||||
|
|
@ -112,7 +111,6 @@ class _SupportsScriptLoad(Protocol):
|
|||
_GENERIC_CALLER_MODULES: Final = frozenset(
|
||||
{
|
||||
__name__,
|
||||
"litellm.caching.redis_batch",
|
||||
"litellm.caching.dual_cache",
|
||||
"litellm.caching.caching",
|
||||
"litellm.rust_bridge.execution",
|
||||
|
|
@ -125,8 +123,6 @@ _GENERIC_CALLER_FRAME_NAMES: Final = frozenset(
|
|||
"wrapper",
|
||||
"_run_under_circuit_breaker",
|
||||
"_run_under_circuit_breaker_sync",
|
||||
"run_alone",
|
||||
"_settle_alone",
|
||||
"get_cache",
|
||||
"set_cache",
|
||||
"async_get_cache",
|
||||
|
|
@ -184,11 +180,9 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
|
|||
Get the function names of the nearest meaningful callers of the cache method.
|
||||
|
||||
Frames that merely forward the call (this module's circuit-breaker guards, the
|
||||
cache facades, the batch pipeline's retry path, generic cache verbs) are
|
||||
skipped, and the walk stops at the event loop, so the chain names the litellm
|
||||
code that wanted the call. When nothing but forwarding frames is found (the call
|
||||
runs in a task of its own, like a batch op retried on the flush) the chain the
|
||||
declaring code threaded through ``service_caller`` is reported, else ``unknown``.
|
||||
cache facades, generic cache verbs) are skipped, and the walk stops at the event
|
||||
loop, so the chain names the litellm code that wanted the call, or ``unknown``
|
||||
when nothing but forwarding frames is found.
|
||||
|
||||
Args:
|
||||
num_frames: Number of previous frames to include (default: 2)
|
||||
|
|
@ -219,9 +213,7 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
|
|||
)
|
||||
function_names: Final = leading_names + further_names
|
||||
|
||||
if function_names:
|
||||
return " <- ".join(function_names)
|
||||
return current_service_caller() or "unknown"
|
||||
return " <- ".join(function_names) if function_names else "unknown"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
|
|
|||
|
|
@ -163,6 +163,8 @@ DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COO
|
|||
DEFAULT_REDIS_BATCH_CACHE_EXPIRY_SECONDS: Final = 10
|
||||
DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5)
|
||||
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
|
||||
DEFAULT_REPLICATE_SYNC_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_SYNC_POLLING_RETRIES", 120)
|
||||
REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES: Final = get_env_int("REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES", 512 * 1024 * 1024)
|
||||
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
|
||||
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
|
||||
|
|
@ -448,6 +450,7 @@ MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body"
|
|||
MCP_ADMISSION_BODY_PEEK_TIMEOUT_SECONDS: Final = 5.0
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
|
||||
REALTIME_METERED_USAGE_KEY: Final = "realtime_metered_usage"
|
||||
FAILED_PROMPT_TOKEN_ESTIMATE_KEY: Final = "failed_prompt_token_estimate"
|
||||
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS: Final = 30
|
||||
OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS: Final = 5
|
||||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
|
||||
|
|
@ -493,6 +496,7 @@ REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_team_spend_upda
|
|||
REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_org_spend_update_buffer"
|
||||
REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_spend_update_buffer"
|
||||
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
|
||||
REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_customer_member_spend_update_buffer"
|
||||
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
|
||||
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
|
||||
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
|
||||
|
|
@ -1997,7 +2001,7 @@ BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = get_env_float(
|
|||
"BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", 60.0
|
||||
)
|
||||
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = get_env_float(
|
||||
"BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 3600.0
|
||||
"BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 7800.0
|
||||
)
|
||||
_background_interaction_cost_polling_env: Final = os.getenv(
|
||||
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ from litellm.llms.openai.cost_calculation import (
|
|||
cost_per_token as openai_cost_per_token,
|
||||
)
|
||||
from litellm.llms.openai.cost_calculation import (
|
||||
video_clip_cost,
|
||||
video_output_cost_per_second,
|
||||
)
|
||||
from litellm.llms.perplexity.cost_calculator import (
|
||||
|
|
@ -126,6 +127,7 @@ from litellm.responses.utils import ResponseAPILoggingUtils
|
|||
from litellm.types.agents import LiteLLMSendMessageResponse
|
||||
from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage
|
||||
from litellm.types.extraction import ExtractionResponse, ExtractionUsage
|
||||
from litellm.types.interactions import InteractionsAPIResponse
|
||||
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
|
|
@ -1144,11 +1146,29 @@ def _served_or_priced_requested_model(
|
|||
if served in (requested, requested.partition("/")[2]):
|
||||
return served
|
||||
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
|
||||
profile_of_served: Final = _is_bedrock_cross_region_profile_of(requested_model, served_model, pricing_provider)
|
||||
if profile_of_served and _resolves_to_rates(requested, pricing_provider):
|
||||
return requested
|
||||
if _resolves_to_rates(served, pricing_provider):
|
||||
return served
|
||||
return requested if _resolves_to_rates(requested, pricing_provider) else served
|
||||
|
||||
|
||||
def _is_bedrock_cross_region_profile_of(profile: str, served_model: str, custom_llm_provider: str | None) -> bool:
|
||||
if custom_llm_provider != "bedrock":
|
||||
return False
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
get_bedrock_cross_region_inference_regions,
|
||||
strip_bedrock_routing_prefix,
|
||||
)
|
||||
|
||||
region, _, profiled_model = strip_bedrock_routing_prefix(profile).partition(".")
|
||||
return (
|
||||
region in get_bedrock_cross_region_inference_regions()
|
||||
and strip_bedrock_routing_prefix(served_model) == profiled_model
|
||||
)
|
||||
|
||||
|
||||
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
|
||||
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
|
||||
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
|
||||
|
|
@ -1187,6 +1207,8 @@ def _get_response_model(completion_response: object) -> str | None:
|
|||
if completion_response is None:
|
||||
return None
|
||||
|
||||
if isinstance(completion_response, InteractionsAPIResponse):
|
||||
return completion_response.model or completion_response.agent
|
||||
if isinstance(completion_response, BaseModel):
|
||||
return getattr(completion_response, "model", None)
|
||||
elif isinstance(completion_response, dict):
|
||||
|
|
@ -2070,25 +2092,21 @@ def completion_cost(
|
|||
litellm_logging_obj, custom_pricing, router_model_id
|
||||
)
|
||||
|
||||
usage_obj = getattr(completion_response, "usage", None)
|
||||
_video_usage_obj: object = attribute_or_none(completion_response, "usage")
|
||||
duration_seconds: float | None = None
|
||||
video_resolution: str | None = None
|
||||
provider_reported_cost: float | None = None
|
||||
if completion_response is not None and usage_obj:
|
||||
# Handle both dict and Pydantic Usage object
|
||||
if isinstance(usage_obj, dict):
|
||||
duration_seconds = usage_obj.get("duration_seconds", None)
|
||||
_vr = usage_obj.get("video_resolution", None)
|
||||
provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
|
||||
_vc = usage_obj.get("video_count", None)
|
||||
else:
|
||||
duration_seconds = getattr(usage_obj, "duration_seconds", None)
|
||||
_vr = getattr(usage_obj, "video_resolution", None)
|
||||
provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
|
||||
_vc = getattr(usage_obj, "video_count", None)
|
||||
if _vr is not None:
|
||||
video_resolution = str(_vr).strip().lower()
|
||||
video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1
|
||||
if completion_response is not None and _video_usage_obj:
|
||||
duration_seconds = _VIDEO_USAGE_NUMBER.validate_python(
|
||||
_video_usage_value(_video_usage_obj, "duration_seconds")
|
||||
)
|
||||
provider_reported_cost = _VIDEO_USAGE_NUMBER.validate_python(
|
||||
_video_usage_value(_video_usage_obj, "provider_reported_cost_usd")
|
||||
)
|
||||
if (
|
||||
video_resolution_value := _video_usage_value(_video_usage_obj, "video_resolution")
|
||||
) is not None:
|
||||
video_resolution = str(video_resolution_value).strip().lower()
|
||||
|
||||
if _video_model_info is None and provider_reported_cost is not None:
|
||||
return float(provider_reported_cost)
|
||||
|
|
@ -2105,7 +2123,7 @@ def completion_cost(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
model_info=_video_model_info,
|
||||
video_resolution=video_resolution,
|
||||
) * video_count + _video_input_image_cost(
|
||||
) * _video_count(_video_usage_obj) + _video_input_image_cost(
|
||||
requested_model or model,
|
||||
custom_llm_provider,
|
||||
_video_model_info,
|
||||
|
|
@ -2118,7 +2136,7 @@ def completion_cost(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
model_info=_video_model_info,
|
||||
video_resolution=video_resolution,
|
||||
)
|
||||
) * _video_count(_video_usage_obj)
|
||||
elif call_type in _SPEECH_CALL_TYPES:
|
||||
prompt_characters = _speech_billed_characters(
|
||||
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
|
||||
|
|
@ -2631,6 +2649,19 @@ def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None
|
|||
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
|
||||
|
||||
|
||||
_VIDEO_USAGE_NUMBER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
||||
|
||||
|
||||
def _video_usage_value(usage: object, field: str) -> object:
|
||||
fields: Final = as_str_mapping(usage)
|
||||
return fields.get(field) if fields is not None else attribute_or_none(usage, field)
|
||||
|
||||
|
||||
def _video_count(usage: object) -> int:
|
||||
count: Final = _video_usage_value(usage, "video_count")
|
||||
return count if type(count) is int and count > 1 else 1
|
||||
|
||||
|
||||
def _video_input_image_cost(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -3122,6 +3153,9 @@ def default_image_cost_calculator(
|
|||
return cost
|
||||
|
||||
|
||||
_VIDEO_COST_INFO: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def default_video_cost_calculator(
|
||||
model: str,
|
||||
duration_seconds: float,
|
||||
|
|
@ -3184,6 +3218,14 @@ def default_video_cost_calculator(
|
|||
if cost_info is None:
|
||||
raise Exception(f"Model not found in cost map for model={model}")
|
||||
|
||||
clip_cost: Final = video_clip_cost(_VIDEO_COST_INFO.validate_python(cost_info), duration_seconds, video_resolution)
|
||||
if clip_cost is not None:
|
||||
return clip_cost
|
||||
|
||||
video_cost: Final = cost_info.get("output_cost_per_video")
|
||||
if video_cost is not None:
|
||||
return video_cost
|
||||
|
||||
# Check for video-specific cost per second first
|
||||
video_cost_per_second: Final = cost_info.get("output_cost_per_video_per_second")
|
||||
if video_cost_per_second is not None:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:ae9575b121f5776e2f04e192064bc2c5665af9d1c425d620dc773aca31698712
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:ae9575b121f5776e2f04e192064bc2c5665af9d1c425d620dc773aca31698712
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
|
|||
|
|
@ -96,25 +96,31 @@
|
|||
/v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*,
|
||||
/v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix.
|
||||
*/}}
|
||||
{{- $gatewayPrefixes := list
|
||||
"/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings"
|
||||
"/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images"
|
||||
"/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning"
|
||||
"/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
|
||||
"/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
|
||||
"/v1/models" "/models" "/openai" "/engines"
|
||||
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" "/api/event_logging"
|
||||
"/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
|
||||
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
|
||||
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
|
||||
"/v1beta" "/interactions"
|
||||
"/anthropic" "/azure" "/azure_ai" "/azure_speech" "/aws" "/bedrock" "/comprehendmedical" "/transcribe" "/cohere" "/gemini" "/google"
|
||||
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
|
||||
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
|
||||
"/toolset"
|
||||
"/v1/realtime" "/realtime"
|
||||
"/health" "/metrics"
|
||||
{{- $gatewayPrefixGroups := list
|
||||
(list "/chat" "/v1/chat") (list "/completions" "/v1/completions") (list "/embeddings" "/v1/embeddings")
|
||||
(list "/moderations" "/v1/moderations") (list "/audio" "/v1/audio") (list "/images" "/v1/images")
|
||||
(list "/files" "/v1/files") (list "/batches" "/v1/batches") (list "/fine_tuning" "/v1/fine_tuning")
|
||||
(list "/fine-tuning" "/v1/fine-tuning") (list "/responses" "/v1/responses") (list "/threads" "/v1/threads")
|
||||
(list "/assistants" "/v1/assistants") (list "/vector_stores" "/v1/vector_stores") (list "/models" "/v1/models")
|
||||
(list "/messages" "/v1/messages") (list "/a2a" "/v1/a2a") (list "/rerank" "/v1/rerank")
|
||||
(list "/decisions" "/v1/decisions") (list "/systemone" "/v1/systemone") (list "/extraction" "/v1/extraction")
|
||||
(list "/ocr" "/v1/ocr") (list "/rag" "/v1/rag") (list "/video" "/v1/video")
|
||||
(list "/videos" "/v1/videos") (list "/search" "/v1/search") (list "/containers" "/v1/containers")
|
||||
(list "/realtime" "/v1/realtime") (list "/live/sessions" "/v1/live/sessions") (list "/v1/skills" "/v1/indexes")
|
||||
(list "/v1/evals" "/v2/rerank") (list "/v1beta" "/v1/memory") (list "/openai" "/engines")
|
||||
(list "/queue/chat" "/api/event_logging") (list "/toolset" "/interactions") (list "/azure" "/anthropic")
|
||||
(list "/azure_ai" "/azure_speech") (list "/aws" "/bedrock") (list "/transcribe" "/comprehendmedical")
|
||||
(list "/cohere" "/gemini") (list "/google" "/gigachat") (list "/vertex_ai" "/vertex-ai")
|
||||
(list "/assemblyai" "/eu.assemblyai") (list "/fal_ai" "/deepgram") (list "/vllm" "/langfuse")
|
||||
(list "/mistral" "/typesafe") (list "/openrouter" "/nvidia_nim") (list "/groq" "/voyage")
|
||||
(list "/cursor" "/milvus") (list "/tinyfish" "/openai_passthrough") (list "/watsonx")
|
||||
(list "/health" "/metrics")
|
||||
-}}
|
||||
{{- $gatewayPrefixes := list -}}
|
||||
{{- range $gatewayPrefixGroups -}}
|
||||
{{- $gatewayPrefixes = concat $gatewayPrefixes . -}}
|
||||
{{- end -}}
|
||||
{{- $gatewayExactPaths := list "/test" "/debug/memory/summary" "/eu_jev/v1/systemone" -}}
|
||||
{{/*
|
||||
/test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its
|
||||
children /test/connection and /test/tools/list are MCP-server management
|
||||
|
|
@ -131,7 +137,16 @@
|
|||
at "/" Prefix would swallow the whole backend management API) instead of
|
||||
adding to it.
|
||||
*/}}
|
||||
{{- $builtinPathKeys := ternary (list "/|Prefix") (list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix") $monolith -}}
|
||||
{{- $builtinPathKeys := list "/|Prefix" -}}
|
||||
{{/*
|
||||
An ALB allows 100 listener rules and the AWS Load Balancer Controller makes
|
||||
one per Ingress path, so on alb each $gatewayPrefixGroups entry is one rule.
|
||||
Its first, shorter prefix is the Ingress path, which keeps any longer child
|
||||
path ahead of it in the controller's longest-first order, and a conditions
|
||||
annotation adds the other prefix to the same rule.
|
||||
*/}}
|
||||
{{- $albGrouping := and (eq $controller "alb") (not $monolith) -}}
|
||||
{{- $gatewayConditions := get (.Values.ingress.annotations | default dict) (printf "alb.ingress.kubernetes.io/conditions.%s" $gatewayName) | default "[]" | trim | trimPrefix "[" | trimSuffix "]" | trim -}}
|
||||
{{- if $hasComponentHosts }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
|
|
@ -198,9 +213,20 @@ metadata:
|
|||
name: {{ include "litellm.fullname" . }}
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
{{- if or .Values.ingress.annotations $albGrouping }}
|
||||
annotations:
|
||||
{{- with .Values.ingress.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if $albGrouping }}
|
||||
{{- range $i, $group := $gatewayPrefixGroups }}
|
||||
alb.ingress.kubernetes.io/actions.litellm-gateway-{{ $i }}: {{ printf `{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"%s","servicePort":"http"}]}}` $gatewayName | quote }}
|
||||
{{- $conditions := compact (list $gatewayConditions (ternary (printf `{"field":"path-pattern","pathPatternConfig":{"values":["%s","%s/*"]}}` (last $group) (last $group)) "" (eq (len $group) 2))) }}
|
||||
{{- with $conditions }}
|
||||
alb.ingress.kubernetes.io/conditions.litellm-gateway-{{ $i }}: {{ printf "[%s]" (join "," .) | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.ingress.className }}
|
||||
|
|
@ -234,21 +260,31 @@ spec:
|
|||
# /test/* MCP management endpoints fall to the backend catch-all.
|
||||
# Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory
|
||||
# gate); the rest of /debug/* stays on the backend.
|
||||
- path: /test
|
||||
{{- range $gatewayExactPaths }}
|
||||
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Exact" .) }}
|
||||
- path: {{ . }}
|
||||
pathType: Exact
|
||||
backend:
|
||||
service:
|
||||
name: {{ $gatewayName }}
|
||||
port:
|
||||
number: {{ $gatewayPort }}
|
||||
- path: /debug/memory/summary
|
||||
pathType: Exact
|
||||
{{- end }}
|
||||
{{- if $albGrouping }}
|
||||
{{- range $i, $group := $gatewayPrefixGroups }}
|
||||
{{- range $group }}
|
||||
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
|
||||
{{- end }}
|
||||
- path: {{ first $group }}
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ $gatewayName }}
|
||||
name: litellm-gateway-{{ $i }}
|
||||
port:
|
||||
number: {{ $gatewayPort }}
|
||||
{{- range $gatewayPrefixes }}
|
||||
name: use-annotation
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range (ternary list $gatewayPrefixes $albGrouping) }}
|
||||
{{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
|
||||
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
|
||||
- path: {{ . }}
|
||||
|
|
|
|||
|
|
@ -18,16 +18,8 @@ tests:
|
|||
name: RELEASE-NAME-litellm-ui
|
||||
port:
|
||||
number: 3000
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /eu.assemblyai
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: RELEASE-NAME-litellm-gateway
|
||||
port:
|
||||
number: 4000
|
||||
- notExists:
|
||||
path: spec.rules[0].http.paths[?(@.backend.service.port.name == "use-annotation" && @.pathType != "Prefix")]
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
|
|
@ -39,6 +31,39 @@ tests:
|
|||
port:
|
||||
number: 3000
|
||||
|
||||
- it: fits the AWS Load Balancer Controller's one rule per path under the default 100 rules per ALB
|
||||
set:
|
||||
ingress.enabled: true
|
||||
asserts:
|
||||
- notExists:
|
||||
path: spec.rules[0].http.paths[100]
|
||||
|
||||
- it: routes each grouped gateway prefix to the gateway Service through a use-annotation action on alb
|
||||
set:
|
||||
ingress.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["alb.ingress.kubernetes.io/actions.litellm-gateway-0"]
|
||||
value: '{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"RELEASE-NAME-litellm-gateway","servicePort":"http"}]}}'
|
||||
- matchRegex:
|
||||
path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"]
|
||||
pattern: '^\[\{"field":"path-pattern","pathPatternConfig":\{"values":\["(/[^"*]+)","/[^"*]+/\*"\]\}\}\]$'
|
||||
- notExists:
|
||||
path: spec.rules[0].http.paths[?(@.pathType == "Prefix" && @.backend.service.name == "RELEASE-NAME-litellm-gateway")]
|
||||
|
||||
- it: keeps an operator's conditions for the gateway Service on every grouped alb rule
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.annotations:
|
||||
alb.ingress.kubernetes.io/conditions.RELEASE-NAME-litellm-gateway: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]'
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"]
|
||||
pattern: '^\[\{"field":"source-ip","sourceIpConfig":\{"values":\["10\.0\.0\.0/8"\]\}\},\{"field":"path-pattern",'
|
||||
- equal:
|
||||
path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-50"]
|
||||
value: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]'
|
||||
|
||||
- it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
|
||||
set:
|
||||
ingress.enabled: true
|
||||
|
|
@ -107,6 +132,26 @@ tests:
|
|||
name: RELEASE-NAME-litellm-gateway
|
||||
port:
|
||||
number: 4000
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /eu_jev/v1/systemone
|
||||
pathType: Exact
|
||||
backend:
|
||||
service:
|
||||
name: RELEASE-NAME-litellm-gateway
|
||||
port:
|
||||
number: 4000
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
content:
|
||||
path: /openrouter
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: RELEASE-NAME-litellm-gateway
|
||||
port:
|
||||
number: 4000
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[-1]
|
||||
value:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ tests:
|
|||
- it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.controller: nginx
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[-1]
|
||||
|
|
@ -33,12 +34,12 @@ tests:
|
|||
set:
|
||||
ingress.enabled: true
|
||||
ingress.extraPaths:
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[-2]
|
||||
value:
|
||||
path: /watsonx
|
||||
path: /hosted_vllm
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
|
|
@ -59,8 +60,9 @@ tests:
|
|||
- it: keeps every built-in path when extra paths are supplied
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.controller: nginx
|
||||
ingress.extraPaths:
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.rules[0].http.paths
|
||||
|
|
@ -117,7 +119,7 @@ tests:
|
|||
set:
|
||||
ingress.enabled: true
|
||||
ingress.extraPaths:
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
service: gateway
|
||||
- path: /my-passthrough
|
||||
pathType: Exact
|
||||
|
|
@ -129,7 +131,7 @@ tests:
|
|||
- equal:
|
||||
path: spec.rules[0].http.paths[-4]
|
||||
value:
|
||||
path: /watsonx
|
||||
path: /hosted_vllm
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
|
|
@ -164,7 +166,7 @@ tests:
|
|||
backend.service.port: 8001
|
||||
ui.service.port: 8080
|
||||
ingress.extraPaths:
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
- path: /my-passthrough
|
||||
service: backend
|
||||
- path: /brand.txt
|
||||
|
|
@ -184,21 +186,21 @@ tests:
|
|||
set:
|
||||
ingress.enabled: true
|
||||
ingress.extraPaths:
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
service: proxy
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
|
||||
errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown service "proxy", expected one of backend, gateway, ui'
|
||||
|
||||
- it: rejects an entry whose pathType is not a kubernetes pathType
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.extraPaths:
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
pathType: prefix
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
|
||||
errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
|
||||
|
||||
- it: rejects an entry with no path
|
||||
set:
|
||||
|
|
@ -322,7 +324,7 @@ tests:
|
|||
set:
|
||||
ingress.enabled: true
|
||||
ingress.extraPaths:
|
||||
- /watsonx
|
||||
- /hosted_vllm
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"
|
||||
|
|
@ -372,7 +374,7 @@ tests:
|
|||
- path: /moyai
|
||||
serviceName: moyai-ingress
|
||||
servicePort: http
|
||||
- path: /watsonx
|
||||
- path: /hosted_vllm
|
||||
service: gateway
|
||||
asserts:
|
||||
- equal:
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ tests:
|
|||
- it: componentized mode still splits traffic across ui, gateway and backend Services
|
||||
set:
|
||||
ingress.enabled: true
|
||||
ingress.controller: nginx
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.rules[0].http.paths[0].backend.service.name
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ ingress:
|
|||
# The target component only answers paths its own route allowlist keeps, so
|
||||
# a path here still has to be one that component serves.
|
||||
extraPaths: []
|
||||
# - path: /watsonx
|
||||
# - path: /hosted_vllm
|
||||
# pathType: Prefix
|
||||
# service: gateway
|
||||
|
||||
|
|
|
|||
|
|
@ -1,326 +1,334 @@
|
|||
"""
|
||||
PagerDuty Alerting Integration
|
||||
|
||||
Handles two types of alerts:
|
||||
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
|
||||
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
|
||||
|
||||
Note: This is a Free feature on the regular litellm docker image.
|
||||
|
||||
However, this is under the enterprise license
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching import DualCache
|
||||
from litellm.exceptions import MissingMessageRoleError
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.integrations.pagerduty import (
|
||||
AlertingConfig,
|
||||
PagerDutyInternalEvent,
|
||||
PagerDutyPayload,
|
||||
PagerDutyRequestBody,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
|
||||
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
|
||||
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
|
||||
|
||||
|
||||
class PagerDutyAlerting(SlackAlerting):
|
||||
"""
|
||||
Tracks failed requests and hanging requests separately.
|
||||
If threshold is crossed for either type, triggers a PagerDuty alert.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
|
||||
):
|
||||
super().__init__()
|
||||
_api_key = os.getenv("PAGERDUTY_API_KEY")
|
||||
if not _api_key:
|
||||
raise ValueError("PAGERDUTY_API_KEY is not set")
|
||||
|
||||
self.api_key: str = _api_key
|
||||
alerting_args = alerting_args or {}
|
||||
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
|
||||
failure_threshold=alerting_args.get(
|
||||
"failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD
|
||||
),
|
||||
failure_threshold_window_seconds=alerting_args.get(
|
||||
"failure_threshold_window_seconds",
|
||||
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
|
||||
),
|
||||
hanging_threshold_seconds=alerting_args.get(
|
||||
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
|
||||
),
|
||||
hanging_threshold_window_seconds=alerting_args.get(
|
||||
"hanging_threshold_window_seconds",
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
|
||||
),
|
||||
)
|
||||
|
||||
# Separate storage for failures vs. hangs
|
||||
self._failure_events: List[PagerDutyInternalEvent] = []
|
||||
self._hanging_events: List[PagerDutyInternalEvent] = []
|
||||
|
||||
# ------------------ MAIN LOGIC ------------------ #
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Record a failure event. Only send an alert to PagerDuty if the
|
||||
configured *failure* threshold is exceeded in the specified window.
|
||||
"""
|
||||
if isinstance(kwargs.get("exception"), MissingMessageRoleError):
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
|
||||
"standard_logging_object"
|
||||
)
|
||||
if not standard_logging_payload:
|
||||
raise ValueError(
|
||||
"standard_logging_object is required for PagerDutyAlerting"
|
||||
)
|
||||
|
||||
# Extract error details
|
||||
error_info: Optional[StandardLoggingPayloadErrorInformation] = (
|
||||
standard_logging_payload.get("error_information") or {}
|
||||
)
|
||||
_meta = standard_logging_payload.get("metadata") or {}
|
||||
|
||||
self._failure_events.append(
|
||||
PagerDutyInternalEvent(
|
||||
failure_event_type="failed_response",
|
||||
timestamp=now,
|
||||
error_class=error_info.get("error_class"),
|
||||
error_code=error_info.get("error_code"),
|
||||
error_llm_provider=error_info.get("llm_provider"),
|
||||
user_api_key_hash=_meta.get("user_api_key_hash"),
|
||||
user_api_key_alias=_meta.get("user_api_key_alias"),
|
||||
user_api_key_spend=_meta.get("user_api_key_spend"),
|
||||
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
|
||||
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
|
||||
user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
|
||||
user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
|
||||
user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
|
||||
user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
|
||||
user_api_key_org_id=_meta.get("user_api_key_org_id"),
|
||||
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
|
||||
user_api_key_team_id=_meta.get("user_api_key_team_id"),
|
||||
user_api_key_project_id=_meta.get("user_api_key_project_id"),
|
||||
user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
|
||||
user_api_key_user_id=_meta.get("user_api_key_user_id"),
|
||||
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
|
||||
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
|
||||
user_api_key_user_email=_meta.get("user_api_key_user_email"),
|
||||
user_api_key_request_route=_meta.get("user_api_key_request_route"),
|
||||
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
|
||||
)
|
||||
)
|
||||
|
||||
# Prune + Possibly alert
|
||||
window_seconds = self.pagerduty_alerting_args.get(
|
||||
"failure_threshold_window_seconds", 60
|
||||
)
|
||||
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
|
||||
|
||||
# If threshold is crossed, send PD alert for failures
|
||||
await self._send_alert_if_thresholds_crossed(
|
||||
events=self._failure_events,
|
||||
window_seconds=window_seconds,
|
||||
threshold=threshold,
|
||||
alert_prefix="High LLM API Failure Rate",
|
||||
)
|
||||
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
cache: DualCache,
|
||||
data: dict,
|
||||
call_type: CallTypesLiteral,
|
||||
) -> Optional[Union[Exception, str, dict]]:
|
||||
"""
|
||||
Example of detecting hanging requests by waiting a given threshold.
|
||||
If the request didn't finish by then, we treat it as 'hanging'.
|
||||
"""
|
||||
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
|
||||
asyncio.create_task(
|
||||
self.hanging_response_handler(
|
||||
request_data=data, user_api_key_dict=user_api_key_dict
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
async def hanging_response_handler(
|
||||
self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth
|
||||
):
|
||||
"""
|
||||
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
|
||||
If not, we classify it as a hanging request.
|
||||
"""
|
||||
verbose_logger.debug(
|
||||
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
|
||||
)
|
||||
await asyncio.sleep(
|
||||
self.pagerduty_alerting_args.get(
|
||||
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
|
||||
)
|
||||
)
|
||||
|
||||
if await self._request_is_completed(request_data=request_data):
|
||||
return # It's not hanging if completed
|
||||
|
||||
# Otherwise, record it as hanging
|
||||
self._hanging_events.append(
|
||||
PagerDutyInternalEvent(
|
||||
failure_event_type="hanging_response",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
error_class="HangingRequest",
|
||||
error_code="HangingRequest",
|
||||
error_llm_provider="HangingRequest",
|
||||
user_api_key_hash=user_api_key_dict.api_key,
|
||||
user_api_key_alias=user_api_key_dict.key_alias,
|
||||
user_api_key_spend=user_api_key_dict.spend,
|
||||
user_api_key_max_budget=user_api_key_dict.max_budget,
|
||||
user_api_key_budget_reset_at=(
|
||||
user_api_key_dict.budget_reset_at.isoformat()
|
||||
if user_api_key_dict.budget_reset_at
|
||||
else None
|
||||
),
|
||||
user_api_key_user_spend=user_api_key_dict.user_spend,
|
||||
user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
|
||||
user_api_key_team_spend=user_api_key_dict.team_spend,
|
||||
user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
|
||||
user_api_key_org_id=user_api_key_dict.org_id,
|
||||
user_api_key_org_alias=user_api_key_dict.organization_alias,
|
||||
user_api_key_team_id=user_api_key_dict.team_id,
|
||||
user_api_key_project_id=user_api_key_dict.project_id,
|
||||
user_api_key_project_alias=user_api_key_dict.project_alias,
|
||||
user_api_key_user_id=user_api_key_dict.user_id,
|
||||
user_api_key_team_alias=user_api_key_dict.team_alias,
|
||||
user_api_key_end_user_id=user_api_key_dict.end_user_id,
|
||||
user_api_key_user_email=user_api_key_dict.user_email,
|
||||
user_api_key_request_route=user_api_key_dict.request_route,
|
||||
user_api_key_auth_metadata=user_api_key_dict.metadata,
|
||||
)
|
||||
)
|
||||
|
||||
# Prune + Possibly alert
|
||||
window_seconds = self.pagerduty_alerting_args.get(
|
||||
"hanging_threshold_window_seconds",
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
|
||||
)
|
||||
threshold: int = self.pagerduty_alerting_args.get(
|
||||
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
|
||||
)
|
||||
|
||||
# If threshold is crossed, send PD alert for hangs
|
||||
await self._send_alert_if_thresholds_crossed(
|
||||
events=self._hanging_events,
|
||||
window_seconds=window_seconds,
|
||||
threshold=threshold,
|
||||
alert_prefix="High Number of Hanging LLM Requests",
|
||||
)
|
||||
|
||||
# ------------------ HELPERS ------------------ #
|
||||
|
||||
async def _send_alert_if_thresholds_crossed(
|
||||
self,
|
||||
events: List[PagerDutyInternalEvent],
|
||||
window_seconds: int,
|
||||
threshold: int,
|
||||
alert_prefix: str,
|
||||
):
|
||||
"""
|
||||
1. Prune old events
|
||||
2. If threshold is reached, build alert, send to PagerDuty
|
||||
3. Clear those events
|
||||
"""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
|
||||
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
|
||||
|
||||
# Update the reference list
|
||||
events.clear()
|
||||
events.extend(pruned)
|
||||
|
||||
# Check threshold
|
||||
verbose_logger.debug(
|
||||
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
|
||||
)
|
||||
if len(events) >= threshold:
|
||||
# Build short summary of last N events
|
||||
error_summaries = self._build_error_summaries(events, max_errors=5)
|
||||
alert_message = (
|
||||
f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
|
||||
)
|
||||
custom_details = {"recent_errors": error_summaries}
|
||||
|
||||
await self.send_alert_to_pagerduty(
|
||||
alert_message=alert_message,
|
||||
custom_details=custom_details,
|
||||
)
|
||||
|
||||
# Clear them after sending an alert, so we don't spam
|
||||
events.clear()
|
||||
|
||||
def _build_error_summaries(
|
||||
self, events: List[PagerDutyInternalEvent], max_errors: int = 5
|
||||
) -> List[PagerDutyInternalEvent]:
|
||||
"""
|
||||
Build short text summaries for the last `max_errors`.
|
||||
Example: "ValueError (code: 500, provider: openai)"
|
||||
"""
|
||||
recent = events[-max_errors:]
|
||||
summaries = []
|
||||
for fe in recent:
|
||||
# If any of these is None, show "N/A" to avoid messing up the summary string
|
||||
fe.pop("timestamp")
|
||||
summaries.append(fe)
|
||||
return summaries
|
||||
|
||||
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
|
||||
"""
|
||||
Send [critical] Alert to PagerDuty
|
||||
|
||||
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
|
||||
"""
|
||||
try:
|
||||
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
|
||||
async_client: AsyncHTTPHandler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
payload: PagerDutyRequestBody = PagerDutyRequestBody(
|
||||
payload=PagerDutyPayload(
|
||||
summary=alert_message,
|
||||
severity="critical",
|
||||
source="LiteLLM Alert",
|
||||
component="LiteLLM",
|
||||
custom_details=custom_details,
|
||||
),
|
||||
routing_key=self.api_key,
|
||||
event_action="trigger",
|
||||
)
|
||||
|
||||
return await async_client.post(
|
||||
url="https://events.pagerduty.com/v2/enqueue",
|
||||
json=dict(payload),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")
|
||||
"""
|
||||
PagerDuty Alerting Integration
|
||||
|
||||
Handles two types of alerts:
|
||||
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
|
||||
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
|
||||
|
||||
Note: This is a Free feature on the regular litellm docker image.
|
||||
|
||||
However, this is under the enterprise license
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final, Literal
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching import DualCache
|
||||
from litellm.exceptions import MissingMessageRoleError
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.integrations.pagerduty import (
|
||||
AlertingConfig,
|
||||
PagerDutyInternalEvent,
|
||||
PagerDutyPayload,
|
||||
PagerDutyRequestBody,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
StandardLoggingMetadata,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
|
||||
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
|
||||
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
|
||||
|
||||
|
||||
def _metadata_of(payload: StandardLoggingPayload) -> StandardLoggingMetadata:
|
||||
return payload["metadata"]
|
||||
|
||||
|
||||
_CustomerIdKey = Literal[
|
||||
"user_api_key_customer_member_id", "user_api_key_customer_team_id", "user_api_key_customer_org_id"
|
||||
]
|
||||
|
||||
|
||||
def _customer_id(metadata: StandardLoggingMetadata, key: _CustomerIdKey) -> str | None:
|
||||
return metadata[key] if key in metadata else None
|
||||
|
||||
|
||||
class PagerDutyAlerting(SlackAlerting):
|
||||
"""
|
||||
Tracks failed requests and hanging requests separately.
|
||||
If threshold is crossed for either type, triggers a PagerDuty alert.
|
||||
"""
|
||||
|
||||
def __init__(self, alerting_args: AlertingConfig | dict | None = None, **kwargs):
|
||||
super().__init__()
|
||||
_api_key = os.getenv("PAGERDUTY_API_KEY")
|
||||
if not _api_key:
|
||||
raise ValueError("PAGERDUTY_API_KEY is not set")
|
||||
|
||||
self.api_key: str = _api_key
|
||||
alerting_args = alerting_args or {}
|
||||
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
|
||||
failure_threshold=alerting_args.get("failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD),
|
||||
failure_threshold_window_seconds=alerting_args.get(
|
||||
"failure_threshold_window_seconds",
|
||||
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
|
||||
),
|
||||
hanging_threshold_seconds=alerting_args.get(
|
||||
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
|
||||
),
|
||||
hanging_threshold_window_seconds=alerting_args.get(
|
||||
"hanging_threshold_window_seconds",
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
|
||||
),
|
||||
)
|
||||
|
||||
# Separate storage for failures vs. hangs
|
||||
self._failure_events: list[PagerDutyInternalEvent] = []
|
||||
self._hanging_events: list[PagerDutyInternalEvent] = []
|
||||
|
||||
# ------------------ MAIN LOGIC ------------------ #
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Record a failure event. Only send an alert to PagerDuty if the
|
||||
configured *failure* threshold is exceeded in the specified window.
|
||||
"""
|
||||
if isinstance(kwargs.get("exception"), MissingMessageRoleError):
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
standard_logging_payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object")
|
||||
if not standard_logging_payload:
|
||||
raise ValueError("standard_logging_object is required for PagerDutyAlerting")
|
||||
|
||||
# Extract error details
|
||||
error_info: StandardLoggingPayloadErrorInformation | None = (
|
||||
standard_logging_payload.get("error_information") or {}
|
||||
)
|
||||
_meta: Final = _metadata_of(standard_logging_payload)
|
||||
|
||||
self._failure_events.append(
|
||||
PagerDutyInternalEvent(
|
||||
failure_event_type="failed_response",
|
||||
timestamp=now,
|
||||
error_class=error_info.get("error_class"),
|
||||
error_code=error_info.get("error_code"),
|
||||
error_llm_provider=error_info.get("llm_provider"),
|
||||
user_api_key_hash=_meta.get("user_api_key_hash"),
|
||||
user_api_key_alias=_meta.get("user_api_key_alias"),
|
||||
user_api_key_spend=_meta.get("user_api_key_spend"),
|
||||
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
|
||||
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
|
||||
user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
|
||||
user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
|
||||
user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
|
||||
user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
|
||||
user_api_key_org_id=_meta.get("user_api_key_org_id"),
|
||||
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
|
||||
user_api_key_team_id=_meta.get("user_api_key_team_id"),
|
||||
user_api_key_project_id=_meta.get("user_api_key_project_id"),
|
||||
user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
|
||||
user_api_key_user_id=_meta.get("user_api_key_user_id"),
|
||||
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
|
||||
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
|
||||
user_api_key_customer_member_id=_customer_id(_meta, "user_api_key_customer_member_id"),
|
||||
user_api_key_customer_team_id=_customer_id(_meta, "user_api_key_customer_team_id"),
|
||||
user_api_key_customer_org_id=_customer_id(_meta, "user_api_key_customer_org_id"),
|
||||
user_api_key_user_email=_meta.get("user_api_key_user_email"),
|
||||
user_api_key_request_route=_meta.get("user_api_key_request_route"),
|
||||
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
|
||||
)
|
||||
)
|
||||
|
||||
# Prune + Possibly alert
|
||||
window_seconds = self.pagerduty_alerting_args.get("failure_threshold_window_seconds", 60)
|
||||
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
|
||||
|
||||
# If threshold is crossed, send PD alert for failures
|
||||
await self._send_alert_if_thresholds_crossed(
|
||||
events=self._failure_events,
|
||||
window_seconds=window_seconds,
|
||||
threshold=threshold,
|
||||
alert_prefix="High LLM API Failure Rate",
|
||||
)
|
||||
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
cache: DualCache,
|
||||
data: dict,
|
||||
call_type: CallTypesLiteral,
|
||||
) -> Exception | str | dict | None:
|
||||
"""
|
||||
Example of detecting hanging requests by waiting a given threshold.
|
||||
If the request didn't finish by then, we treat it as 'hanging'.
|
||||
"""
|
||||
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
|
||||
asyncio.create_task(self.hanging_response_handler(request_data=data, user_api_key_dict=user_api_key_dict))
|
||||
return None
|
||||
|
||||
async def hanging_response_handler(self, request_data: dict | None, user_api_key_dict: UserAPIKeyAuth):
|
||||
"""
|
||||
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
|
||||
If not, we classify it as a hanging request.
|
||||
"""
|
||||
verbose_logger.debug(
|
||||
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
|
||||
)
|
||||
await asyncio.sleep(
|
||||
self.pagerduty_alerting_args.get("hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)
|
||||
)
|
||||
|
||||
if await self._request_is_completed(request_data=request_data):
|
||||
return # It's not hanging if completed
|
||||
|
||||
# Otherwise, record it as hanging
|
||||
self._hanging_events.append(
|
||||
PagerDutyInternalEvent(
|
||||
failure_event_type="hanging_response",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
error_class="HangingRequest",
|
||||
error_code="HangingRequest",
|
||||
error_llm_provider="HangingRequest",
|
||||
user_api_key_hash=user_api_key_dict.api_key,
|
||||
user_api_key_alias=user_api_key_dict.key_alias,
|
||||
user_api_key_spend=user_api_key_dict.spend,
|
||||
user_api_key_max_budget=user_api_key_dict.max_budget,
|
||||
user_api_key_budget_reset_at=(
|
||||
user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None
|
||||
),
|
||||
user_api_key_user_spend=user_api_key_dict.user_spend,
|
||||
user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
|
||||
user_api_key_team_spend=user_api_key_dict.team_spend,
|
||||
user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
|
||||
user_api_key_org_id=user_api_key_dict.org_id,
|
||||
user_api_key_org_alias=user_api_key_dict.organization_alias,
|
||||
user_api_key_team_id=user_api_key_dict.team_id,
|
||||
user_api_key_project_id=user_api_key_dict.project_id,
|
||||
user_api_key_project_alias=user_api_key_dict.project_alias,
|
||||
user_api_key_user_id=user_api_key_dict.user_id,
|
||||
user_api_key_team_alias=user_api_key_dict.team_alias,
|
||||
user_api_key_end_user_id=user_api_key_dict.end_user_id,
|
||||
user_api_key_customer_member_id=(
|
||||
user_api_key_dict.customer_hierarchy.customer_member_id
|
||||
if user_api_key_dict.customer_hierarchy
|
||||
else None
|
||||
),
|
||||
user_api_key_customer_team_id=(
|
||||
user_api_key_dict.customer_hierarchy.customer_team_id
|
||||
if user_api_key_dict.customer_hierarchy
|
||||
else None
|
||||
),
|
||||
user_api_key_customer_org_id=(
|
||||
user_api_key_dict.customer_hierarchy.customer_org_id
|
||||
if user_api_key_dict.customer_hierarchy
|
||||
else None
|
||||
),
|
||||
user_api_key_user_email=user_api_key_dict.user_email,
|
||||
user_api_key_request_route=user_api_key_dict.request_route,
|
||||
user_api_key_auth_metadata=user_api_key_dict.metadata,
|
||||
)
|
||||
)
|
||||
|
||||
# Prune + Possibly alert
|
||||
window_seconds = self.pagerduty_alerting_args.get(
|
||||
"hanging_threshold_window_seconds",
|
||||
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
|
||||
)
|
||||
threshold: int = self.pagerduty_alerting_args.get(
|
||||
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
|
||||
)
|
||||
|
||||
# If threshold is crossed, send PD alert for hangs
|
||||
await self._send_alert_if_thresholds_crossed(
|
||||
events=self._hanging_events,
|
||||
window_seconds=window_seconds,
|
||||
threshold=threshold,
|
||||
alert_prefix="High Number of Hanging LLM Requests",
|
||||
)
|
||||
|
||||
# ------------------ HELPERS ------------------ #
|
||||
|
||||
async def _send_alert_if_thresholds_crossed(
|
||||
self,
|
||||
events: list[PagerDutyInternalEvent],
|
||||
window_seconds: int,
|
||||
threshold: int,
|
||||
alert_prefix: str,
|
||||
):
|
||||
"""
|
||||
1. Prune old events
|
||||
2. If threshold is reached, build alert, send to PagerDuty
|
||||
3. Clear those events
|
||||
"""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
|
||||
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
|
||||
|
||||
# Update the reference list
|
||||
events.clear()
|
||||
events.extend(pruned)
|
||||
|
||||
# Check threshold
|
||||
verbose_logger.debug(
|
||||
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
|
||||
)
|
||||
if len(events) >= threshold:
|
||||
# Build short summary of last N events
|
||||
error_summaries = self._build_error_summaries(events, max_errors=5)
|
||||
alert_message = f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
|
||||
custom_details = {"recent_errors": error_summaries}
|
||||
|
||||
await self.send_alert_to_pagerduty(
|
||||
alert_message=alert_message,
|
||||
custom_details=custom_details,
|
||||
)
|
||||
|
||||
# Clear them after sending an alert, so we don't spam
|
||||
events.clear()
|
||||
|
||||
def _build_error_summaries(
|
||||
self, events: list[PagerDutyInternalEvent], max_errors: int = 5
|
||||
) -> list[PagerDutyInternalEvent]:
|
||||
"""
|
||||
Build short text summaries for the last `max_errors`.
|
||||
Example: "ValueError (code: 500, provider: openai)"
|
||||
"""
|
||||
recent = events[-max_errors:]
|
||||
summaries = []
|
||||
for fe in recent:
|
||||
# If any of these is None, show "N/A" to avoid messing up the summary string
|
||||
fe.pop("timestamp")
|
||||
summaries.append(fe)
|
||||
return summaries
|
||||
|
||||
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
|
||||
"""
|
||||
Send [critical] Alert to PagerDuty
|
||||
|
||||
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
|
||||
"""
|
||||
try:
|
||||
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
|
||||
async_client: AsyncHTTPHandler = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
|
||||
payload: PagerDutyRequestBody = PagerDutyRequestBody(
|
||||
payload=PagerDutyPayload(
|
||||
summary=alert_message,
|
||||
severity="critical",
|
||||
source="LiteLLM Alert",
|
||||
component="LiteLLM",
|
||||
custom_details=custom_details,
|
||||
),
|
||||
routing_key=self.api_key,
|
||||
event_action="trigger",
|
||||
)
|
||||
|
||||
return await async_client.post(
|
||||
url="https://events.pagerduty.com/v2/enqueue",
|
||||
json=dict(payload),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
TeamProviderDeployment,
|
||||
_is_base64_encoded_unified_file_id,
|
||||
apply_unified_file_ids,
|
||||
canonical_provider_file_id,
|
||||
decode_model_from_file_id,
|
||||
ensure_batch_response_managed_file_ids,
|
||||
get_batch_id_from_unified_batch_id,
|
||||
|
|
@ -91,6 +92,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
is_litellm_executed_batch,
|
||||
map_raw_file_ids_to_unified,
|
||||
normalize_mime_type_for_provider,
|
||||
provider_file_id_spellings,
|
||||
resolve_managed_output_file_model_name,
|
||||
validate_file_list_limit,
|
||||
validate_file_list_purpose,
|
||||
|
|
@ -597,14 +599,27 @@ async def _scoped_file_rows(
|
|||
return _SCOPED_FILE_ROWS.validate_python(rows)
|
||||
|
||||
|
||||
def _iter_provider_file_id_pairs(
|
||||
rows: Sequence[_ManagedFileRow],
|
||||
requested_provider_file_ids: frozenset[str],
|
||||
) -> Iterator[tuple[str, str]]:
|
||||
def _iter_canonical_provider_file_ids(rows: Sequence[_ManagedFileRow]) -> Iterator[tuple[str, str]]:
|
||||
for row in rows:
|
||||
for provider_file_id in row.flat_model_file_ids:
|
||||
if provider_file_id in requested_provider_file_ids:
|
||||
yield provider_file_id, row.unified_file_id
|
||||
yield canonical_provider_file_id(provider_file_id), row.unified_file_id
|
||||
|
||||
|
||||
def _iter_requested_provider_file_id_spellings(provider_file_ids: Sequence[str]) -> Iterator[str]:
|
||||
for provider_file_id in provider_file_ids:
|
||||
yield from provider_file_id_spellings(provider_file_id)
|
||||
|
||||
|
||||
def _iter_provider_file_id_pairs(
|
||||
rows: Sequence[_ManagedFileRow],
|
||||
requested_provider_file_ids: Sequence[str],
|
||||
) -> Iterator[tuple[str, str]]:
|
||||
unified_by_canonical_id: Final = dict(_iter_canonical_provider_file_ids(rows))
|
||||
for requested_provider_file_id in requested_provider_file_ids:
|
||||
if (
|
||||
unified_file_id := unified_by_canonical_id.get(canonical_provider_file_id(requested_provider_file_id))
|
||||
) is not None:
|
||||
yield requested_provider_file_id, unified_file_id
|
||||
|
||||
|
||||
def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedObjectTableActions:
|
||||
|
|
@ -767,12 +782,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
) -> None:
|
||||
verbose_logger.info(f"Storing LiteLLM Managed File object with id={file_id} in cache")
|
||||
storage_metadata: Final = _storage_metadata_of(file_object)
|
||||
flat_model_file_ids: Final = [
|
||||
canonical_provider_file_id(provider_file_id) for provider_file_id in model_mappings.values()
|
||||
]
|
||||
if file_object is not None:
|
||||
litellm_managed_file_object = LiteLLM_ManagedFileTable(
|
||||
unified_file_id=file_id,
|
||||
file_object=file_object,
|
||||
model_mappings=model_mappings,
|
||||
flat_model_file_ids=list(model_mappings.values()),
|
||||
flat_model_file_ids=flat_model_file_ids,
|
||||
created_by=resolve_resource_owner_id(user_api_key_dict),
|
||||
team_id=user_api_key_dict.team_id,
|
||||
updated_by=user_api_key_dict.user_id,
|
||||
|
|
@ -790,14 +808,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
db_data = {
|
||||
"unified_file_id": file_id,
|
||||
"model_mappings": json.dumps(model_mappings),
|
||||
"flat_model_file_ids": list(model_mappings.values()),
|
||||
"flat_model_file_ids": flat_model_file_ids,
|
||||
"created_by": resolve_resource_owner_id(user_api_key_dict),
|
||||
"team_id": user_api_key_dict.team_id,
|
||||
"updated_by": user_api_key_dict.user_id,
|
||||
}
|
||||
update_data = {
|
||||
"model_mappings": json.dumps(model_mappings),
|
||||
"flat_model_file_ids": list(model_mappings.values()),
|
||||
"flat_model_file_ids": flat_model_file_ids,
|
||||
"updated_by": user_api_key_dict.user_id,
|
||||
}
|
||||
|
||||
|
|
@ -1094,8 +1112,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
"""
|
||||
if self.prisma_client is None:
|
||||
return
|
||||
spellings: Final = list(provider_file_id_spellings(file_id))
|
||||
managed_file = await _managed_file_table(self.prisma_client).find_first(
|
||||
where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
|
||||
where={
|
||||
"OR": [
|
||||
{"unified_file_id": {"in": spellings}}, # bounded-ok: at most four spellings of one file id
|
||||
{"flat_model_file_ids": {"hasSome": spellings}},
|
||||
]
|
||||
}
|
||||
)
|
||||
if managed_file is None:
|
||||
return
|
||||
|
|
@ -1139,7 +1163,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
)
|
||||
for file_id in provider_file_ids:
|
||||
model_name = decode_model_from_file_id(file_id)
|
||||
raw_file_id = get_original_file_id(file_id)
|
||||
raw_file_id = canonical_provider_file_id(get_original_file_id(file_id))
|
||||
await self.store_batch_output_file(
|
||||
unified_file_id=file_id,
|
||||
provider_file_id=raw_file_id,
|
||||
|
|
@ -1629,7 +1653,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
return MappingProxyType({})
|
||||
|
||||
provider_file_ids_list: Final = [ # mutable-ok: Prisma hasSome requires a list
|
||||
provider_file_id for provider_file_id in unique_provider_file_ids
|
||||
*dict.fromkeys(_iter_requested_provider_file_id_spellings(unique_provider_file_ids))
|
||||
]
|
||||
rows: Final = await _managed_file_table(self.prisma_client).find_many(
|
||||
where={ # mutable-ok: Prisma requires a plain dictionary for where
|
||||
|
|
@ -1641,10 +1665,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
)
|
||||
return MappingProxyType(
|
||||
dict(
|
||||
_iter_provider_file_id_pairs(
|
||||
rows,
|
||||
frozenset(unique_provider_file_ids),
|
||||
)
|
||||
_iter_provider_file_id_pairs(rows, unique_provider_file_ids)
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from litellm.proxy.management_helpers.model_quota_policy import (
|
|||
)
|
||||
from litellm.proxy.management_helpers.utils import (
|
||||
management_endpoint_wrapper,
|
||||
validate_scalar_budget_attachment,
|
||||
)
|
||||
from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
|
||||
from litellm.repositories.base_repository import record_to_dict
|
||||
|
|
@ -637,6 +638,7 @@ async def new_project(
|
|||
)
|
||||
|
||||
# Create budget if not provided
|
||||
await validate_scalar_budget_attachment(prisma_client, data.budget_id)
|
||||
if data.budget_id is None:
|
||||
data.budget_id = await _create_budget_for_project(
|
||||
data=data,
|
||||
|
|
@ -869,6 +871,7 @@ async def update_project(
|
|||
)
|
||||
|
||||
# Prepare update data
|
||||
await validate_scalar_budget_attachment(prisma_client, data.budget_id)
|
||||
update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
|
||||
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ class SendKeyRotatedEmailEvent(WebhookEvent):
|
|||
|
||||
class EmailEvent(str, enum.Enum):
|
||||
model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT
|
||||
performance_alerts = "Performance Alerts"
|
||||
virtual_key_created = "Virtual Key Created"
|
||||
new_user_invitation = "New User Invitation"
|
||||
virtual_key_rotated = "Virtual Key Rotated"
|
||||
|
|
@ -60,7 +61,8 @@ class DefaultEmailSettings(LiteLLMBaseModel):
|
|||
|
||||
settings: dict[EmailEvent, bool] = Field(
|
||||
default_factory=lambda: {
|
||||
EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders.
|
||||
EmailEvent.model_deprecations: False,
|
||||
EmailEvent.performance_alerts: False,
|
||||
EmailEvent.virtual_key_created: True, # On by default
|
||||
EmailEvent.new_user_invitation: True, # On by default
|
||||
EmailEvent.virtual_key_rotated: True, # On by default
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.83"
|
||||
version = "0.1.85"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.83"
|
||||
version = "0.1.85"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -465,6 +465,7 @@ def image_generation(
|
|||
litellm.LlmProviders.AIML,
|
||||
litellm.LlmProviders.GEMINI,
|
||||
litellm.LlmProviders.FAL_AI,
|
||||
litellm.LlmProviders.REPLICATE,
|
||||
litellm.LlmProviders.STABILITY,
|
||||
litellm.LlmProviders.RUNWAYML,
|
||||
litellm.LlmProviders.VERTEX_AI,
|
||||
|
|
@ -496,6 +497,7 @@ def image_generation(
|
|||
logging_obj=litellm_logging_obj,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
_is_async=aimg_generation is True and custom_llm_provider == litellm.LlmProviders.REPLICATE,
|
||||
)
|
||||
elif custom_llm_provider == "black_forest_labs":
|
||||
# Route to BFL-specific handler (polling required)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
# Class for sending Slack Alerts #
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
|
||||
|
|
@ -38,6 +39,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.repositories.table_repositories import InvitationLinkRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
|
|
@ -87,6 +89,12 @@ def _proxy_llm_router() -> Router | None:
|
|||
_DAILY_REPORT_TARGET: Final = "daily_report_schedule"
|
||||
|
||||
|
||||
async def _model_deprecations_enabled() -> bool:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
return prisma_client is None or await ConfigRepository(prisma_client, use_writer=True).model_deprecations_enabled()
|
||||
|
||||
|
||||
class SlackAlerting(CustomBatchLogger):
|
||||
"""
|
||||
Class for sending Slack Alerts
|
||||
|
|
@ -97,12 +105,13 @@ class SlackAlerting(CustomBatchLogger):
|
|||
self,
|
||||
internal_usage_cache: DualCache | None = None,
|
||||
alerting_threshold: float | None = None, # threshold for slow / hanging llm responses (in seconds)
|
||||
alerting: list | None = [],
|
||||
alerting: list[str] | None = [],
|
||||
alert_types: list[AlertType] = DEFAULT_ALERT_TYPES,
|
||||
alert_to_webhook_url: dict[AlertType, list[str] | str]
|
||||
| None = None, # if user wants to separate alerts to diff channels
|
||||
alerting_args={},
|
||||
default_webhook_url: str | None = None,
|
||||
model_deprecations_enabled: Callable[[], Awaitable[bool]] = _model_deprecations_enabled,
|
||||
alert_type_config: dict[str, dict] | None = None,
|
||||
async_http_handler: AsyncHTTPHandler | None = None,
|
||||
**kwargs,
|
||||
|
|
@ -120,6 +129,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
self.is_running = False
|
||||
self.alerting_args = SlackAlertingArgs(**alerting_args)
|
||||
self.default_webhook_url = default_webhook_url
|
||||
self.model_deprecations_enabled: Final = model_deprecations_enabled
|
||||
self.flush_lock = asyncio.Lock()
|
||||
self.periodic_started = False
|
||||
self._periodic_flush_task: asyncio.Task[None] | None = None
|
||||
|
|
@ -142,7 +152,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
|
||||
def update_values(
|
||||
self,
|
||||
alerting: list | None = None,
|
||||
alerting: list[str] | None = None,
|
||||
alerting_threshold: float | None = None,
|
||||
alert_types: list[AlertType] | None = None,
|
||||
alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None,
|
||||
|
|
@ -1114,7 +1124,7 @@ Model Info:
|
|||
The daily lock is claimed only once there is something to say, so an empty pass never blocks a
|
||||
later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking
|
||||
"""
|
||||
if not self._deprecation_alerts_enabled():
|
||||
if not self._deprecation_alerts_enabled() or not await self.model_deprecations_enabled():
|
||||
return False
|
||||
|
||||
from litellm.proxy.common_utils.model_deprecation import (
|
||||
|
|
@ -1194,6 +1204,26 @@ Model Info:
|
|||
continue
|
||||
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
|
||||
|
||||
def performance_alert_webhooks(self) -> tuple[str, ...]:
|
||||
if not self.alerting or "slack" not in self.alerting or AlertType.performance_alerts not in self.alert_types:
|
||||
return ()
|
||||
configured: Final = (self.alert_to_webhook_url or {}).get(AlertType.performance_alerts)
|
||||
destination: Final = (
|
||||
configured
|
||||
or self.default_webhook_url
|
||||
or os.getenv("SLACK_WEBHOOK_URL")
|
||||
or os.getenv("ALERTING_WEBHOOK_URL")
|
||||
)
|
||||
if isinstance(destination, list):
|
||||
return tuple(dict.fromkeys(destination))
|
||||
return (destination,) if destination else ()
|
||||
|
||||
async def send_performance_alert_summary(self, webhook: str, message: str) -> bool:
|
||||
response: Final = await self.async_http_handler.post(
|
||||
url=webhook, headers={"Content-Type": "application/json"}, data=json.dumps({"text": message})
|
||||
)
|
||||
return response.status_code == 200
|
||||
|
||||
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
|
||||
"""
|
||||
Sends structured alert to webhook, if set.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
import re
|
||||
import traceback
|
||||
from collections.abc import AsyncGenerator, Mapping, Sequence
|
||||
from functools import cache
|
||||
from itertools import takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
|
||||
|
||||
|
|
@ -36,6 +38,7 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.types.integrations.customer_budget import CustomerBudgetEvent
|
||||
from litellm.types.mcp import (
|
||||
MCPPostCallResponseObject,
|
||||
MCPPreCallRequestObject,
|
||||
|
|
@ -66,6 +69,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
|
|||
class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
|
||||
# Class variables or attributes
|
||||
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
|
||||
wants_failed_prompt_token_estimate: ClassVar[bool] = False
|
||||
|
||||
enforces_request_content: bool = False
|
||||
"""
|
||||
|
|
@ -196,6 +200,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
async def async_log_audit_log_event(self, audit_log: "StandardAuditLogPayload"):
|
||||
"""Called when an audit log is created. Override in subclasses to handle."""
|
||||
|
||||
async def async_on_customer_budget_event(self, event: "CustomerBudgetEvent") -> None:
|
||||
pass
|
||||
|
||||
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
|
||||
"""Called once per part of a finished batch's lines, when this logger is listed in batch_line_item_callbacks."""
|
||||
|
||||
|
|
@ -1115,3 +1122,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
msg[key] = self._redact_base64(value=val, max_depth=max_depth)
|
||||
filtered_messages.append(msg)
|
||||
return filtered_messages
|
||||
|
||||
|
||||
@cache
|
||||
def subclass_overrides_hook(cls: type, hook_name: str, base: type = CustomLogger) -> bool:
|
||||
below_base: Final = takewhile(lambda klass: klass is not base, cls.__mro__)
|
||||
return any(hook_name in vars(klass) for klass in below_base)
|
||||
|
|
|
|||
74
litellm/integrations/custom_logger_v2/__init__.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
from litellm.integrations.custom_logger_v2.call import CallContext, CallFacts, CallIds, CallKind, CallTarget, CallTiming
|
||||
from litellm.integrations.custom_logger_v2.dispatcher import Dispatcher, HookResult, RequestScope
|
||||
from litellm.integrations.custom_logger_v2.enforcement import ContractViolation, UndeclaredReadError
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, Field, FieldTier, Hook, OptOutBy
|
||||
from litellm.integrations.custom_logger_v2.hooks import (
|
||||
HOOK_SPECS,
|
||||
HookSpec,
|
||||
InvalidDeclarationError,
|
||||
Subscription,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
|
||||
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest, build_manifest
|
||||
from litellm.integrations.custom_logger_v2.offpath import InProcessSink, OffPathSink
|
||||
from litellm.integrations.custom_logger_v2.outcomes import (
|
||||
Abort,
|
||||
InvalidPatchError,
|
||||
JsonValue,
|
||||
MergeField,
|
||||
Outcome,
|
||||
Patch,
|
||||
Reroute,
|
||||
Respond,
|
||||
SetField,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.patches import apply_patch, apply_patches
|
||||
from litellm.integrations.custom_logger_v2.planner import HookPlan, plan_hook
|
||||
from litellm.integrations.custom_logger_v2.registry import CallbackRegistry
|
||||
from litellm.integrations.custom_logger_v2.selection import RequestSelection, Selection, is_selected
|
||||
|
||||
__all__ = [
|
||||
"HOOK_SPECS",
|
||||
"Abort",
|
||||
"Activation",
|
||||
"CallContext",
|
||||
"CallFacts",
|
||||
"CallIds",
|
||||
"CallKind",
|
||||
"CallTarget",
|
||||
"CallTiming",
|
||||
"CallbackManifest",
|
||||
"CallbackRegistry",
|
||||
"ContractViolation",
|
||||
"CustomLoggerV2",
|
||||
"Dispatcher",
|
||||
"Field",
|
||||
"FieldTier",
|
||||
"Hook",
|
||||
"HookPlan",
|
||||
"HookResult",
|
||||
"HookSpec",
|
||||
"InProcessSink",
|
||||
"InvalidDeclarationError",
|
||||
"InvalidPatchError",
|
||||
"JsonValue",
|
||||
"MergeField",
|
||||
"OffPathSink",
|
||||
"OptOutBy",
|
||||
"Outcome",
|
||||
"Patch",
|
||||
"RequestScope",
|
||||
"RequestSelection",
|
||||
"Reroute",
|
||||
"Respond",
|
||||
"Selection",
|
||||
"SetField",
|
||||
"Subscription",
|
||||
"UndeclaredReadError",
|
||||
"apply_patch",
|
||||
"apply_patches",
|
||||
"build_manifest",
|
||||
"is_selected",
|
||||
"on",
|
||||
"plan_hook",
|
||||
]
|
||||
55
litellm/integrations/custom_logger_v2/call.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Final, Generic, TypeVar
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Hook
|
||||
|
||||
StateT: Final = TypeVar("StateT")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallIds:
|
||||
call_id: str
|
||||
trace_id: str | None = None
|
||||
session_id: str | None = None
|
||||
parent_call_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallKind:
|
||||
call_type: str
|
||||
route: str | None = None
|
||||
stream: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallTarget:
|
||||
model: str
|
||||
model_group: str | None = None
|
||||
provider: str | None = None
|
||||
api_base: str | None = None
|
||||
deployment_id: str | None = None
|
||||
attempt: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallTiming:
|
||||
start: float
|
||||
provider_start: float | None = None
|
||||
first_token: float | None = None
|
||||
end: float | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallFacts:
|
||||
ids: CallIds
|
||||
kind: CallKind
|
||||
target: CallTarget
|
||||
timing: CallTiming
|
||||
cache_hit: bool | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallContext(Generic[StateT]):
|
||||
hook: Hook
|
||||
call: CallFacts | None
|
||||
state: StateT
|
||||
400
litellm/integrations/custom_logger_v2/dispatcher.py
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from operator import itemgetter
|
||||
from types import TracebackType
|
||||
from typing import Final, Protocol, TypeAlias, TypeVar
|
||||
|
||||
from typing_extensions import Self
|
||||
|
||||
from litellm.integrations.custom_logger_v2.call import CallContext, CallFacts
|
||||
from litellm.integrations.custom_logger_v2.enforcement import ContractViolation, Enforced, Policy, Unchecked
|
||||
from litellm.integrations.custom_logger_v2.enums import Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS
|
||||
from litellm.integrations.custom_logger_v2.manifest import LegacyBinding
|
||||
from litellm.integrations.custom_logger_v2.offpath import InProcessSink, OffPathSink
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, Outcome, Patch, Reroute, Respond
|
||||
from litellm.integrations.custom_logger_v2.patches import apply_patch, apply_patches
|
||||
from litellm.integrations.custom_logger_v2.planner import EMPTY_PLAN, Entry, HookPlan
|
||||
from litellm.integrations.custom_logger_v2.registry import CallbackRegistry, RegistrySnapshot
|
||||
from litellm.integrations.custom_logger_v2.selection import RequestSelection, is_selected
|
||||
|
||||
LegacyT: Final = TypeVar("LegacyT")
|
||||
|
||||
Between: TypeAlias = Callable[[Mapping[Field, object]], Awaitable[Mapping[Field, object]]]
|
||||
SyncBinding: TypeAlias = Callable[[object, object], object]
|
||||
|
||||
SWALLOWING_HOOKS: Final = frozenset({Hook.response_headers, Hook.post_call_failure})
|
||||
|
||||
_LOGGER: Final = logging.getLogger("LiteLLM")
|
||||
|
||||
|
||||
class _Handler(Protocol):
|
||||
def __call__(self, view: object, ctx: CallContext[object], /) -> object: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HookResult:
|
||||
fields: Mapping[Field, object]
|
||||
patches: tuple[Patch, ...] = ()
|
||||
outcome: Outcome | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Chain:
|
||||
fields: Mapping[Field, object]
|
||||
patches: tuple[tuple[int, Patch], ...] = ()
|
||||
reroute: Reroute | None = None
|
||||
terminal: Abort | Respond | None = None
|
||||
|
||||
@property
|
||||
def ordered_patches(self) -> tuple[Patch, ...]:
|
||||
return tuple(patch for _, patch in sorted(self.patches, key=itemgetter(0)))
|
||||
|
||||
|
||||
def _handler(logger: object, name: str) -> _Handler:
|
||||
found: Final[object] = getattr(logger, name, None)
|
||||
if not callable(found):
|
||||
raise ContractViolation(f"{name} is not callable")
|
||||
return found
|
||||
|
||||
|
||||
async def _awaited(returned: object) -> object:
|
||||
return await returned if inspect.isawaitable(returned) else returned
|
||||
|
||||
|
||||
def _decides(hook: Hook, result: object) -> bool:
|
||||
match result:
|
||||
case Abort() | Respond():
|
||||
return True
|
||||
case ContractViolation():
|
||||
return True
|
||||
case Exception():
|
||||
return hook not in SWALLOWING_HOOKS
|
||||
case BaseException():
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _fatal(error: BaseException) -> bool:
|
||||
return isinstance(error, ContractViolation) or not isinstance(error, Exception)
|
||||
|
||||
|
||||
def _split(
|
||||
stages: tuple[tuple[Entry, ...], ...],
|
||||
) -> tuple[tuple[tuple[Entry, ...], ...], tuple[tuple[Entry, ...], ...]]:
|
||||
if stages and stages[-1][0].subscription.parallel:
|
||||
return stages[:-1], stages[-1:]
|
||||
return stages, ()
|
||||
|
||||
|
||||
def _legacy_start(
|
||||
entry: Entry, binding: Callable[[object, object], LegacyT] | None, fields: Mapping[Field, object]
|
||||
) -> LegacyT | None:
|
||||
call: Final = entry.manifest.legacy_call
|
||||
if call is None or binding is None:
|
||||
raise ContractViolation(
|
||||
f"{entry.manifest.selection.name} is a legacy entry, so the hook point must pass a binding"
|
||||
)
|
||||
return call(binding, fields)
|
||||
|
||||
|
||||
async def _run_legacy(entry: Entry, binding: LegacyBinding | None, fields: Mapping[Field, object]) -> object:
|
||||
started: Final = _legacy_start(entry, binding, fields)
|
||||
return None if started is None else await started
|
||||
|
||||
|
||||
async def _gather_started(started: tuple[Awaitable[object] | None, ...], fail_fast: bool) -> tuple[object, ...]:
|
||||
pending: Final = tuple(run for run in started if run is not None)
|
||||
if not pending:
|
||||
return tuple(None for _ in started)
|
||||
done: Final = iter(
|
||||
await asyncio.gather(*pending) if fail_fast else await asyncio.gather(*pending, return_exceptions=True)
|
||||
)
|
||||
return tuple(None if run is None else next(done) for run in started)
|
||||
|
||||
|
||||
def _fail_fast(hook: Hook, stage: tuple[Entry, ...]) -> bool:
|
||||
return HOOK_SPECS[hook].fail_fast_parallel and all(entry.subscription.parallel for entry in stage)
|
||||
|
||||
|
||||
async def run_legacy_plan(
|
||||
hook: Hook,
|
||||
plan: HookPlan,
|
||||
fields: Mapping[Field, object],
|
||||
binding: LegacyBinding,
|
||||
between: Between | None = None,
|
||||
) -> Mapping[Field, object]:
|
||||
serial, parallel = _split(plan.stages)
|
||||
current: Mapping[Field, object] = fields # rebind-ok: each legacy entry may replace the kwargs
|
||||
for stage in serial:
|
||||
for entry in stage:
|
||||
result = await _run_legacy(entry, binding, current)
|
||||
if isinstance(result, Patch):
|
||||
current = apply_patch(current, result)
|
||||
if between is not None:
|
||||
current = await between(current)
|
||||
for stage in parallel:
|
||||
started = tuple(_legacy_start(entry, binding, current) for entry in stage)
|
||||
results = await _gather_started(started, _fail_fast(hook, stage))
|
||||
failure = next((result for result in results if isinstance(result, BaseException)), None)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
for entry in plan.inline:
|
||||
_ = await _run_legacy(entry, binding, current)
|
||||
return current
|
||||
|
||||
|
||||
def run_legacy_plan_sync(
|
||||
plan: HookPlan, fields: Mapping[Field, object], binding: SyncBinding
|
||||
) -> Mapping[Field, object]:
|
||||
staged: Final = tuple(chain.from_iterable(plan.stages))
|
||||
if any(entry.subscription.parallel for entry in staged):
|
||||
raise ContractViolation("a synchronous hook cannot run parallel legacy entries")
|
||||
current: Mapping[Field, object] = fields # rebind-ok: each legacy entry may replace the kwargs
|
||||
for entry in (*staged, *plan.inline):
|
||||
result = _legacy_start(entry, binding, current)
|
||||
if isinstance(result, Patch):
|
||||
current = apply_patch(current, result)
|
||||
return current
|
||||
|
||||
|
||||
class Dispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
registry: CallbackRegistry,
|
||||
sink: OffPathSink | None = None,
|
||||
enforce: bool = False,
|
||||
) -> None:
|
||||
self.registry: Final = registry
|
||||
self.sink: Final[OffPathSink] = sink or InProcessSink()
|
||||
self.policy: Final[Policy] = Enforced() if enforce else Unchecked()
|
||||
self.failures: Final[Counter[tuple[str, Hook]]] = Counter() # mutable-ok: process-wide failure counts
|
||||
|
||||
def open_scope(self, selection: RequestSelection) -> "RequestScope":
|
||||
return RequestScope(self, self.registry.snapshot, selection)
|
||||
|
||||
def record_failure(self, entry: Entry, hook: Hook, error: BaseException) -> None:
|
||||
self.failures[(entry.manifest.selection.name, hook)] += 1
|
||||
_LOGGER.warning("custom logger v2 %s failed on %s: %r", entry.manifest.selection.name, hook.value, error)
|
||||
|
||||
|
||||
class RequestScope:
|
||||
def __init__(self, dispatcher: Dispatcher, snapshot: RegistrySnapshot, selection: RequestSelection) -> None:
|
||||
self._dispatcher: Final = dispatcher
|
||||
self._snapshot: Final = snapshot
|
||||
self._selection: Final = selection
|
||||
self._states: dict[int, object] = {} # mutable-ok: request_state instances are created lazily per logger
|
||||
self.reroute: Reroute | None = None
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None
|
||||
) -> None:
|
||||
self.close()
|
||||
|
||||
def close(self) -> None:
|
||||
self._states.clear()
|
||||
|
||||
def selects(self, entry: Entry) -> bool:
|
||||
return is_selected(entry.manifest.selection, entry.activation, self._selection)
|
||||
|
||||
def plan(self, hook: Hook) -> HookPlan:
|
||||
entries: Final = self._snapshot.entries[hook]
|
||||
if not entries:
|
||||
return EMPTY_PLAN
|
||||
selected: Final = tuple(index for index, entry in enumerate(entries) if self.selects(entry))
|
||||
return self._snapshot.plan(hook, selected) if selected else EMPTY_PLAN
|
||||
|
||||
def reads(self, hook: Hook) -> frozenset[Field]:
|
||||
return self.plan(hook).reads
|
||||
|
||||
async def run_hook(
|
||||
self,
|
||||
hook: Hook,
|
||||
fields: Mapping[Field, object],
|
||||
call: CallFacts | None,
|
||||
) -> HookResult:
|
||||
plan: Final = self.plan(hook)
|
||||
if plan is EMPTY_PLAN:
|
||||
return HookResult(fields=fields)
|
||||
chain: Final = await self._run_stages(hook, plan.stages, _Chain(fields=fields), call, None)
|
||||
patches: Final = chain.ordered_patches
|
||||
if chain.terminal is not None:
|
||||
return HookResult(fields=chain.fields, patches=patches, outcome=chain.terminal)
|
||||
await self._run_awaited(hook, plan.awaited, chain.fields, call)
|
||||
if plan.off_path:
|
||||
self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, chain.fields, call))
|
||||
if chain.reroute is not None:
|
||||
self.reroute = chain.reroute
|
||||
return HookResult(fields=chain.fields, patches=patches, outcome=chain.reroute)
|
||||
|
||||
async def run_plan(
|
||||
self,
|
||||
hook: Hook,
|
||||
plan: HookPlan,
|
||||
fields: Mapping[Field, object],
|
||||
call: CallFacts | None,
|
||||
binding: LegacyBinding,
|
||||
between: Between | None = None,
|
||||
) -> HookResult:
|
||||
serial, parallel = _split(plan.stages)
|
||||
chain: Final = await self._run_stages(hook, serial, _Chain(fields=fields), call, binding)
|
||||
if chain.terminal is not None:
|
||||
return HookResult(fields=chain.fields, patches=chain.ordered_patches, outcome=chain.terminal)
|
||||
joined: Final = (
|
||||
chain
|
||||
if between is None
|
||||
else _Chain(fields=await between(chain.fields), patches=chain.patches, reroute=chain.reroute)
|
||||
)
|
||||
done: Final = await self._run_stages(hook, parallel, joined, call, binding)
|
||||
if done.terminal is not None:
|
||||
return HookResult(fields=done.fields, patches=done.ordered_patches, outcome=done.terminal)
|
||||
for entry in plan.inline:
|
||||
_ = await _run_legacy(entry, binding, done.fields)
|
||||
await self._run_awaited(hook, plan.awaited, done.fields, call)
|
||||
if plan.off_path:
|
||||
self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, done.fields, call))
|
||||
if done.reroute is not None:
|
||||
self.reroute = done.reroute
|
||||
return HookResult(fields=done.fields, patches=done.ordered_patches, outcome=done.reroute)
|
||||
|
||||
def run_plan_sync(
|
||||
self,
|
||||
hook: Hook,
|
||||
plan: HookPlan,
|
||||
fields: Mapping[Field, object],
|
||||
call: CallFacts | None,
|
||||
binding: SyncBinding,
|
||||
) -> Mapping[Field, object]:
|
||||
v2_entries: Final = tuple(entry for entry in chain.from_iterable(plan.stages) if not entry.subscription.legacy)
|
||||
if v2_entries or plan.awaited:
|
||||
raise ContractViolation(
|
||||
f"{hook.value} runs synchronously here, so only off-path V2 callbacks may subscribe"
|
||||
)
|
||||
result: Final = run_legacy_plan_sync(plan, fields, binding)
|
||||
if plan.off_path:
|
||||
self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, result, call))
|
||||
return result
|
||||
|
||||
def _state(self, entry: Entry) -> object:
|
||||
factory: Final = entry.manifest.request_state
|
||||
if factory is None:
|
||||
return None
|
||||
key: Final = id(entry.manifest.logger)
|
||||
if key not in self._states:
|
||||
self._states[key] = factory()
|
||||
return self._states[key]
|
||||
|
||||
async def _invoke(
|
||||
self, hook: Hook, entry: Entry, fields: Mapping[Field, object], call: CallFacts | None, state: object
|
||||
) -> object:
|
||||
policy: Final = self._dispatcher.policy
|
||||
handler: Final = _handler(entry.manifest.logger, entry.subscription.handler)
|
||||
context: Final = CallContext[object](hook=hook, call=call, state=state)
|
||||
view: Final = policy.view(entry.subscription, fields)
|
||||
result: Final = (
|
||||
await _awaited(handler(view, context))
|
||||
if inspect.iscoroutinefunction(handler)
|
||||
else await asyncio.to_thread(handler, view, context)
|
||||
)
|
||||
policy.check(entry.subscription, result)
|
||||
return result
|
||||
|
||||
def _start(
|
||||
self,
|
||||
hook: Hook,
|
||||
entry: Entry,
|
||||
fields: Mapping[Field, object],
|
||||
call: CallFacts | None,
|
||||
live: bool,
|
||||
binding: LegacyBinding | None,
|
||||
) -> Awaitable[object] | None:
|
||||
if entry.subscription.legacy:
|
||||
return _legacy_start(entry, binding, fields)
|
||||
return self._invoke(hook, entry, fields, call, self._state(entry) if live else None)
|
||||
|
||||
async def _gather(
|
||||
self,
|
||||
hook: Hook,
|
||||
entries: Sequence[Entry],
|
||||
fields: Mapping[Field, object],
|
||||
call: CallFacts | None,
|
||||
live: bool,
|
||||
binding: LegacyBinding | None = None,
|
||||
) -> tuple[object, ...]:
|
||||
if len(entries) == 1 and entries[0].subscription.legacy:
|
||||
return (await _run_legacy(entries[0], binding, fields),)
|
||||
started: Final = tuple(self._start(hook, entry, fields, call, live, binding) for entry in entries)
|
||||
return await _gather_started(started, _fail_fast(hook, tuple(entries)))
|
||||
|
||||
async def _run_stages(
|
||||
self,
|
||||
hook: Hook,
|
||||
stages: tuple[tuple[Entry, ...], ...],
|
||||
chain: _Chain,
|
||||
call: CallFacts | None,
|
||||
binding: LegacyBinding | None,
|
||||
) -> _Chain:
|
||||
if not stages:
|
||||
return chain
|
||||
stage: Final = stages[0]
|
||||
results: Final = await self._gather(hook, stage, chain.fields, call, live=True, binding=binding)
|
||||
folded: Final = self._fold(hook, stage, results, chain)
|
||||
if folded.terminal is not None:
|
||||
return folded
|
||||
return await self._run_stages(hook, stages[1:], folded, call, binding)
|
||||
|
||||
def _fold(self, hook: Hook, stage: tuple[Entry, ...], results: tuple[object, ...], chain: _Chain) -> _Chain:
|
||||
pairs: Final = tuple(zip(stage, results, strict=True))
|
||||
decisive: Final = next((pair for pair in pairs if _decides(hook, pair[1])), None)
|
||||
self._count_failures(hook, tuple(pair for pair in pairs if pair is not decisive))
|
||||
match decisive:
|
||||
case (_, BaseException() as error):
|
||||
raise error
|
||||
case (_, Abort() | Respond() as outcome):
|
||||
return _Chain(fields=chain.fields, patches=chain.patches, reroute=chain.reroute, terminal=outcome)
|
||||
case _:
|
||||
return self._merge(pairs, chain)
|
||||
|
||||
def _merge(self, pairs: tuple[tuple[Entry, object], ...], chain: _Chain) -> _Chain:
|
||||
stage_patches: Final = tuple((entry.position, result) for entry, result in pairs if isinstance(result, Patch))
|
||||
reroutes: Final = tuple(result for _, result in pairs if isinstance(result, Reroute))
|
||||
return _Chain(
|
||||
fields=apply_patches(chain.fields, (patch for _, patch in stage_patches)),
|
||||
patches=(*chain.patches, *stage_patches),
|
||||
reroute=chain.reroute or next(iter(reroutes), None),
|
||||
)
|
||||
|
||||
def _count_failures(self, hook: Hook, pairs: Sequence[tuple[Entry, object]]) -> None:
|
||||
for entry, result in pairs:
|
||||
if isinstance(result, BaseException) and not entry.subscription.legacy:
|
||||
self._dispatcher.record_failure(entry, hook, result)
|
||||
|
||||
def _swallow(self, hook: Hook, entries: Sequence[Entry], results: tuple[object, ...]) -> None:
|
||||
pairs: Final = tuple(zip(entries, results, strict=True))
|
||||
fatal: Final = next(
|
||||
(result for _, result in pairs if isinstance(result, BaseException) and _fatal(result)),
|
||||
None,
|
||||
)
|
||||
self._count_failures(hook, tuple(pair for pair in pairs if pair[1] is not fatal))
|
||||
if fatal is not None:
|
||||
raise fatal
|
||||
|
||||
async def _run_awaited(
|
||||
self, hook: Hook, entries: tuple[Entry, ...], fields: Mapping[Field, object], call: CallFacts | None
|
||||
) -> None:
|
||||
if entries:
|
||||
self._swallow(hook, entries, await self._gather(hook, entries, fields, call, live=True))
|
||||
|
||||
async def _deliver_off_path(
|
||||
self, hook: Hook, entries: tuple[Entry, ...], fields: Mapping[Field, object], call: CallFacts | None
|
||||
) -> None:
|
||||
self._swallow(hook, entries, await self._gather(hook, entries, fields, call, live=False))
|
||||
95
litellm/integrations/custom_logger_v2/enforcement.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS, Subscription
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch, Reroute, Respond
|
||||
|
||||
|
||||
class ContractViolation(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UndeclaredReadError(ContractViolation, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
def _field_named(name: str) -> Field | None:
|
||||
member: Final = Field._value2member_map_.get(name)
|
||||
return member if isinstance(member, Field) else None
|
||||
|
||||
|
||||
class FieldView:
|
||||
__slots__ = ("_fields",)
|
||||
|
||||
def __init__(self, fields: Mapping[Field, object]) -> None:
|
||||
self._fields = fields
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
field: Final = _field_named(name)
|
||||
if field is None or field not in self._fields:
|
||||
raise AttributeError(name)
|
||||
return self._fields[field]
|
||||
|
||||
|
||||
class DeclaredView:
|
||||
__slots__ = ("_fields", "_reads")
|
||||
|
||||
def __init__(self, fields: Mapping[Field, object], reads: frozenset[Field]) -> None:
|
||||
self._fields = fields
|
||||
self._reads = reads
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
field: Final = _field_named(name)
|
||||
if field is None or field not in self._reads:
|
||||
raise UndeclaredReadError(f"{name} is not among the declared reads")
|
||||
if field not in self._fields:
|
||||
raise AttributeError(name)
|
||||
return self._fields[field]
|
||||
|
||||
|
||||
class Policy(Protocol):
|
||||
def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object: ...
|
||||
|
||||
def check(self, subscription: Subscription, result: object) -> None: ...
|
||||
|
||||
|
||||
class Unchecked:
|
||||
def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object:
|
||||
return FieldView(fields)
|
||||
|
||||
def check(self, subscription: Subscription, result: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _outcome_problem(hook: Hook, subscription: Subscription, outcome: Abort | Respond | Reroute) -> str | None:
|
||||
if not subscription.may_abort:
|
||||
return f"returned {type(outcome).__name__} without may_abort"
|
||||
if type(outcome) not in HOOK_SPECS[hook].outcomes:
|
||||
return f"returned {type(outcome).__name__}, which {hook.value} does not allow"
|
||||
return None
|
||||
|
||||
|
||||
def _result_problem(subscription: Subscription, result: object) -> str | None:
|
||||
match result:
|
||||
case None:
|
||||
return None
|
||||
case Patch():
|
||||
undeclared: Final = result.fields - subscription.writes
|
||||
if not undeclared:
|
||||
return None
|
||||
return f"patched {', '.join(sorted(field.value for field in undeclared))} outside its declared writes"
|
||||
case Abort() | Respond() | Reroute():
|
||||
return _outcome_problem(subscription.hook, subscription, result)
|
||||
case _:
|
||||
return f"returned {type(result).__name__}, which is neither None, a Patch nor an outcome"
|
||||
|
||||
|
||||
class Enforced:
|
||||
def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object:
|
||||
return DeclaredView(fields, subscription.reads)
|
||||
|
||||
def check(self, subscription: Subscription, result: object) -> None:
|
||||
problem: Final = _result_problem(subscription, result)
|
||||
if problem is not None:
|
||||
raise ContractViolation(f"{subscription.handler} on {subscription.hook.value} {problem}")
|
||||
97
litellm/integrations/custom_logger_v2/enums.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
from enum import Enum, IntEnum
|
||||
from typing import Final
|
||||
|
||||
|
||||
class Hook(str, Enum):
|
||||
prompt_resolve = "prompt_resolve"
|
||||
pre_route = "pre_route"
|
||||
filter_deployments = "filter_deployments"
|
||||
pre_call = "pre_call"
|
||||
during_call = "during_call"
|
||||
pre_deployment_call = "pre_deployment_call"
|
||||
post_deployment_success = "post_deployment_success"
|
||||
post_deployment_failure = "post_deployment_failure"
|
||||
post_call_success = "post_call_success"
|
||||
post_call_failure = "post_call_failure"
|
||||
response_headers = "response_headers"
|
||||
stream_chunk = "stream_chunk"
|
||||
list_models = "list_models"
|
||||
mcp_tool_result = "mcp_tool_result"
|
||||
log_prepare = "log_prepare"
|
||||
provider_request_sent = "provider_request_sent"
|
||||
provider_response_received = "provider_response_received"
|
||||
attempt_ended = "attempt_ended"
|
||||
call_ended = "call_ended"
|
||||
router_event = "router_event"
|
||||
audit_event = "audit_event"
|
||||
|
||||
|
||||
class FieldTier(IntEnum):
|
||||
small_record = 1
|
||||
content = 2
|
||||
wire = 3
|
||||
compatibility = 4
|
||||
|
||||
|
||||
class Field(str, Enum):
|
||||
_value_: str
|
||||
bit: int
|
||||
tier: FieldTier
|
||||
|
||||
def __new__(cls, value: str, bit: int, tier: FieldTier) -> "Field":
|
||||
member: Final = str.__new__(cls, value)
|
||||
member._value_ = value
|
||||
member.bit = bit
|
||||
member.tier = tier
|
||||
return member
|
||||
|
||||
model = ("model", 0, FieldTier.small_record)
|
||||
key_identity = ("key_identity", 1, FieldTier.small_record)
|
||||
deployment = ("deployment", 2, FieldTier.small_record)
|
||||
candidate_deployments = ("candidate_deployments", 3, FieldTier.small_record)
|
||||
request_tags = ("request_tags", 4, FieldTier.small_record)
|
||||
dynamic_params = ("dynamic_params", 5, FieldTier.small_record)
|
||||
guardrail_info = ("guardrail_info", 6, FieldTier.small_record)
|
||||
outcome = ("outcome", 7, FieldTier.small_record)
|
||||
error = ("error", 8, FieldTier.small_record)
|
||||
usage = ("usage", 9, FieldTier.small_record)
|
||||
cost = ("cost", 10, FieldTier.small_record)
|
||||
metadata = ("metadata", 11, FieldTier.content)
|
||||
messages = ("messages", 12, FieldTier.content)
|
||||
input = ("input", 13, FieldTier.content)
|
||||
tools = ("tools", 14, FieldTier.content)
|
||||
optional_params = ("optional_params", 15, FieldTier.content)
|
||||
request_headers = ("request_headers", 16, FieldTier.content)
|
||||
response = ("response", 17, FieldTier.content)
|
||||
response_headers = ("response_headers", 18, FieldTier.content)
|
||||
stream_chunk = ("stream_chunk", 19, FieldTier.content)
|
||||
assembled_stream = ("assembled_stream", 20, FieldTier.content)
|
||||
tool_result = ("tool_result", 21, FieldTier.content)
|
||||
model_list = ("model_list", 22, FieldTier.content)
|
||||
request_body = ("request_body", 23, FieldTier.wire)
|
||||
provider_request_body = ("provider_request_body", 24, FieldTier.wire)
|
||||
provider_response_raw = ("provider_response_raw", 25, FieldTier.wire)
|
||||
provider_response_headers = ("provider_response_headers", 26, FieldTier.wire)
|
||||
standard_logging_payload = ("standard_logging_payload", 27, FieldTier.compatibility)
|
||||
deprecated_kwargs = ("deprecated_kwargs", 28, FieldTier.compatibility)
|
||||
deprecated_response_obj = ("deprecated_response_obj", 29, FieldTier.compatibility)
|
||||
|
||||
|
||||
class Activation(str, Enum):
|
||||
required = "required"
|
||||
default_on = "default_on"
|
||||
opt_in = "opt_in"
|
||||
|
||||
|
||||
class OptOutBy(str, Enum):
|
||||
admin = "admin"
|
||||
caller = "caller"
|
||||
|
||||
|
||||
ALL_FIELDS_MASK: Final = sum(1 << field.bit for field in Field)
|
||||
|
||||
|
||||
def field_mask(fields: frozenset[Field]) -> int:
|
||||
if Field.deprecated_kwargs in fields:
|
||||
return ALL_FIELDS_MASK
|
||||
return sum(1 << field.bit for field in fields)
|
||||
98
litellm/integrations/custom_logger_v2/hook_point_fields.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from collections.abc import Iterator, Mapping
|
||||
from functools import reduce
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Field
|
||||
from litellm.integrations.custom_logger_v2.legacy import KwargsFields
|
||||
from litellm.integrations.custom_logger_v2.outcomes import InvalidPatchError, Patch, PatchOp, SetField
|
||||
from litellm.integrations.custom_logger_v2.patches import LiveFields
|
||||
|
||||
_CANDIDATE_FIELDS: Final = (Field.deprecated_kwargs, Field.candidate_deployments, Field.model)
|
||||
|
||||
|
||||
class CandidateFields(LiveFields):
|
||||
__slots__ = ("_deployments", "_model")
|
||||
|
||||
def __init__(self, deployments: object, model: str) -> None:
|
||||
self._deployments: Final = deployments
|
||||
self._model: Final = model
|
||||
|
||||
@property
|
||||
def deployments(self) -> object:
|
||||
return self._deployments
|
||||
|
||||
def __getitem__(self, field: Field) -> object:
|
||||
if field is Field.deprecated_kwargs:
|
||||
return self._deployments
|
||||
if field is Field.model:
|
||||
return self._model
|
||||
if field is Field.candidate_deployments and isinstance(self._deployments, list | tuple):
|
||||
return tuple(self._deployments) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts
|
||||
raise KeyError(field)
|
||||
|
||||
def __iter__(self) -> Iterator[Field]:
|
||||
return iter(tuple(field for field in _CANDIDATE_FIELDS if self._has(field)))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return sum(1 for field in _CANDIDATE_FIELDS if self._has(field))
|
||||
|
||||
def _has(self, field: Field) -> bool:
|
||||
return field is not Field.candidate_deployments or isinstance(self._deployments, list | tuple)
|
||||
|
||||
def apply(self, patch: Patch) -> Mapping[Field, object]:
|
||||
return reduce(_narrow, patch.ops, self)
|
||||
|
||||
|
||||
def _narrow(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]:
|
||||
if not isinstance(fields, CandidateFields) or not isinstance(op, SetField):
|
||||
raise InvalidPatchError("candidate deployments are replaced whole, never merged")
|
||||
if op.field is Field.deprecated_kwargs:
|
||||
return CandidateFields(op.value, str(fields[Field.model]))
|
||||
if op.field is not Field.candidate_deployments:
|
||||
raise InvalidPatchError(f"{op.field.value} has no place in filter_deployments")
|
||||
return CandidateFields(_narrowed(fields.deployments, op.value), str(fields[Field.model]))
|
||||
|
||||
|
||||
def _narrowed(current: object, proposed: object) -> list[object]: # mutable-ok: legacy filters take a list
|
||||
if not isinstance(current, list | tuple) or not isinstance(proposed, list | tuple):
|
||||
raise InvalidPatchError("candidate_deployments must stay a sequence of the current candidates")
|
||||
allowed: Final = frozenset(map(id, current)) # pyright: ignore[reportUnknownArgumentType] # untyped deployment dicts
|
||||
picked: Final = tuple(map(id, proposed)) # pyright: ignore[reportUnknownArgumentType] # see above
|
||||
if not allowed.issuperset(picked) or len(frozenset(picked)) != len(picked):
|
||||
raise InvalidPatchError("filter_deployments may only narrow the candidates, never add or repeat one")
|
||||
return list(proposed) # pyright: ignore[reportUnknownArgumentType] # see above
|
||||
|
||||
|
||||
class ResultFields(LiveFields):
|
||||
__slots__ = ("_request", "_slot", "value")
|
||||
|
||||
def __init__(self, request: object, keys: Mapping[Field, str], slot: Field, value: object) -> None:
|
||||
self._request: Final = KwargsFields(request, keys)
|
||||
self._slot: Final = slot
|
||||
self.value = value
|
||||
|
||||
def __getitem__(self, field: Field) -> object:
|
||||
if field is self._slot or field is Field.deprecated_response_obj:
|
||||
return self.value
|
||||
return self._request[field]
|
||||
|
||||
def __iter__(self) -> Iterator[Field]:
|
||||
return iter((*self._request, self._slot, Field.deprecated_response_obj))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._request) + 2
|
||||
|
||||
def apply(self, patch: Patch) -> Mapping[Field, object]:
|
||||
values: Final = tuple(_slot_value(op, self._slot) for op in patch.ops)
|
||||
if values:
|
||||
self.value = values[-1]
|
||||
return self
|
||||
|
||||
|
||||
def _slot_value(op: PatchOp, slot: Field) -> object:
|
||||
if not isinstance(op, SetField) or op.field not in (slot, Field.deprecated_response_obj):
|
||||
raise InvalidPatchError(f"{op.field.value} cannot be patched here, only {slot.value}")
|
||||
return op.value
|
||||
|
||||
|
||||
__all__ = ["CandidateFields", "ResultFields"]
|
||||
211
litellm/integrations/custom_logger_v2/hooks.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, Reroute, Respond
|
||||
|
||||
LegacyGuardrailEvent: TypeAlias = Literal["pre_call", "during_call", "post_call", "post_mcp_call"]
|
||||
OutcomeType: TypeAlias = type[Abort] | type[Respond] | type[Reroute]
|
||||
|
||||
|
||||
class InvalidDeclarationError(TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HookSpec:
|
||||
legacy_methods: tuple[str, ...]
|
||||
request_path: bool
|
||||
writes: frozenset[Field] = frozenset()
|
||||
outcomes: frozenset[OutcomeType] = frozenset()
|
||||
legacy_guardrail_events: frozenset[LegacyGuardrailEvent] = frozenset()
|
||||
has_call: bool = True
|
||||
subscribable: bool = True
|
||||
fail_fast_parallel: bool = False
|
||||
|
||||
|
||||
_NO_OUTCOME: Final[frozenset[OutcomeType]] = frozenset()
|
||||
_ABORT: Final[frozenset[OutcomeType]] = frozenset({Abort})
|
||||
_ABORT_OR_RESPOND: Final[frozenset[OutcomeType]] = frozenset({Abort, Respond})
|
||||
_ANY_OUTCOME: Final[frozenset[OutcomeType]] = frozenset({Abort, Respond, Reroute})
|
||||
|
||||
HOOK_SPECS: Final[Mapping[Hook, HookSpec]] = MappingProxyType(
|
||||
{
|
||||
Hook.prompt_resolve: HookSpec(
|
||||
legacy_methods=("get_chat_completion_prompt", "async_get_chat_completion_prompt"),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.messages, Field.optional_params, Field.model}),
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
),
|
||||
Hook.pre_route: HookSpec(
|
||||
legacy_methods=("async_pre_routing_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.model, Field.messages}),
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
),
|
||||
Hook.filter_deployments: HookSpec(
|
||||
legacy_methods=("async_filter_deployments",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.candidate_deployments}),
|
||||
outcomes=_ABORT,
|
||||
),
|
||||
Hook.pre_call: HookSpec(
|
||||
legacy_methods=("async_pre_call_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset(
|
||||
{
|
||||
Field.messages,
|
||||
Field.optional_params,
|
||||
Field.metadata,
|
||||
Field.request_headers,
|
||||
Field.guardrail_info,
|
||||
Field.tools,
|
||||
Field.model,
|
||||
Field.input,
|
||||
}
|
||||
),
|
||||
outcomes=_ANY_OUTCOME,
|
||||
legacy_guardrail_events=frozenset({"pre_call"}),
|
||||
),
|
||||
Hook.during_call: HookSpec(
|
||||
legacy_methods=("async_moderation_hook",),
|
||||
request_path=True,
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
legacy_guardrail_events=frozenset({"during_call"}),
|
||||
fail_fast_parallel=True,
|
||||
),
|
||||
Hook.pre_deployment_call: HookSpec(
|
||||
legacy_methods=("async_pre_call_deployment_hook", "async_pre_request_hook"),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.provider_request_body, Field.request_headers}),
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
),
|
||||
Hook.post_deployment_success: HookSpec(
|
||||
legacy_methods=("async_post_call_success_deployment_hook", "async_post_call_streaming_deployment_hook"),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.response}),
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
),
|
||||
Hook.post_deployment_failure: HookSpec(
|
||||
legacy_methods=("async_post_call_failure_deployment_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.error}),
|
||||
outcomes=_ABORT,
|
||||
),
|
||||
Hook.post_call_success: HookSpec(
|
||||
legacy_methods=("async_post_call_success_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.response, Field.guardrail_info}),
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
legacy_guardrail_events=frozenset({"post_call"}),
|
||||
),
|
||||
Hook.post_call_failure: HookSpec(
|
||||
legacy_methods=("async_post_call_failure_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.error}),
|
||||
outcomes=_ABORT,
|
||||
),
|
||||
Hook.response_headers: HookSpec(
|
||||
legacy_methods=("async_post_call_response_headers_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.response_headers}),
|
||||
outcomes=_NO_OUTCOME,
|
||||
),
|
||||
Hook.stream_chunk: HookSpec(
|
||||
legacy_methods=(
|
||||
"async_post_call_streaming_hook",
|
||||
"async_post_call_streaming_iterator_hook",
|
||||
"log_stream_event",
|
||||
),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.stream_chunk}),
|
||||
outcomes=_ABORT,
|
||||
subscribable=False,
|
||||
),
|
||||
Hook.list_models: HookSpec(
|
||||
legacy_methods=("async_filter_listed_models",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.model_list}),
|
||||
outcomes=_ABORT,
|
||||
has_call=False,
|
||||
),
|
||||
Hook.mcp_tool_result: HookSpec(
|
||||
legacy_methods=("async_post_mcp_tool_call_hook",),
|
||||
request_path=True,
|
||||
writes=frozenset({Field.tool_result}),
|
||||
outcomes=_ABORT_OR_RESPOND,
|
||||
legacy_guardrail_events=frozenset({"post_mcp_call"}),
|
||||
),
|
||||
Hook.log_prepare: HookSpec(
|
||||
legacy_methods=(
|
||||
"logging_hook",
|
||||
"async_logging_hook",
|
||||
"redact_standard_logging_payload_from_model_call_details",
|
||||
),
|
||||
request_path=False,
|
||||
writes=frozenset({Field.standard_logging_payload}),
|
||||
),
|
||||
Hook.provider_request_sent: HookSpec(legacy_methods=("log_pre_api_call",), request_path=False),
|
||||
Hook.provider_response_received: HookSpec(legacy_methods=("log_post_api_call",), request_path=False),
|
||||
Hook.attempt_ended: HookSpec(
|
||||
legacy_methods=(
|
||||
"log_success_event",
|
||||
"log_failure_event",
|
||||
"async_log_success_event",
|
||||
"async_log_failure_event",
|
||||
),
|
||||
request_path=False,
|
||||
),
|
||||
Hook.call_ended: HookSpec(legacy_methods=(), request_path=False),
|
||||
Hook.router_event: HookSpec(
|
||||
legacy_methods=(
|
||||
"log_success_fallback_event",
|
||||
"log_failure_fallback_event",
|
||||
"log_model_group_rate_limit_error",
|
||||
),
|
||||
request_path=False,
|
||||
),
|
||||
Hook.audit_event: HookSpec(
|
||||
legacy_methods=("async_log_audit_log_event", "async_log_batch_line_items"),
|
||||
request_path=False,
|
||||
has_call=False,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Subscription:
|
||||
hook: Hook
|
||||
handler: str
|
||||
reads: frozenset[Field] = frozenset()
|
||||
writes: frozenset[Field] = frozenset()
|
||||
may_abort: bool = False
|
||||
parallel: bool = False
|
||||
awaited: bool = False
|
||||
activation: Activation | None = None
|
||||
legacy: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
problem: Final = subscription_problem(self)
|
||||
if problem is not None:
|
||||
raise InvalidDeclarationError(f"{self.handler} on {self.hook.value}: {problem}")
|
||||
|
||||
|
||||
def subscription_problem(subscription: Subscription) -> str | None:
|
||||
spec: Final = HOOK_SPECS[subscription.hook]
|
||||
if not spec.subscribable:
|
||||
return "this hook point has no V2 subscription support yet"
|
||||
legacy_writes: Final = frozenset({Field.deprecated_kwargs}) if subscription.legacy else frozenset[Field]()
|
||||
forbidden: Final = subscription.writes - spec.writes - legacy_writes
|
||||
if forbidden:
|
||||
return f"writes {', '.join(sorted(field.value for field in forbidden))}, which this hook point cannot apply"
|
||||
if subscription.may_abort and not spec.outcomes:
|
||||
return "may_abort is set, but this hook point allows no outcome"
|
||||
if subscription.awaited and (subscription.writes or subscription.may_abort):
|
||||
return "an awaited subscription must neither write nor abort"
|
||||
if subscription.parallel and (subscription.writes or not subscription.may_abort):
|
||||
return "parallel is only for subscriptions that may abort and write nothing"
|
||||
return None
|
||||
62
litellm/integrations/custom_logger_v2/host.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.custom_logger_v2.dispatcher import Dispatcher, RequestScope
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Outcome, Patch
|
||||
from litellm.integrations.custom_logger_v2.registry import CallbackRegistry
|
||||
from litellm.integrations.custom_logger_v2.selection import RequestSelection
|
||||
|
||||
_NO_SELECTION: Final = RequestSelection()
|
||||
_CURRENT_SCOPE: Final[ContextVar[RequestScope | None]] = ContextVar("custom_logger_v2_scope", default=None)
|
||||
|
||||
|
||||
class V2Outcome(Exception):
|
||||
def __init__(self, outcome: Outcome | None, patches: tuple[Patch, ...]) -> None:
|
||||
super().__init__(f"a V2 callback ended the hook with {type(outcome).__name__}")
|
||||
self.outcome: Final = outcome
|
||||
self.patches: Final = patches
|
||||
|
||||
|
||||
class _Host:
|
||||
def __init__(self) -> None:
|
||||
self._dispatcher: Dispatcher | None = None
|
||||
|
||||
def dispatcher(self) -> Dispatcher:
|
||||
if self._dispatcher is None:
|
||||
self._dispatcher = Dispatcher(CallbackRegistry())
|
||||
return self._dispatcher
|
||||
|
||||
def reset(self) -> None:
|
||||
self._dispatcher = None
|
||||
|
||||
|
||||
_HOST: Final = _Host()
|
||||
|
||||
|
||||
def get_dispatcher() -> Dispatcher:
|
||||
return _HOST.dispatcher()
|
||||
|
||||
|
||||
def reset_host_for_tests() -> None:
|
||||
_HOST.reset()
|
||||
|
||||
|
||||
def current_scope() -> RequestScope | None:
|
||||
return _CURRENT_SCOPE.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def bind_scope(scope: RequestScope) -> Generator[RequestScope]:
|
||||
token: Final = _CURRENT_SCOPE.set(scope)
|
||||
try:
|
||||
yield scope
|
||||
finally:
|
||||
_CURRENT_SCOPE.reset(token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def request_scope(selection: RequestSelection = _NO_SELECTION) -> Generator[RequestScope]:
|
||||
with _HOST.dispatcher().open_scope(selection) as scope, bind_scope(scope):
|
||||
yield scope
|
||||
434
litellm/integrations/custom_logger_v2/legacy.py
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
import weakref
|
||||
from collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import lru_cache, reduce
|
||||
from itertools import count
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, TypeVar
|
||||
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows the untyped legacy kwargs without copying them
|
||||
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger, subclass_overrides_hook
|
||||
from litellm.integrations.custom_logger_v2.call import CallFacts
|
||||
from litellm.integrations.custom_logger_v2.dispatcher import (
|
||||
Between,
|
||||
HookResult,
|
||||
RequestScope,
|
||||
run_legacy_plan,
|
||||
run_legacy_plan_sync,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS, Subscription
|
||||
from litellm.integrations.custom_logger_v2.host import V2Outcome, current_scope, get_dispatcher
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
|
||||
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest, LegacyBinding, build_manifest
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, InvalidPatchError, MergeField, Patch, PatchOp, Respond
|
||||
from litellm.integrations.custom_logger_v2.patches import LiveFields
|
||||
from litellm.integrations.custom_logger_v2.planner import Entry, HookPlan, plan_hook
|
||||
from litellm.integrations.custom_logger_v2.selection import RequestSelection, Selection
|
||||
|
||||
ResultT: Final = TypeVar("ResultT")
|
||||
|
||||
SyncLegacyBinding: TypeAlias = Callable[[object, object], object]
|
||||
|
||||
# These hook points are the only ones where v1 custom logger subclasses are
|
||||
# checked to see if they run in parallel
|
||||
PARALLEL_GUARDRAIL_HOOKS: Final = frozenset({Hook.pre_call, Hook.post_call_success})
|
||||
LEGACY_READS: Final = frozenset({Field.deprecated_kwargs, Field.deprecated_response_obj})
|
||||
PROXY_REQUEST_KEYS: Final[Mapping[Field, str]] = MappingProxyType(
|
||||
{
|
||||
Field.model: "model",
|
||||
Field.messages: "messages",
|
||||
Field.input: "input",
|
||||
Field.tools: "tools",
|
||||
}
|
||||
)
|
||||
SUBSCRIBERS_CACHE_SIZE: Final = 128
|
||||
PLAN_CACHE_SIZE: Final = 64
|
||||
|
||||
_SERIALS: Final = count()
|
||||
|
||||
|
||||
class Admit(str, Enum):
|
||||
overriders = "overriders"
|
||||
every_logger = "every_logger"
|
||||
every_entry = "every_entry"
|
||||
|
||||
|
||||
_ABSENT: Final = object()
|
||||
_NO_SELECTION: Final = RequestSelection()
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def legacy_subscription(hook: Hook, parallel: bool) -> Subscription:
|
||||
spec: Final = HOOK_SPECS[hook]
|
||||
may_abort: Final = bool(spec.outcomes)
|
||||
gathered: Final = may_abort and (parallel or hook is Hook.during_call)
|
||||
return Subscription(
|
||||
hook=hook,
|
||||
handler="call",
|
||||
reads=LEGACY_READS,
|
||||
writes=frozenset({Field.deprecated_kwargs}) if spec.writes and not gathered else frozenset(),
|
||||
may_abort=may_abort,
|
||||
parallel=gathered,
|
||||
legacy=True,
|
||||
)
|
||||
|
||||
|
||||
def _subscribes(cls: type, guardrail: bool, hook: Hook) -> bool:
|
||||
spec: Final = HOOK_SPECS[hook]
|
||||
if not spec.subscribable:
|
||||
return False
|
||||
if guardrail and spec.legacy_guardrail_events:
|
||||
return True
|
||||
return any(subclass_overrides_hook(cls, method) for method in spec.legacy_methods)
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def _derived_subscriptions(cls: type, guardrail: bool, parallel: bool) -> Mapping[Hook, Subscription]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
hook: legacy_subscription(hook, parallel and hook in PARALLEL_GUARDRAIL_HOOKS)
|
||||
for hook in Hook
|
||||
if _subscribes(cls, guardrail, hook)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class LegacyLogger(CustomLoggerV2):
|
||||
def __init__(self, inner: CustomLogger) -> None:
|
||||
guardrail: Final = isinstance(inner, CustomGuardrail)
|
||||
self.parallel = guardrail and bool(getattr(inner, "run_in_parallel", False))
|
||||
self.serial: Final = next(_SERIALS)
|
||||
self._inner: Final = weakref.ref(inner)
|
||||
self.manifest: Final = CallbackManifest(
|
||||
logger=self,
|
||||
selection=Selection(name=type(inner).__name__, activation=Activation.required),
|
||||
subscriptions=_derived_subscriptions(type(inner), guardrail, self.parallel),
|
||||
request_state=None,
|
||||
legacy_call=self.call,
|
||||
)
|
||||
|
||||
@property
|
||||
def inner(self) -> CustomLogger | None:
|
||||
return self._inner()
|
||||
|
||||
def subscription(self, hook: Hook, admit: Admit) -> Subscription | None:
|
||||
derived: Final = self.manifest.subscriptions.get(hook)
|
||||
if derived is not None or admit is Admit.overriders or not HOOK_SPECS[hook].subscribable:
|
||||
return derived
|
||||
return legacy_subscription(hook, self.parallel and hook in PARALLEL_GUARDRAIL_HOOKS)
|
||||
|
||||
def call(self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object]) -> ResultT | None:
|
||||
inner: Final = self._inner()
|
||||
return None if inner is None else binding(inner, fields[Field.deprecated_kwargs])
|
||||
|
||||
|
||||
_ADAPTERS: Final[dict[int, LegacyLogger]] = {} # mutable-ok: adapters live as long as the logger they wrap
|
||||
|
||||
|
||||
def adapter_for(inner: CustomLogger) -> LegacyLogger:
|
||||
key: Final = id(inner)
|
||||
cached: Final = _ADAPTERS.get(key)
|
||||
if cached is not None and cached.inner is inner:
|
||||
return cached
|
||||
adapter: Final = LegacyLogger(inner)
|
||||
_ADAPTERS[key] = adapter
|
||||
_ = weakref.finalize(inner, _ADAPTERS.pop, key, None)
|
||||
return adapter
|
||||
|
||||
|
||||
def _readable(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: legacy kwargs are an untyped dict
|
||||
return isinstance(value, Mapping)
|
||||
|
||||
|
||||
def _writable(
|
||||
value: object,
|
||||
) -> TypeIs[MutableMapping[str, object]]: # guard-ok: legacy kwargs are an untyped dict # mutable-ok: written in place
|
||||
return isinstance(value, MutableMapping)
|
||||
|
||||
|
||||
class OpaqueLegacy(CustomLoggerV2):
|
||||
"""Holds a callback list entry that is not a CustomLogger (a string name or a plain function) at its list
|
||||
position, for hook points that admit every entry. There is no class to derive subscriptions from, so it
|
||||
subscribes to the one hook it was built for and runs serially."""
|
||||
|
||||
def __init__(self, inner: object, hook: Hook) -> None:
|
||||
self.inner: Final = inner
|
||||
self.serial: Final = next(_SERIALS)
|
||||
self.manifest: Final = CallbackManifest(
|
||||
logger=self,
|
||||
selection=Selection(name=type(inner).__name__, activation=Activation.required),
|
||||
subscriptions=MappingProxyType({hook: legacy_subscription(hook, False)}),
|
||||
request_state=None,
|
||||
legacy_call=self.call,
|
||||
)
|
||||
|
||||
def call(self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object]) -> ResultT | None:
|
||||
return binding(self.inner, fields[Field.deprecated_kwargs])
|
||||
|
||||
|
||||
class KwargsFields(LiveFields):
|
||||
"""A live Field view over a legacy hook's untyped kwargs dict, so the dispatcher can run over it without copying.
|
||||
|
||||
Two fields cover the whole call: deprecated_kwargs is the dict itself, and deprecated_response_obj is the
|
||||
response, when the hook point passes one. Patching either one swaps in the new value.
|
||||
|
||||
Every other field maps to one key in the dict through the hook point's key table (PROXY_REQUEST_KEYS or
|
||||
LOGGING_KEYS), and is present only when that key is. Patching one writes straight into the dict, so later
|
||||
callbacks and the request see the change."""
|
||||
|
||||
__slots__ = ("_keys", "_kwargs", "_response")
|
||||
|
||||
def __init__(self, kwargs: object, keys: Mapping[Field, str], response: object = _ABSENT) -> None:
|
||||
self._kwargs: Final = kwargs
|
||||
self._keys: Final = keys
|
||||
self._response: Final = response
|
||||
|
||||
def _key(self, field: Field) -> str | None:
|
||||
key: Final = self._keys.get(field)
|
||||
return key if key is not None and _readable(self._kwargs) and key in self._kwargs else None
|
||||
|
||||
def __getitem__(self, field: Field) -> object:
|
||||
if field is Field.deprecated_kwargs:
|
||||
return self._kwargs
|
||||
if field is Field.deprecated_response_obj and self._response is not _ABSENT:
|
||||
return self._response
|
||||
key: Final = self._key(field)
|
||||
if key is None or not _readable(self._kwargs):
|
||||
raise KeyError(field)
|
||||
return self._kwargs[key]
|
||||
|
||||
def __iter__(self) -> Iterator[Field]:
|
||||
return iter(tuple(field for field in Field if self._has(field)))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return sum(1 for field in Field if self._has(field))
|
||||
|
||||
def _has(self, field: Field) -> bool:
|
||||
if field is Field.deprecated_kwargs:
|
||||
return True
|
||||
if field is Field.deprecated_response_obj:
|
||||
return self._response is not _ABSENT
|
||||
return self._key(field) is not None
|
||||
|
||||
def apply(self, patch: Patch) -> Mapping[Field, object]:
|
||||
return reduce(_apply_op, patch.ops, self)
|
||||
|
||||
def replaced(self, field: Field, value: object) -> "KwargsFields":
|
||||
if field is Field.deprecated_kwargs:
|
||||
return KwargsFields(value, self._keys, self._response)
|
||||
return KwargsFields(self._kwargs, self._keys, value)
|
||||
|
||||
def write(self, field: Field, value: object) -> None:
|
||||
target: Final = self._target(field)
|
||||
target[self._keys[field]] = value
|
||||
|
||||
def merge(self, field: Field, keys: Mapping[str, object]) -> None:
|
||||
target: Final = self._target(field)
|
||||
current: Final = target.get(self._keys[field])
|
||||
if current is None:
|
||||
target[self._keys[field]] = dict(keys)
|
||||
return
|
||||
if not _writable(current):
|
||||
raise InvalidPatchError(f"{field.value} holds no mapping here, so keys cannot be merged into it")
|
||||
current.update(keys)
|
||||
|
||||
def _target(self, field: Field) -> MutableMapping[str, object]: # mutable-ok: legacy kwargs are the live request
|
||||
if field not in self._keys:
|
||||
raise InvalidPatchError(f"{field.value} has no place in the legacy kwargs of this hook")
|
||||
if not _writable(self._kwargs):
|
||||
raise InvalidPatchError(f"{field.value} cannot be written because the legacy kwargs are not a mapping")
|
||||
return self._kwargs
|
||||
|
||||
|
||||
def _apply_op(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]:
|
||||
if not isinstance(fields, KwargsFields):
|
||||
raise InvalidPatchError("legacy kwargs patches need KwargsFields")
|
||||
if op.field in (Field.deprecated_kwargs, Field.deprecated_response_obj):
|
||||
if isinstance(op, MergeField):
|
||||
raise InvalidPatchError(f"{op.field.value} is replaced whole, never merged")
|
||||
return fields.replaced(op.field, op.value)
|
||||
if isinstance(op, MergeField):
|
||||
fields.merge(op.field, op.keys)
|
||||
return fields
|
||||
fields.write(op.field, op.value)
|
||||
return fields
|
||||
|
||||
|
||||
def kwargs_patch(before: object, after: object) -> Patch | None:
|
||||
return None if after is before else Patch().set(Field.deprecated_kwargs, after)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, eq=False)
|
||||
class HookSubscribers:
|
||||
hook: Hook
|
||||
entries: tuple[Entry, ...]
|
||||
plan: HookPlan
|
||||
has_v2: bool
|
||||
plans: Callable[[tuple[int, ...]], HookPlan]
|
||||
|
||||
@property
|
||||
def legacy(self) -> tuple[object, ...]:
|
||||
return tuple(inner for entry in self.entries if (inner := _wrapped(entry.manifest.logger)) is not None)
|
||||
|
||||
def plan_for(self, scope: RequestScope) -> HookPlan:
|
||||
selected: Final = tuple(
|
||||
index for index, entry in enumerate(self.entries) if entry.subscription.legacy or scope.selects(entry)
|
||||
)
|
||||
return self.plan if len(selected) == len(self.entries) else self.plans(selected)
|
||||
|
||||
|
||||
def _wrapped(logger: CustomLoggerV2) -> object:
|
||||
if isinstance(logger, LegacyLogger | OpaqueLegacy):
|
||||
return logger.inner
|
||||
return None
|
||||
|
||||
|
||||
def _v2_entry(v2_callback: CustomLoggerV2, hook: Hook) -> tuple[CallbackManifest, Subscription] | None:
|
||||
manifest: Final = build_manifest(v2_callback)
|
||||
subscription: Final = manifest.subscriptions.get(hook)
|
||||
return None if subscription is None else (manifest, subscription)
|
||||
|
||||
|
||||
def _legacy_entry(
|
||||
inner: CustomLogger, hook: Hook, admit: Admit, parallel: bool
|
||||
) -> tuple[CallbackManifest, Subscription] | None:
|
||||
adapter: Final = adapter_for(inner)
|
||||
subscription: Final = adapter.subscription(hook, admit)
|
||||
if subscription is None:
|
||||
return None
|
||||
if subscription.parallel and not parallel:
|
||||
return adapter.manifest, legacy_subscription(hook, False)
|
||||
return adapter.manifest, subscription
|
||||
|
||||
|
||||
def _opaque_entry(item: object, hook: Hook) -> tuple[CallbackManifest, Subscription] | None:
|
||||
if not HOOK_SPECS[hook].subscribable:
|
||||
return None
|
||||
opaque: Final = OpaqueLegacy(item, hook)
|
||||
return opaque.manifest, opaque.manifest.subscriptions[hook]
|
||||
|
||||
|
||||
def _subscriber(
|
||||
item: object, hook: Hook, admit: Admit, include_v2: bool, parallel: bool
|
||||
) -> tuple[CallbackManifest, Subscription] | None:
|
||||
if isinstance(item, CustomLoggerV2):
|
||||
return _v2_entry(item, hook) if include_v2 else None
|
||||
if isinstance(item, CustomLogger):
|
||||
return _legacy_entry(item, hook, admit, parallel)
|
||||
return _opaque_entry(item, hook) if admit is Admit.every_entry else None
|
||||
|
||||
|
||||
def _build_subscribers(
|
||||
hook: Hook, callbacks: Sequence[object], admit: Admit, include_v2: bool, parallel: bool
|
||||
) -> HookSubscribers:
|
||||
members: Final = tuple(
|
||||
member for item in callbacks if (member := _subscriber(item, hook, admit, include_v2, parallel)) is not None
|
||||
)
|
||||
entries: Final = tuple(
|
||||
Entry(position, manifest, subscription) for position, (manifest, subscription) in enumerate(members)
|
||||
)
|
||||
|
||||
def subset_plan(selected: tuple[int, ...]) -> HookPlan:
|
||||
return plan_hook(tuple(entries[index] for index in selected))
|
||||
|
||||
return HookSubscribers(
|
||||
hook=hook,
|
||||
entries=entries,
|
||||
plan=plan_hook(entries),
|
||||
has_v2=any(not entry.subscription.legacy for entry in entries),
|
||||
plans=lru_cache(maxsize=PLAN_CACHE_SIZE)(subset_plan),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, eq=False)
|
||||
class _CachedSubscribers:
|
||||
subscribers: HookSubscribers
|
||||
pinned: tuple[object, ...]
|
||||
|
||||
|
||||
_SUBSCRIBERS: Final[dict[tuple[object, ...], _CachedSubscribers]] = {} # mutable-ok: bounded subscribers cache
|
||||
|
||||
|
||||
def _pinned(item: object) -> bool:
|
||||
return not isinstance(item, CustomLogger) or isinstance(item, CustomLoggerV2)
|
||||
|
||||
|
||||
def _cache_subscribers(key: tuple[object, ...], subscribers: HookSubscribers, callbacks: Sequence[object]) -> None:
|
||||
if len(_SUBSCRIBERS) >= SUBSCRIBERS_CACHE_SIZE:
|
||||
_SUBSCRIBERS.clear()
|
||||
_SUBSCRIBERS[key] = _CachedSubscribers(
|
||||
subscribers=subscribers, pinned=tuple(item for item in callbacks if _pinned(item))
|
||||
)
|
||||
for item in callbacks:
|
||||
if not _pinned(item):
|
||||
_ = weakref.finalize(item, _SUBSCRIBERS.pop, key, None)
|
||||
|
||||
|
||||
def hook_subscribers(
|
||||
hook: Hook,
|
||||
callbacks: Sequence[object],
|
||||
*,
|
||||
admit: Admit = Admit.overriders,
|
||||
include_v2: bool = True,
|
||||
parallel: bool = True,
|
||||
) -> HookSubscribers:
|
||||
key: Final = (hook, admit, include_v2, parallel, *map(id, callbacks))
|
||||
cached: Final = _SUBSCRIBERS.get(key)
|
||||
if cached is not None:
|
||||
return cached.subscribers
|
||||
subscribers: Final = _build_subscribers(hook, callbacks, admit, include_v2, parallel)
|
||||
_cache_subscribers(key, subscribers, callbacks)
|
||||
return subscribers
|
||||
|
||||
|
||||
def clear_subscribers_cache_for_tests() -> None:
|
||||
_SUBSCRIBERS.clear()
|
||||
|
||||
|
||||
async def run_subscribers(
|
||||
subscribers: HookSubscribers,
|
||||
fields: Mapping[Field, object],
|
||||
binding: LegacyBinding,
|
||||
*,
|
||||
between: Between | None = None,
|
||||
call: CallFacts | None = None,
|
||||
scope: RequestScope | None = None,
|
||||
) -> HookResult:
|
||||
if not subscribers.has_v2:
|
||||
return HookResult(fields=await run_legacy_plan(subscribers.hook, subscribers.plan, fields, binding, between))
|
||||
bound: Final = scope or current_scope()
|
||||
if bound is not None:
|
||||
return await bound.run_plan(subscribers.hook, subscribers.plan_for(bound), fields, call, binding, between)
|
||||
with get_dispatcher().open_scope(_NO_SELECTION) as transient:
|
||||
return await transient.run_plan(
|
||||
subscribers.hook, subscribers.plan_for(transient), fields, call, binding, between
|
||||
)
|
||||
|
||||
|
||||
def run_subscribers_sync(
|
||||
subscribers: HookSubscribers,
|
||||
fields: Mapping[Field, object],
|
||||
binding: SyncLegacyBinding,
|
||||
*,
|
||||
call: CallFacts | None = None,
|
||||
scope: RequestScope | None = None,
|
||||
) -> Mapping[Field, object]:
|
||||
if not subscribers.has_v2:
|
||||
return run_legacy_plan_sync(subscribers.plan, fields, binding)
|
||||
bound: Final = scope or current_scope()
|
||||
if bound is not None:
|
||||
return bound.run_plan_sync(subscribers.hook, subscribers.plan_for(bound), fields, call, binding)
|
||||
with get_dispatcher().open_scope(_NO_SELECTION) as transient:
|
||||
return transient.run_plan_sync(subscribers.hook, subscribers.plan_for(transient), fields, call, binding)
|
||||
|
||||
|
||||
def kwargs_after(result: HookResult) -> object:
|
||||
match result.outcome:
|
||||
case Abort() | Respond() as terminal:
|
||||
raise V2Outcome(terminal, result.patches)
|
||||
case _:
|
||||
return result.fields[Field.deprecated_kwargs]
|
||||
109
litellm/integrations/custom_logger_v2/logger.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import inspect
|
||||
from collections.abc import Callable, Mapping, Sequence, Set
|
||||
from dataclasses import replace
|
||||
from types import MappingProxyType
|
||||
from typing import ClassVar, Final, TypeVar
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.hooks import InvalidDeclarationError, Subscription
|
||||
from litellm.integrations.custom_logger_v2.selection import Selection
|
||||
|
||||
HandlerT: Final = TypeVar("HandlerT", bound=Callable[..., object])
|
||||
|
||||
_SUBSCRIPTION_ATTR: Final = "__custom_logger_v2_subscription__"
|
||||
|
||||
|
||||
def on(
|
||||
hook: Hook,
|
||||
*,
|
||||
reads: Set[Field] = frozenset(),
|
||||
writes: Set[Field] = frozenset(),
|
||||
may_abort: bool = False,
|
||||
parallel: bool | None = None,
|
||||
awaited: bool = False,
|
||||
) -> Callable[[HandlerT], HandlerT]:
|
||||
def decorate(handler: HandlerT) -> HandlerT:
|
||||
subscription: Final = Subscription(
|
||||
hook=hook,
|
||||
handler=handler.__name__,
|
||||
reads=frozenset(reads),
|
||||
writes=frozenset(writes),
|
||||
may_abort=may_abort,
|
||||
parallel=(hook is Hook.during_call and may_abort) if parallel is None else parallel,
|
||||
awaited=awaited,
|
||||
)
|
||||
setattr(handler, _SUBSCRIPTION_ATTR, subscription)
|
||||
return handler
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
def _declared_subscription(value: object) -> Subscription | None:
|
||||
marker: Final[object] = getattr(value, _SUBSCRIPTION_ATTR, None)
|
||||
return marker if isinstance(marker, Subscription) else None
|
||||
|
||||
|
||||
def _namespace(cls: type) -> Mapping[str, object]:
|
||||
return vars(cls)
|
||||
|
||||
|
||||
def _declared_in(cls: type) -> Mapping[Hook, Subscription]:
|
||||
declared: Final = tuple(
|
||||
replace(subscription, handler=name)
|
||||
for name, value in _namespace(cls).items()
|
||||
if (subscription := _declared_subscription(value)) is not None
|
||||
)
|
||||
hooks: Final = tuple(subscription.hook for subscription in declared)
|
||||
duplicated: Final = sorted({hook.value for hook in hooks if hooks.count(hook) > 1})
|
||||
if duplicated:
|
||||
raise InvalidDeclarationError(f"{cls.__name__} declares more than one handler for {', '.join(duplicated)}")
|
||||
return MappingProxyType({subscription.hook: subscription for subscription in declared})
|
||||
|
||||
|
||||
def _nearest(layers: Sequence[Mapping[Hook, Subscription]], hook: Hook) -> Subscription | None:
|
||||
return next((layer[hook] for layer in layers if hook in layer), None)
|
||||
|
||||
|
||||
def _handler_problem(cls: type, subscription: Subscription) -> str | None:
|
||||
handler: Final[object] = getattr(cls, subscription.handler, None)
|
||||
if not callable(handler):
|
||||
return f"{subscription.handler} handles {subscription.hook.value} but is not callable"
|
||||
redeclared: Final = _declared_subscription(handler)
|
||||
if redeclared is not None and redeclared.hook is not subscription.hook:
|
||||
return (
|
||||
f"{subscription.handler} is decorated for {redeclared.hook.value} but also inherits "
|
||||
f"the {subscription.hook.value} subscription"
|
||||
)
|
||||
blocking: Final = bool(subscription.writes) or subscription.may_abort or subscription.awaited
|
||||
if blocking and not inspect.iscoroutinefunction(handler):
|
||||
return f"{subscription.handler} must be async because it writes, may abort or is awaited"
|
||||
return None
|
||||
|
||||
|
||||
class CustomLoggerV2:
|
||||
_own_subscriptions: ClassVar[Mapping[Hook, Subscription]] = MappingProxyType({})
|
||||
subscriptions: ClassVar[Mapping[Hook, Subscription]] = MappingProxyType({})
|
||||
request_state: ClassVar[type | None] = None
|
||||
declared_hooks: ClassVar[frozenset[Hook]] = frozenset()
|
||||
|
||||
selection: Selection | None = None
|
||||
enabled_hooks: frozenset[Hook] | None = None
|
||||
parallel: bool = False
|
||||
activation_overrides: Mapping[Hook, Activation] = MappingProxyType({})
|
||||
|
||||
def __init_subclass__(cls) -> None:
|
||||
super().__init_subclass__()
|
||||
cls._own_subscriptions = _declared_in(cls)
|
||||
layers: Final = tuple(base._own_subscriptions for base in cls.__mro__ if issubclass(base, CustomLoggerV2))
|
||||
resolved: Final = MappingProxyType(
|
||||
{hook: subscription for hook in Hook if (subscription := _nearest(layers, hook)) is not None}
|
||||
)
|
||||
problems: Final = tuple(
|
||||
problem
|
||||
for subscription in resolved.values()
|
||||
if (problem := _handler_problem(cls, subscription)) is not None
|
||||
)
|
||||
if problems:
|
||||
raise InvalidDeclarationError(f"{cls.__name__}: {'; '.join(problems)}")
|
||||
cls.subscriptions = resolved
|
||||
CustomLoggerV2.declared_hooks = CustomLoggerV2.declared_hooks | resolved.keys()
|
||||
73
litellm/integrations/custom_logger_v2/manifest.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, TypeAlias, TypeVar
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook, OptOutBy
|
||||
from litellm.integrations.custom_logger_v2.hooks import InvalidDeclarationError, Subscription
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
|
||||
from litellm.integrations.custom_logger_v2.selection import Selection
|
||||
|
||||
ResultT: Final = TypeVar("ResultT")
|
||||
|
||||
LegacyBinding: TypeAlias = Callable[[object, object], Awaitable[object] | None]
|
||||
|
||||
|
||||
class LegacyCall(Protocol):
|
||||
def __call__(
|
||||
self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object], /
|
||||
) -> ResultT | None: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallbackManifest:
|
||||
logger: CustomLoggerV2
|
||||
selection: Selection
|
||||
subscriptions: Mapping[Hook, Subscription]
|
||||
request_state: type | None
|
||||
legacy_call: LegacyCall | None = None
|
||||
|
||||
|
||||
def _instance_subscription(subscription: Subscription, parallel: bool, activation: Activation | None) -> Subscription:
|
||||
return replace(
|
||||
subscription,
|
||||
parallel=subscription.parallel or (parallel and subscription.may_abort),
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
|
||||
def _caller_can_remove(selection: Selection) -> bool:
|
||||
return selection.honors_no_log or OptOutBy.caller in selection.opt_out_by
|
||||
|
||||
|
||||
def build_manifest(logger: CustomLoggerV2) -> CallbackManifest:
|
||||
name: Final = type(logger).__name__
|
||||
declared: Final = type(logger).subscriptions
|
||||
enabled: Final = frozenset(declared) if logger.enabled_hooks is None else logger.enabled_hooks
|
||||
not_implemented: Final = enabled - declared.keys()
|
||||
if not_implemented:
|
||||
hooks: Final = ", ".join(sorted(hook.value for hook in not_implemented))
|
||||
raise InvalidDeclarationError(f"{name} enables {hooks}, which it does not implement")
|
||||
stray_overrides: Final = logger.activation_overrides.keys() - enabled
|
||||
if stray_overrides:
|
||||
overridden: Final = ", ".join(sorted(hook.value for hook in stray_overrides))
|
||||
raise InvalidDeclarationError(f"{name} overrides activation for {overridden}, which it does not enable")
|
||||
subscriptions: Final = MappingProxyType(
|
||||
{
|
||||
hook: _instance_subscription(declared[hook], logger.parallel, logger.activation_overrides.get(hook))
|
||||
for hook in Hook
|
||||
if hook in enabled
|
||||
}
|
||||
)
|
||||
selection: Final = logger.selection or Selection(name=name)
|
||||
blocking: Final = any(subscription.writes or subscription.may_abort for subscription in subscriptions.values())
|
||||
if blocking and _caller_can_remove(selection):
|
||||
raise InvalidDeclarationError(
|
||||
f"{selection.name} writes or may abort, so neither no-log nor a caller opt-out may remove it"
|
||||
)
|
||||
return CallbackManifest(
|
||||
logger=logger,
|
||||
selection=selection,
|
||||
subscriptions=subscriptions,
|
||||
request_state=type(logger).request_state,
|
||||
)
|
||||
27
litellm/integrations/custom_logger_v2/offpath.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import asyncio
|
||||
from collections.abc import Coroutine
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Final, Protocol
|
||||
|
||||
|
||||
class OffPathSink(Protocol):
|
||||
def submit(self, delivery: Coroutine[object, object, None]) -> None: ...
|
||||
|
||||
|
||||
class InProcessSink:
|
||||
def __init__(self) -> None:
|
||||
self._pending: set[asyncio.Task[None]] = set() # mutable-ok: strong references keep running tasks alive
|
||||
self._loopless: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="custom-logger-v2-off-path")
|
||||
|
||||
def submit(self, delivery: Coroutine[object, object, None]) -> None:
|
||||
try:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_ = self._loopless.submit(asyncio.run, delivery)
|
||||
return
|
||||
task: Final[asyncio.Task[None]] = loop.create_task(delivery)
|
||||
self._pending.add(task)
|
||||
task.add_done_callback(self._pending.discard)
|
||||
|
||||
async def drain(self) -> None:
|
||||
await asyncio.gather(*self._pending)
|
||||
106
litellm/integrations/custom_logger_v2/outcomes.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import ClassVar, Final, TypeAlias
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Field
|
||||
|
||||
JsonValue: TypeAlias = "str | int | float | bool | None | Sequence[JsonValue] | Mapping[str, JsonValue]"
|
||||
|
||||
MERGEABLE_FIELDS: Final = frozenset(
|
||||
{
|
||||
Field.dynamic_params,
|
||||
Field.guardrail_info,
|
||||
Field.metadata,
|
||||
Field.optional_params,
|
||||
Field.request_headers,
|
||||
Field.response_headers,
|
||||
Field.provider_request_body,
|
||||
Field.standard_logging_payload,
|
||||
}
|
||||
)
|
||||
OUTCOME_PATCH_FIELDS: Final = frozenset({Field.guardrail_info, Field.metadata})
|
||||
|
||||
|
||||
class InvalidPatchError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SetField:
|
||||
field: Field
|
||||
value: object
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MergeField:
|
||||
field: Field
|
||||
keys: Mapping[str, object]
|
||||
|
||||
|
||||
PatchOp: TypeAlias = SetField | MergeField
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Patch:
|
||||
ops: tuple[PatchOp, ...] = ()
|
||||
|
||||
def set(self, field: Field, value: object) -> "Patch":
|
||||
return Patch((*self.ops, SetField(field, value)))
|
||||
|
||||
def merge(self, field: Field, keys: Mapping[str, object]) -> "Patch":
|
||||
if field not in MERGEABLE_FIELDS:
|
||||
raise InvalidPatchError(f"{field.value} is not a mapping field, so it cannot be merged by key")
|
||||
return Patch((*self.ops, MergeField(field, MappingProxyType(dict(keys)))))
|
||||
|
||||
@property
|
||||
def fields(self) -> frozenset[Field]:
|
||||
return frozenset(op.field for op in self.ops)
|
||||
|
||||
|
||||
def _require_outcome_patch(patch: Patch | None) -> None:
|
||||
if patch is None:
|
||||
return
|
||||
outside: Final = patch.fields - OUTCOME_PATCH_FIELDS
|
||||
if outside:
|
||||
names: Final = ", ".join(sorted(field.value for field in outside))
|
||||
raise InvalidPatchError(f"an outcome may only patch guardrail_info and metadata, not {names}")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Abort:
|
||||
terminal: ClassVar[bool] = True
|
||||
|
||||
status: int
|
||||
message: str
|
||||
code: str | None = None
|
||||
headers: Mapping[str, str] | None = None
|
||||
body: JsonValue = None
|
||||
patch: Patch | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_require_outcome_patch(self.patch)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Respond:
|
||||
terminal: ClassVar[bool] = True
|
||||
|
||||
response: "JsonValue | BaseModel"
|
||||
status: int = 200
|
||||
patch: Patch | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_require_outcome_patch(self.patch)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Reroute:
|
||||
terminal: ClassVar[bool] = False
|
||||
|
||||
model: str
|
||||
|
||||
|
||||
Outcome: TypeAlias = Abort | Respond | Reroute
|
||||
70
litellm/integrations/custom_logger_v2/patches.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
from abc import abstractmethod
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from functools import reduce
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Field
|
||||
from litellm.integrations.custom_logger_v2.outcomes import InvalidPatchError, MergeField, Patch, PatchOp, SetField
|
||||
|
||||
_STRING_KEYED: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class LiveFields(Mapping[Field, object]):
|
||||
__slots__ = ()
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, patch: Patch) -> Mapping[Field, object]: ...
|
||||
|
||||
|
||||
class PatchedFields(Mapping[Field, object]):
|
||||
__slots__ = ("_base", "_changed")
|
||||
|
||||
def __init__(self, base: Mapping[Field, object], changed: Mapping[Field, object]) -> None:
|
||||
self._base = base
|
||||
self._changed = changed
|
||||
|
||||
def with_value(self, field: Field, value: object) -> "PatchedFields":
|
||||
return PatchedFields(self._base, MappingProxyType({**self._changed, field: value}))
|
||||
|
||||
def __getitem__(self, field: Field) -> object:
|
||||
return self._changed[field] if field in self._changed else self._base[field]
|
||||
|
||||
def __iter__(self) -> Iterator[Field]:
|
||||
return iter(self._changed.keys() | self._base.keys())
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._changed.keys() | self._base.keys())
|
||||
|
||||
|
||||
def _with_value(fields: Mapping[Field, object], field: Field, value: object) -> PatchedFields:
|
||||
patched: Final = fields if isinstance(fields, PatchedFields) else PatchedFields(fields, MappingProxyType({}))
|
||||
return patched.with_value(field, value)
|
||||
|
||||
|
||||
def _merged(current: object, field: Field, keys: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if current is None:
|
||||
return MappingProxyType(dict(keys))
|
||||
if not isinstance(current, Mapping):
|
||||
raise InvalidPatchError(f"{field.value} holds no mapping, so keys cannot be merged into it")
|
||||
return MappingProxyType({**_STRING_KEYED.validate_python(current), **keys})
|
||||
|
||||
|
||||
def _apply_op(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]:
|
||||
match op:
|
||||
case SetField(field=field, value=value):
|
||||
return _with_value(fields, field, value)
|
||||
case MergeField(field=field, keys=keys):
|
||||
return _with_value(fields, field, _merged(fields.get(field), field, keys))
|
||||
|
||||
|
||||
def apply_patch(fields: Mapping[Field, object], patch: Patch) -> Mapping[Field, object]:
|
||||
if isinstance(fields, LiveFields):
|
||||
return fields.apply(patch)
|
||||
return reduce(_apply_op, patch.ops, fields)
|
||||
|
||||
|
||||
def apply_patches(fields: Mapping[Field, object], patches: Iterable[Patch]) -> Mapping[Field, object]:
|
||||
return reduce(apply_patch, patches, fields)
|
||||
77
litellm/integrations/custom_logger_v2/planner.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, Field, field_mask
|
||||
from litellm.integrations.custom_logger_v2.hooks import Subscription
|
||||
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, eq=False)
|
||||
class Entry:
|
||||
position: int
|
||||
manifest: CallbackManifest
|
||||
subscription: Subscription
|
||||
|
||||
@property
|
||||
def activation(self) -> Activation:
|
||||
return self.subscription.activation or self.manifest.selection.activation
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HookPlan:
|
||||
stages: tuple[tuple[Entry, ...], ...] = ()
|
||||
inline: tuple[Entry, ...] = ()
|
||||
awaited: tuple[Entry, ...] = ()
|
||||
off_path: tuple[Entry, ...] = ()
|
||||
reads: frozenset[Field] = frozenset()
|
||||
|
||||
|
||||
EMPTY_PLAN: Final = HookPlan()
|
||||
|
||||
|
||||
def _blocks(subscription: Subscription) -> bool:
|
||||
return bool(subscription.writes) or subscription.may_abort
|
||||
|
||||
|
||||
def _level(placed: tuple[tuple[Subscription, int], ...], subscription: Subscription) -> int:
|
||||
if subscription.may_abort:
|
||||
return max((level for _, level in placed), default=-1) + 1
|
||||
touched: Final = field_mask(subscription.reads | subscription.writes)
|
||||
writes: Final = field_mask(subscription.writes)
|
||||
after_writers: Final = tuple(level + 1 for prior, level in placed if field_mask(prior.writes) & touched)
|
||||
not_before_readers: Final = tuple(level for prior, level in placed if field_mask(prior.reads) & writes)
|
||||
return max((*after_writers, *not_before_readers), default=0)
|
||||
|
||||
|
||||
def _place(
|
||||
placed: tuple[tuple[Subscription, int], ...], subscription: Subscription
|
||||
) -> tuple[tuple[Subscription, int], ...]:
|
||||
return (*placed, (subscription, _level(placed, subscription)))
|
||||
|
||||
|
||||
def _levels(entries: Sequence[Entry]) -> tuple[tuple[Entry, ...], ...]:
|
||||
placed: Final = reduce(_place, (entry.subscription for entry in entries), ())
|
||||
levels: Final = tuple(level for _, level in placed)
|
||||
return tuple(_at_level(entries, levels, level) for level in range(max(levels, default=-1) + 1))
|
||||
|
||||
|
||||
def _at_level(entries: Sequence[Entry], levels: tuple[int, ...], level: int) -> tuple[Entry, ...]:
|
||||
return tuple(entry for entry, placed_at in zip(entries, levels, strict=True) if placed_at == level)
|
||||
|
||||
|
||||
def plan_hook(entries: Sequence[Entry]) -> HookPlan:
|
||||
if not entries:
|
||||
return EMPTY_PLAN
|
||||
blocking: Final = tuple(entry for entry in entries if _blocks(entry.subscription))
|
||||
serial: Final = tuple(entry for entry in blocking if not entry.subscription.parallel)
|
||||
parallel: Final = tuple(entry for entry in blocking if entry.subscription.parallel)
|
||||
passive: Final = tuple(entry for entry in entries if not _blocks(entry.subscription))
|
||||
return HookPlan(
|
||||
stages=(*_levels(serial), *((parallel,) if parallel else ())),
|
||||
inline=tuple(entry for entry in passive if entry.subscription.legacy),
|
||||
awaited=tuple(entry for entry in passive if entry.subscription.awaited and not entry.subscription.legacy),
|
||||
off_path=tuple(entry for entry in passive if not (entry.subscription.awaited or entry.subscription.legacy)),
|
||||
reads=frozenset[Field]().union(*(entry.subscription.reads for entry in entries)),
|
||||
)
|
||||
51
litellm/integrations/custom_logger_v2/registry.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
from collections.abc import Callable, Mapping, Sequence
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Hook
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
|
||||
from litellm.integrations.custom_logger_v2.manifest import CallbackManifest
|
||||
from litellm.integrations.custom_logger_v2.planner import Entry, HookPlan, plan_hook
|
||||
|
||||
DEFAULT_PLAN_CACHE_SIZE: Final = 256
|
||||
|
||||
|
||||
def _hook_entries(manifests: tuple[CallbackManifest, ...], hook: Hook) -> tuple[Entry, ...]:
|
||||
return tuple(
|
||||
Entry(position, manifest, manifest.subscriptions[hook])
|
||||
for position, manifest in enumerate(manifests)
|
||||
if hook in manifest.subscriptions
|
||||
)
|
||||
|
||||
|
||||
class RegistrySnapshot:
|
||||
__slots__ = ("entries", "manifests", "plan")
|
||||
|
||||
def __init__(self, manifests: tuple[CallbackManifest, ...], plan_cache_size: int) -> None:
|
||||
self.manifests: Final = manifests
|
||||
self.entries: Final[Mapping[Hook, tuple[Entry, ...]]] = MappingProxyType(
|
||||
{hook: _hook_entries(manifests, hook) for hook in Hook}
|
||||
)
|
||||
self.plan: Final[Callable[[Hook, tuple[int, ...]], HookPlan]] = lru_cache(maxsize=plan_cache_size)(
|
||||
self._build_plan
|
||||
)
|
||||
|
||||
def _build_plan(self, hook: Hook, selected: tuple[int, ...]) -> HookPlan:
|
||||
hook_entries: Final = self.entries[hook]
|
||||
return plan_hook(tuple(hook_entries[index] for index in selected))
|
||||
|
||||
|
||||
class CallbackRegistry:
|
||||
def __init__(
|
||||
self, manifests: Sequence[CallbackManifest] = (), plan_cache_size: int = DEFAULT_PLAN_CACHE_SIZE
|
||||
) -> None:
|
||||
self._plan_cache_size: Final = plan_cache_size
|
||||
self.snapshot: RegistrySnapshot = RegistrySnapshot(tuple(manifests), plan_cache_size)
|
||||
|
||||
def register(self, manifest: CallbackManifest) -> None:
|
||||
self.snapshot = RegistrySnapshot((*self.snapshot.manifests, manifest), self._plan_cache_size)
|
||||
|
||||
def unregister(self, logger: CustomLoggerV2) -> None:
|
||||
kept: Final = tuple(manifest for manifest in self.snapshot.manifests if manifest.logger is not logger)
|
||||
self.snapshot = RegistrySnapshot(kept, self._plan_cache_size)
|
||||
36
litellm/integrations/custom_logger_v2/selection.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.custom_logger_v2.enums import Activation, OptOutBy
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Selection:
|
||||
name: str
|
||||
aliases: frozenset[str] = frozenset()
|
||||
activation: Activation = Activation.default_on
|
||||
opt_out_by: frozenset[OptOutBy] = frozenset({OptOutBy.admin})
|
||||
honors_no_log: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RequestSelection:
|
||||
requested: frozenset[str] = frozenset()
|
||||
admin_opt_outs: frozenset[str] = frozenset()
|
||||
caller_opt_outs: frozenset[str] = frozenset()
|
||||
no_log: bool = False
|
||||
|
||||
|
||||
def _opted_out(selection: Selection, names: frozenset[str], request: RequestSelection) -> bool:
|
||||
by_admin: Final = OptOutBy.admin in selection.opt_out_by and not names.isdisjoint(request.admin_opt_outs)
|
||||
by_caller: Final = OptOutBy.caller in selection.opt_out_by and not names.isdisjoint(request.caller_opt_outs)
|
||||
return by_admin or by_caller or (request.no_log and selection.honors_no_log)
|
||||
|
||||
|
||||
def is_selected(selection: Selection, activation: Activation, request: RequestSelection) -> bool:
|
||||
if activation is Activation.required:
|
||||
return True
|
||||
names: Final = selection.aliases | {selection.name}
|
||||
if _opted_out(selection, names, request):
|
||||
return False
|
||||
return activation is Activation.default_on or not names.isdisjoint(request.requested)
|
||||
61
litellm/integrations/customer_budget_webhook.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, HttpUrl, TypeAdapter
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # Legacy params are untyped.
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.types.integrations.customer_budget import CustomerBudgetEvent
|
||||
|
||||
|
||||
class _Settings(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
endpoint: HttpUrl | None = None
|
||||
headers: Mapping[str, str] | None = None
|
||||
|
||||
|
||||
_URL: Final = TypeAdapter(HttpUrl)
|
||||
_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
|
||||
|
||||
class CustomerBudgetWebhook(CustomLogger):
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str | None = None,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
*,
|
||||
async_httpx_client: AsyncHTTPHandler | None = None,
|
||||
) -> None:
|
||||
super().__init__() # pyright: ignore[reportUnknownMemberType] # Base logger kwargs are untyped.
|
||||
self.endpoint: Final = str(_URL.validate_python(endpoint or os.environ.get("CUSTOMER_BUDGET_WEBHOOK_URL")))
|
||||
self.headers: Final = {
|
||||
"Content-Type": "application/json",
|
||||
**_HEADERS.validate_json(os.environ.get("CUSTOMER_BUDGET_WEBHOOK_HEADERS", "{}")),
|
||||
**(headers or {}),
|
||||
}
|
||||
self.async_httpx_client: Final = (
|
||||
async_httpx_client
|
||||
if async_httpx_client is not None
|
||||
else get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def from_config(value: object) -> "CustomerBudgetWebhook":
|
||||
settings: Final = _Settings.model_validate(value or {})
|
||||
return CustomerBudgetWebhook(
|
||||
endpoint=str(settings.endpoint) if settings.endpoint is not None else None,
|
||||
headers=settings.headers,
|
||||
)
|
||||
|
||||
async def async_on_customer_budget_event(self, event: CustomerBudgetEvent) -> None:
|
||||
client: Final = self.async_httpx_client
|
||||
response: Final = await client.post( # pyright: ignore[reportUnknownMemberType] # Legacy HTTP params are untyped.
|
||||
url=self.endpoint, headers=dict(self.headers), json=event.model_dump(mode="json"), timeout=10
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
|
@ -77,6 +77,11 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
|
|||
prompt_data=prompt_data,
|
||||
prompt_file=prompt_file,
|
||||
prompt_id=registration_prompt_id,
|
||||
ignore_prompt_manager_optional_params=(
|
||||
litellm_params.ignore_prompt_manager_optional_params
|
||||
if hasattr(litellm_params, "ignore_prompt_manager_optional_params")
|
||||
else False
|
||||
),
|
||||
)
|
||||
|
||||
return dot_prompt_manager
|
||||
|
|
|
|||
|
|
@ -54,9 +54,11 @@ class DotpromptManager(CustomPromptManagement):
|
|||
prompt_file: str | None = None,
|
||||
prompt_data: dict | str | None = None,
|
||||
prompt_id: str | None = None,
|
||||
ignore_prompt_manager_optional_params: bool | None = False,
|
||||
):
|
||||
import litellm
|
||||
|
||||
self.ignore_prompt_manager_optional_params = ignore_prompt_manager_optional_params
|
||||
self.prompt_directory = prompt_directory or litellm.global_prompt_directory
|
||||
# Support for JSON-based prompts stored in memory/database
|
||||
if isinstance(prompt_data, str):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.constants import FAILED_PROMPT_TOKEN_ESTIMATE_KEY, REDACTED_BY_LITELLM
|
||||
from litellm.integrations._types.open_inference import ErrorAttributes
|
||||
from litellm.integrations.arize import _utils
|
||||
from litellm.integrations.langfuse.langfuse_otel_attributes import (
|
||||
|
|
@ -19,8 +19,10 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
|
|||
serialize_langfuse_datetime,
|
||||
)
|
||||
from litellm.integrations.opentelemetry import (
|
||||
ESTIMATED_INPUT_TOKENS_ATTRIBUTE,
|
||||
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
|
||||
LITELLM_PROXY_REQUEST_SPAN_NAME,
|
||||
USAGE_PROVIDER_STATUS_ATTRIBUTE,
|
||||
OpenTelemetry,
|
||||
OpenTelemetryConfig,
|
||||
)
|
||||
|
|
@ -378,9 +380,15 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
limit: Final = span_attribute_limit(span)
|
||||
carried: Final = carried_keys(span)
|
||||
new_keys: Final = tuple(key for key, _value in fields if key not in carried)
|
||||
room: Final = (
|
||||
len(new_keys) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0)
|
||||
pending_usage: Final[frozenset[str]] = (
|
||||
frozenset((USAGE_PROVIDER_STATUS_ATTRIBUTE, ESTIMATED_INPUT_TOKENS_ATTRIBUTE))
|
||||
if isinstance(kwargs.get(FAILED_PROMPT_TOKEN_ESTIMATE_KEY), int)
|
||||
else frozenset((USAGE_PROVIDER_STATUS_ATTRIBUTE,))
|
||||
if FAILED_PROMPT_TOKEN_ESTIMATE_KEY in kwargs
|
||||
else frozenset()
|
||||
)
|
||||
reserved: Final = _ERROR_ATTRIBUTES | pending_usage
|
||||
room: Final = len(new_keys) if limit is None else max(limit - len(carried) - len(reserved - carried), 0)
|
||||
written: Final = carried | frozenset(new_keys[:room])
|
||||
for key, value in fields:
|
||||
if key in written:
|
||||
|
|
|
|||
|
|
@ -214,9 +214,13 @@ class ModelDeprecationEmailSender:
|
|||
return True
|
||||
|
||||
async def run(self, llm_router: Router, *, now: datetime | None = None) -> int:
|
||||
if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
|
||||
return 0
|
||||
if not await self.config.model_deprecations_enabled():
|
||||
return 0
|
||||
current: Final = now or datetime.now(timezone.utc)
|
||||
settings: Final = await self.config.get_param("general_settings")
|
||||
if settings is None or not _OptIn.model_validate(settings.param_value).enabled:
|
||||
if not _OptIn.model_validate(settings.param_value if settings else {}).enabled:
|
||||
return 0
|
||||
base_url: Final = os.getenv("PROXY_BASE_URL", "")
|
||||
if not base_url:
|
||||
|
|
@ -253,7 +257,7 @@ class ModelDeprecationEmailSender:
|
|||
|
||||
|
||||
async def run_scheduled_model_deprecation_emails() -> None:
|
||||
"""Poll the email opt-in independently; a large owner mailing never delays Slack checks."""
|
||||
"""Poll the email settings independently; a large owner mailing never delays Slack checks."""
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -6,14 +6,14 @@ from concurrent.futures import ThreadPoolExecutor
|
|||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Protocol, cast
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import KUBERNETES_POD_ROUTING_KEY
|
||||
from litellm.constants import FAILED_PROMPT_TOKEN_ESTIMATE_KEY, KUBERNETES_POD_ROUTING_KEY
|
||||
from litellm.integrations._types.open_inference import (
|
||||
OpenInferenceSpanKindValues,
|
||||
)
|
||||
|
|
@ -33,9 +33,11 @@ from litellm.integrations.otel.model.baggage import promoted_metadata
|
|||
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
|
||||
from litellm.integrations.otel.model.metadata import flatten_metadata
|
||||
from litellm.integrations.otel.model.semconv import LiteLLM, Metric
|
||||
from litellm.integrations.otel.model.utils import as_str
|
||||
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
|
||||
from litellm.integrations.otel.routing import routed_model_name, routing_decision_attributes
|
||||
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
|
|
@ -140,6 +142,18 @@ def _response_field(response_obj: object, key: str) -> object:
|
|||
return require_attributes(response_obj, _SupportsGet, "get").get(key)
|
||||
|
||||
|
||||
def _semconv_response_model(
|
||||
response_obj: object, response_model: object, model_group: str | None, deployment_model: str | None
|
||||
) -> object:
|
||||
hidden_params: Final = get_hidden_params(response_obj)
|
||||
provider_model: Final = hidden_params.get("provider_response_model") if hidden_params is not None else None
|
||||
if isinstance(provider_model, str) and provider_model:
|
||||
return provider_model
|
||||
if response_model and response_model == model_group and deployment_model:
|
||||
return deployment_model
|
||||
return response_model
|
||||
|
||||
|
||||
def _billed_usage(response_obj: object, call_type: str | None, litellm_params: Mapping[str, object] | None) -> object:
|
||||
if not response_obj or is_unbilled_non_inference_call_from_params(call_type, litellm_params, response_obj):
|
||||
return None
|
||||
|
|
@ -152,6 +166,7 @@ class _CacheHitView(TypedDict, total=False):
|
|||
|
||||
|
||||
_CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView)
|
||||
_OPAQUE_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
# Dedicated so a slow exporter shutdown cannot starve the shared logging executor.
|
||||
_PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown")
|
||||
|
|
@ -166,6 +181,8 @@ HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE: Final = "http.response.status_code"
|
|||
HTTP_ROUTE_ATTRIBUTE: Final = "http.route"
|
||||
URL_PATH_ATTRIBUTE: Final = "url.path"
|
||||
PREPROCESSING_DURATION_MS_ATTRIBUTE: Final = "litellm.preprocessing.duration_ms"
|
||||
USAGE_PROVIDER_STATUS_ATTRIBUTE: Final = "litellm.usage.provider_status"
|
||||
ESTIMATED_INPUT_TOKENS_ATTRIBUTE: Final = "litellm.usage.estimated_input_tokens"
|
||||
TEAM_METADATA_ATTRIBUTE: Final = "litellm.team.metadata"
|
||||
MODEL_GROUP_ATTRIBUTE: Final = "litellm.model_group"
|
||||
PROVIDER_MODEL_ATTRIBUTE: Final = "litellm.provider.model"
|
||||
|
|
@ -440,6 +457,8 @@ class OpenTelemetryConfig:
|
|||
|
||||
|
||||
class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
||||
wants_failed_prompt_token_estimate: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: OpenTelemetryConfig | None = None,
|
||||
|
|
@ -2225,6 +2244,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# Always create if no parent span exists (backward compatibility)
|
||||
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
|
||||
should_create_primary_span: Final = parent_otel_span is None or get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN")
|
||||
failure_kwargs: Final = _JSON_OBJECT.validate_python(kwargs)
|
||||
|
||||
span = None
|
||||
if should_create_primary_span:
|
||||
|
|
@ -2239,6 +2259,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
span_kwargs["kind"] = self.span_kind.CLIENT
|
||||
span = otel_tracer.start_span(**span_kwargs)
|
||||
self.set_attributes(span, kwargs, response_obj)
|
||||
self._set_failure_usage_attributes(span, failure_kwargs)
|
||||
|
||||
# Record exception information using OTEL standard method
|
||||
span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
|
|
@ -2252,6 +2273,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# Note: parent_otel_span is guaranteed to be not None here
|
||||
if parent_otel_span is not None and parent_otel_span.is_recording():
|
||||
self.set_attributes(parent_otel_span, kwargs, response_obj)
|
||||
self._set_failure_usage_attributes(parent_otel_span, failure_kwargs)
|
||||
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
|
||||
|
||||
|
|
@ -2271,6 +2293,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
):
|
||||
parent_otel_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
def _set_failure_usage_attributes(self, span: Span, kwargs: Mapping[str, object]) -> None:
|
||||
if FAILED_PROMPT_TOKEN_ESTIMATE_KEY not in kwargs:
|
||||
return
|
||||
self.safe_set_attribute(span=span, key=USAGE_PROVIDER_STATUS_ATTRIBUTE, value="unknown")
|
||||
estimate: Final = kwargs[FAILED_PROMPT_TOKEN_ESTIMATE_KEY]
|
||||
if isinstance(estimate, int):
|
||||
self.safe_set_attribute(span=span, key=ESTIMATED_INPUT_TOKENS_ATTRIBUTE, value=estimate)
|
||||
|
||||
def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None:
|
||||
"""
|
||||
Record exception information on the span using OTEL standard methods.
|
||||
|
|
@ -2521,11 +2551,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
#############################################
|
||||
|
||||
# The name of the LLM a request is being made to
|
||||
if kwargs.get("model"):
|
||||
request_model: Final = as_str(_OPAQUE_VALUE.validate_python(kwargs.get("model")))
|
||||
if request_model:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=SpanAttributes.LLM_REQUEST_MODEL.value,
|
||||
value=kwargs.get("model"),
|
||||
value=request_model,
|
||||
)
|
||||
|
||||
# The LLM request type
|
||||
|
|
@ -2617,10 +2648,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
)
|
||||
|
||||
# The model used to generate the response.
|
||||
response_model: Final = routed_model_name(
|
||||
_response_field(response_obj, "model"),
|
||||
standard_logging_payload.get("model_group"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped
|
||||
decision,
|
||||
model_group: Final = as_str(standard_logging_payload.get("model_group")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped
|
||||
routed_response_model: Final = routed_model_name(
|
||||
_response_field(response_obj, "model"), model_group, decision
|
||||
)
|
||||
response_model: Final = (
|
||||
_semconv_response_model(response_obj, routed_response_model, model_group, request_model)
|
||||
if self._gen_ai_semconv_latest_experimental
|
||||
else routed_response_model
|
||||
)
|
||||
if response_model:
|
||||
self.safe_set_attribute(
|
||||
|
|
|
|||
|
|
@ -89,23 +89,13 @@ shows as `redis.incr router_cooldowns` followed by `redis.ttl router_cooldowns`
|
|||
way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue`,
|
||||
`managed_files`, `mcp_servers`, ...), so a job tick renders `redis.set pod_lock`
|
||||
rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every
|
||||
module under `litellm/` and `enterprise/` that calls a shared cache or declares a
|
||||
read or write on the request batch (`reserve_redis_batch_reads`,
|
||||
`batch.mget`, `batch.set`, `batch.script`) and fails when
|
||||
module under `litellm/` and `enterprise/` that calls a shared cache and fails when
|
||||
one has no declared family, with the process-local `InMemoryCache` callers listed
|
||||
as the only exemptions. A batch op carries the family that was active when it was
|
||||
declared, so the routing prefetch armed before deployment selection
|
||||
(`RoutingPrefetch.arm`) is `router_cooldowns` when only cooldown keys go out,
|
||||
`router_usage` when only usage counters do, and `router_cooldowns_usage` when both
|
||||
ride the same MGET, whichever pipeline or standalone read later settles it. A
|
||||
per-request pipeline (`RedisBatch`) that carries ops of
|
||||
one family is `"redis.pipeline auth_objects"`; one that carries several owners'
|
||||
ops is `"redis.pipeline mixed"` with the sorted family list on
|
||||
`litellm.redis.families` and the op count on `litellm.metadata.op_count` (an int,
|
||||
never stringified). A cluster client cannot pipeline across slots, so there every
|
||||
batch op settles on its own and one write-back of three auth objects shows as three
|
||||
parallel `redis.set auth_objects` spans with the same caller, not one pipeline
|
||||
span. Every `call_type` the Redis cache layer emits maps to a verb,
|
||||
ride the same MGET, whichever pipeline or standalone read later settles it. Every `call_type` the Redis cache layer emits maps to a verb,
|
||||
so the `{service} {call_type}` fallback is unreachable for Redis (a test asserts
|
||||
it). Postgres helpers are `postgres.{verb} {table}`
|
||||
(`"postgres.select LiteLLM_VerificationToken"`, `"postgres.update LiteLLM_TeamTable"`,
|
||||
|
|
@ -130,12 +120,9 @@ Redis it is also `db.operation.name`), the target lands on
|
|||
`litellm.service.target`, and the litellm call chain that issued the call
|
||||
(`_retrieve_from_cache <- _async_get_cache`) travels as
|
||||
`ServiceLoggerPayload.caller` onto `litellm.service.caller`, with the forwarding
|
||||
frames (cache facades, circuit-breaker guards, batch retry wrappers, the native
|
||||
execution's `lifecycle`/`streams` drivers) skipped so it names the code that wanted
|
||||
the call. A call whose own frames are all forwarders
|
||||
(a batch op settled in a task of its own, on a cluster client or a NOSCRIPT retry)
|
||||
reports the chain its declaring code captured and threaded through
|
||||
`service_caller(...)`, never the forwarders, and `unknown` when there is none.
|
||||
frames (cache facades, circuit-breaker guards, the native execution's
|
||||
`lifecycle`/`streams` drivers) skipped so it names the code that wanted the call,
|
||||
and `unknown` when its own frames are all forwarders.
|
||||
The cache key itself is never on the span: it is unbounded and carries key hashes
|
||||
and session ids, and the span is already named by key family. Like every other
|
||||
span they parent to the
|
||||
|
|
|
|||
|
|
@ -20,14 +20,27 @@ from litellm.integrations.otel.plumbing.context import request_root_span
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
from litellm.types.utils import CallTypesLiteral, ModelResponseStream
|
||||
|
||||
|
||||
class LangfuseOpenTelemetryV2(OpenTelemetryV2):
|
||||
"""Stamps the caller's trace controls (name, user, session, tags) on the request. Langfuse reads them off
|
||||
the root observation, and the proxy's root span is still recording when the LLM call starts."""
|
||||
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
cache: "DualCache",
|
||||
data: dict,
|
||||
call_type: "CallTypesLiteral",
|
||||
) -> dict:
|
||||
# Through v1.111.0 the proxy's leaf-only override check never ran OpenTelemetryV2's pre-call identity
|
||||
# seeding for the Langfuse subclasses. The MRO-aware check that replaced it would, so this no-op keeps them
|
||||
# as they were.
|
||||
return data
|
||||
|
||||
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
root: Final = request_root_span()
|
||||
if root is not None and root.is_recording():
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ table: one lambda per mapping operation, applied against the typed span data.
|
|||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
from litellm._internal_context import REDIS_FAMILIES_METADATA_KEY
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import (
|
||||
MAX_TOOL_DEFINITION_ATTRS_PER_SPAN,
|
||||
|
|
@ -200,12 +199,5 @@ class GenAIMapper:
|
|||
# semconv naming the server it reached. Internal services (router, budget
|
||||
# jobs, …) have no db.system, so they get only the litellm.service.* keys.
|
||||
attrs.update(db_span_attributes(data.service_name, data.call_type, postgres_operation(data)))
|
||||
attrs.update(
|
||||
{
|
||||
LiteLLM.REDIS_FAMILIES
|
||||
if key == REDIS_FAMILIES_METADATA_KEY
|
||||
else f"{LiteLLM.METADATA_PREFIX}{key}": value
|
||||
for key, value in data.event_metadata.items()
|
||||
}
|
||||
)
|
||||
attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()})
|
||||
return attrs
|
||||
|
|
|
|||
|
|
@ -334,8 +334,6 @@ class LiteLLM:
|
|||
SERVICE_CALL_TYPE: Final = "litellm.service.call_type"
|
||||
SERVICE_CALLER: Final = "litellm.service.caller"
|
||||
SERVICE_TARGET: Final = "litellm.service.target"
|
||||
# The sorted, comma-joined key families one Redis pipeline carried ops for; bounded, unlike the keys.
|
||||
REDIS_FAMILIES: Final = "litellm.redis.families"
|
||||
PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms"
|
||||
# The logical name of the MCP server a tool call was routed to. There is no
|
||||
# semconv key for an MCP server's *name* (the convention uses ``server.address``
|
||||
|
|
|
|||
|
|
@ -263,6 +263,10 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_Alert",
|
||||
"LiteLLM_PerformanceAlertState",
|
||||
"LiteLLM_BudgetTable",
|
||||
"LiteLLM_CustomerOrganizationTable",
|
||||
"LiteLLM_CustomerTeamTable",
|
||||
"LiteLLM_CustomerMemberTable",
|
||||
"LiteLLM_CustomerBudgetIncreaseRequest",
|
||||
"LiteLLM_CredentialsTable",
|
||||
"LiteLLM_ProxyModelTable",
|
||||
"LiteLLM_AgentsTable",
|
||||
|
|
@ -305,9 +309,11 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_DailyUserSpend",
|
||||
"LiteLLM_DailyGlobalSpend",
|
||||
"LiteLLM_DailyOrganizationSpend",
|
||||
"LiteLLM_DailyCustomerMemberSpend",
|
||||
"LiteLLM_DailyEndUserSpend",
|
||||
"LiteLLM_DailyAgentSpend",
|
||||
"LiteLLM_DailyTeamSpend",
|
||||
"LiteLLM_DailyCustomerMemberSpend",
|
||||
"LiteLLM_DailyTagSpend",
|
||||
"LiteLLM_ProxyWorkerHeartbeat",
|
||||
"LiteLLM_CronJob",
|
||||
|
|
|
|||
183
litellm/integrations/performance_alert_notifications.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import html
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import AliasPath, BaseModel, ConfigDict, Field
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.models.alert import AlertSummary
|
||||
from litellm.repositories.alert_repository import AlertRepository
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
|
||||
|
||||
class _EmailSettings(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
enabled: bool = Field(False, strict=True, validation_alias=AliasPath("email_settings", "Performance Alerts"))
|
||||
|
||||
|
||||
class _Delivery(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
|
||||
|
||||
revision: int = 1
|
||||
cursor: int
|
||||
first_window_start: datetime
|
||||
next_send_at: datetime
|
||||
lease_until: datetime
|
||||
|
||||
|
||||
def _description(alert: AlertSummary) -> str:
|
||||
labels: Final = {
|
||||
"spend": "Spend spike",
|
||||
"traffic": "Traffic spike",
|
||||
"latency": "Slower responses",
|
||||
"reliability": "More failed requests",
|
||||
}
|
||||
metric: Final = alert.evidence
|
||||
|
||||
def amount(value: float) -> str:
|
||||
match metric.unit:
|
||||
case "usd":
|
||||
return f"${value:,.2f}"
|
||||
case "ratio":
|
||||
return f"{value:.1%}"
|
||||
case "milliseconds":
|
||||
return f"{value:,.0f} ms"
|
||||
case "requests":
|
||||
return f"{value:,.0f} requests"
|
||||
|
||||
return f"{labels[alert.type]} for {alert.resource_name}: {amount(metric.value)} vs {amount(metric.baseline)}"
|
||||
|
||||
|
||||
def slack_text(alerts: Sequence[AlertSummary], base_url: str) -> str:
|
||||
rows: Final = tuple(
|
||||
f"<{base_url}/ui/alerts?alert={alert.id}|{html.escape(_description(alert), quote=False)}>"
|
||||
for alert in alerts[:5]
|
||||
)
|
||||
return "New performance alerts\n\n" + "\n".join(rows) + f"\n\n<{base_url}/ui/alerts|See all alerts>"
|
||||
|
||||
|
||||
def email_body(alerts: Sequence[AlertSummary], base_url: str) -> tuple[str, str]:
|
||||
rows: Final = tuple(
|
||||
f'<li><a href="{html.escape(base_url, quote=True)}/ui/alerts?alert={alert.id}">{html.escape(_description(alert))}</a></li>'
|
||||
for alert in alerts[:5]
|
||||
)
|
||||
text: Final = (
|
||||
"New performance alerts\n\n"
|
||||
+ "\n".join(f"{_description(alert)}\n{base_url}/ui/alerts?alert={alert.id}" for alert in alerts[:5])
|
||||
+ f"\n\nSee all alerts: {base_url}/ui/alerts"
|
||||
)
|
||||
return (
|
||||
f'<h2>New performance alerts</h2><ul>{"".join(rows)}</ul><p><a href="{html.escape(base_url, quote=True)}/ui/alerts">See all alerts</a></p>',
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
class PerformanceAlertNotifier:
|
||||
def __init__(self, config: ConfigRepository, alerts: AlertRepository) -> None:
|
||||
self.config: Final = config
|
||||
self.alerts: Final = alerts
|
||||
|
||||
async def deliver(
|
||||
self,
|
||||
destination: str,
|
||||
send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]],
|
||||
now: datetime,
|
||||
) -> None:
|
||||
key: Final = "performance_alert_digest:" + hashlib.sha256(destination.encode()).hexdigest()
|
||||
row: Final = await self.config.get_param(key)
|
||||
previous: Final = (
|
||||
_Delivery.model_validate(row.param_value)
|
||||
if row
|
||||
else _Delivery(
|
||||
revision=0,
|
||||
cursor=0,
|
||||
first_window_start=now - timedelta(days=1),
|
||||
next_send_at=now,
|
||||
lease_until=now,
|
||||
)
|
||||
)
|
||||
if previous.next_send_at > now or previous.lease_until > now:
|
||||
return
|
||||
through: Final = await self.alerts.latest_id()
|
||||
found: Final = await self.alerts.summaries(previous.cursor, through)
|
||||
alerts: Final = (
|
||||
found
|
||||
if previous.cursor
|
||||
else tuple(alert for alert in found if alert.created_at >= previous.first_window_start)
|
||||
)
|
||||
claim: Final = previous.model_copy(
|
||||
update={
|
||||
"revision": previous.revision + 1,
|
||||
"lease_until": now + timedelta(minutes=2),
|
||||
"next_send_at": now + timedelta(days=1) if alerts else now,
|
||||
}
|
||||
)
|
||||
if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), previous.revision):
|
||||
return
|
||||
if alerts and not await send(alerts, f"{key}-{through}"):
|
||||
return
|
||||
sent: Final = claim.model_copy(
|
||||
update={
|
||||
"revision": claim.revision + 1,
|
||||
"cursor": through,
|
||||
"next_send_at": now + timedelta(days=1) if alerts else now,
|
||||
"lease_until": now,
|
||||
}
|
||||
)
|
||||
if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision):
|
||||
verbose_proxy_logger.warning("Could not record performance alert summary delivery")
|
||||
|
||||
async def run(self, slack: SlackAlerting | None, *, now: datetime | None = None) -> None:
|
||||
from litellm.proxy.utils import send_email
|
||||
|
||||
base_url: Final = os.getenv("PROXY_BASE_URL", "").rstrip("/")
|
||||
if not base_url:
|
||||
return
|
||||
if slack is not None:
|
||||
for webhook in slack.performance_alert_webhooks():
|
||||
|
||||
async def send_slack(alerts: Sequence[AlertSummary], receipt: str, webhook: str = webhook) -> bool:
|
||||
return await slack.send_performance_alert_summary(webhook, slack_text(alerts, base_url))
|
||||
|
||||
await self._try_deliver("slack:" + webhook, send_slack, now or datetime.now(timezone.utc))
|
||||
if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"):
|
||||
return
|
||||
settings: Final = await self.config.get_param("general_settings")
|
||||
if not _EmailSettings.model_validate(settings.param_value if settings else {}).enabled:
|
||||
return
|
||||
after: str = "" # rebind-ok: advances the bounded recipient page
|
||||
while recipients := await self.alerts.recipients(after):
|
||||
for recipient in recipients:
|
||||
|
||||
async def send_mail(
|
||||
alerts: Sequence[AlertSummary], receipt: str, address: str = recipient.email
|
||||
) -> bool:
|
||||
body, text = email_body(alerts, base_url)
|
||||
return await send_email(
|
||||
receiver_email=address,
|
||||
subject="New performance alerts",
|
||||
html=body,
|
||||
text=text,
|
||||
message_id=f"<{receipt}@alerts.litellm.ai>",
|
||||
)
|
||||
|
||||
await self._try_deliver("email:" + recipient.email, send_mail, now or datetime.now(timezone.utc))
|
||||
after = recipients[-1].user_id
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
async def _try_deliver(
|
||||
self, destination: str, send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], now: datetime
|
||||
) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(self.deliver(destination, send, now), timeout=60)
|
||||
except Exception as exc: # noqa: BLE001 # one unavailable destination must not stop the other recipients
|
||||
verbose_proxy_logger.warning("Performance alert summary delivery failed (%s)", type(exc).__name__)
|
||||
|
|
@ -1061,18 +1061,39 @@ class PrometheusLogger(CustomLogger):
|
|||
labelnames=self.get_labels_for_metric("litellm_deployment_latency_per_output_token"),
|
||||
)
|
||||
|
||||
self.litellm_deployment_successful_fallbacks = self._counter_factory(
|
||||
self.litellm_deployment_successful_fallbacks: _MetricLike = self._counter_factory(
|
||||
"litellm_deployment_successful_fallbacks",
|
||||
"LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model",
|
||||
labelnames=self.get_labels_for_metric("litellm_deployment_successful_fallbacks"),
|
||||
)
|
||||
|
||||
self.litellm_deployment_failed_fallbacks = self._counter_factory(
|
||||
self.litellm_deployment_failed_fallbacks: _MetricLike = self._counter_factory(
|
||||
"litellm_deployment_failed_fallbacks",
|
||||
"LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model",
|
||||
labelnames=self.get_labels_for_metric("litellm_deployment_failed_fallbacks"),
|
||||
)
|
||||
|
||||
self.litellm_dynamic_limiter_share_tokens: _MetricLike = self._gauge_factory(
|
||||
"litellm_dynamic_limiter_share_tokens",
|
||||
"Dynamic rate limiter: the team's share of the model's TPM, as of the team's last settled request",
|
||||
labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_share_tokens"),
|
||||
multiprocess_mode="livemostrecent",
|
||||
)
|
||||
|
||||
self.litellm_dynamic_limiter_borrowed_tokens: _MetricLike = self._gauge_factory(
|
||||
"litellm_dynamic_limiter_borrowed_tokens",
|
||||
"Dynamic rate limiter: tokens the team has used above its share this window, 0 while under its share",
|
||||
labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_borrowed_tokens"),
|
||||
multiprocess_mode="livemostrecent",
|
||||
)
|
||||
|
||||
self.litellm_dynamic_limiter_throttled_total: _MetricLike = self._counter_factory(
|
||||
"litellm_dynamic_limiter_throttled_total",
|
||||
"Dynamic rate limiter: requests refused per team, model and reason, "
|
||||
"the reason being the value of the response's rate_limited_by header",
|
||||
labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_throttled_total"),
|
||||
)
|
||||
|
||||
# Callback Logging Failure Metrics
|
||||
self.litellm_callback_logging_failures_metric = self._counter_factory(
|
||||
name="litellm_callback_logging_failures_metric",
|
||||
|
|
@ -4172,6 +4193,66 @@ class PrometheusLogger(CustomLogger):
|
|||
sanitize_prometheus_label_value(exception_status),
|
||||
).inc()
|
||||
|
||||
def set_dynamic_limiter_share(
|
||||
self,
|
||||
model_group: str,
|
||||
team: str,
|
||||
team_alias: str | None,
|
||||
priority: str,
|
||||
share_tokens: int,
|
||||
borrowed_tokens: int,
|
||||
) -> None:
|
||||
enum_values: Final = UserAPIKeyLabelValues(
|
||||
model_group=model_group, team=team, team_alias=team_alias, priority=priority
|
||||
)
|
||||
self._set_labeled_gauge(
|
||||
self.litellm_dynamic_limiter_share_tokens, "litellm_dynamic_limiter_share_tokens", enum_values, share_tokens
|
||||
)
|
||||
self._set_labeled_gauge(
|
||||
self.litellm_dynamic_limiter_borrowed_tokens,
|
||||
"litellm_dynamic_limiter_borrowed_tokens",
|
||||
enum_values,
|
||||
borrowed_tokens,
|
||||
)
|
||||
|
||||
def _set_labeled_gauge(
|
||||
self,
|
||||
gauge: _MetricLike,
|
||||
metric_name: DEFINED_PROMETHEUS_METRICS,
|
||||
enum_values: UserAPIKeyLabelValues,
|
||||
value: float,
|
||||
) -> None:
|
||||
from prometheus_client import Gauge
|
||||
|
||||
labels: Final = prometheus_label_factory(
|
||||
supported_enum_labels=self.get_labels_for_metric(metric_name=metric_name), enum_values=enum_values
|
||||
)
|
||||
series: Final = gauge.labels(**labels)
|
||||
if isinstance(series, Gauge):
|
||||
series.set(value)
|
||||
|
||||
def increment_dynamic_limiter_throttled(
|
||||
self,
|
||||
model_group: str,
|
||||
team: str,
|
||||
team_alias: str | None,
|
||||
priority: str,
|
||||
rate_limit_type: str,
|
||||
reason: str,
|
||||
) -> None:
|
||||
self._inc_labeled_counter(
|
||||
self.litellm_dynamic_limiter_throttled_total,
|
||||
"litellm_dynamic_limiter_throttled_total",
|
||||
UserAPIKeyLabelValues(
|
||||
model_group=model_group,
|
||||
team=team,
|
||||
team_alias=team_alias,
|
||||
priority=priority,
|
||||
rate_limit_type=rate_limit_type,
|
||||
reason=reason,
|
||||
),
|
||||
)
|
||||
|
||||
def increment_callback_logging_failure(
|
||||
self,
|
||||
callback_name: str,
|
||||
|
|
@ -5221,7 +5302,7 @@ def prometheus_label_factory(
|
|||
tag: str | None = None,
|
||||
*,
|
||||
label_context: PrometheusLabelFactoryContext | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str | None]:
|
||||
"""
|
||||
Returns a dictionary of label + values for prometheus.
|
||||
|
||||
|
|
|
|||
|
|
@ -5852,6 +5852,17 @@
|
|||
"legendFormat": "self",
|
||||
"range": true,
|
||||
"refId": "M"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_daily_customer_member_spend_update_queue_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "redis_daily_customer_member_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "N"
|
||||
}
|
||||
],
|
||||
"title": "Service latency p95 (litellm_<service>_latency)",
|
||||
|
|
@ -6052,6 +6063,17 @@
|
|||
"legendFormat": "self",
|
||||
"range": true,
|
||||
"refId": "N"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_redis_daily_customer_member_spend_update_queue_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "redis_daily_customer_member_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "O"
|
||||
}
|
||||
],
|
||||
"title": "Service request rate (litellm_<service>_total_requests)",
|
||||
|
|
@ -6252,6 +6274,17 @@
|
|||
"legendFormat": "self / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "N"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_redis_daily_customer_member_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "redis_daily_customer_member_spend_update_queue / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "O"
|
||||
}
|
||||
],
|
||||
"title": "Service failure rate (litellm_<service>_failed_requests)",
|
||||
|
|
@ -6644,6 +6677,190 @@
|
|||
],
|
||||
"title": "litellm_lens_export_rows_written and litellm_lens_export_rows_dropped rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 456
|
||||
},
|
||||
"id": 117,
|
||||
"panels": [],
|
||||
"title": "Dynamic rate limiter (needs dynamic_rate_limiter_v3 and prometheus callbacks)",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "The team's token share on the model this window, per team and model group",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 457
|
||||
},
|
||||
"id": 118,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "litellm_dynamic_limiter_share_tokens",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "litellm_dynamic_limiter_share_tokens",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Tokens the team has used above its share this window, 0 while under it",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 457
|
||||
},
|
||||
"id": 119,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "litellm_dynamic_limiter_borrowed_tokens",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "litellm_dynamic_limiter_borrowed_tokens",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Requests the dynamic limiter refused per second, by team, model and the 429's rate_limited_by reason",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 465
|
||||
},
|
||||
"id": 120,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (team, team_alias, model_group, reason) (rate(litellm_dynamic_limiter_throttled_total[5m]))",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "litellm_dynamic_limiter_throttled_total",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ def resolve_prompt_manager_ignore_flags(
|
|||
|
||||
|
||||
class PromptManagementBase(ABC):
|
||||
ignore_prompt_manager_optional_params: bool | None = False
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def integration_name(self) -> str:
|
||||
|
|
@ -198,7 +200,8 @@ class PromptManagementBase(ABC):
|
|||
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
|
||||
prompt_spec=prompt_spec,
|
||||
ignore_prompt_manager_model=ignore_prompt_manager_model,
|
||||
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
|
||||
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params
|
||||
or self.ignore_prompt_manager_optional_params,
|
||||
)
|
||||
return self.post_compile_prompt_processing(
|
||||
prompt_template=prompt_template,
|
||||
|
|
@ -245,7 +248,8 @@ class PromptManagementBase(ABC):
|
|||
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
|
||||
prompt_spec=prompt_spec,
|
||||
ignore_prompt_manager_model=ignore_prompt_manager_model,
|
||||
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
|
||||
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params
|
||||
or self.ignore_prompt_manager_optional_params,
|
||||
)
|
||||
return self.post_compile_prompt_processing(
|
||||
prompt_template=prompt_template,
|
||||
|
|
|
|||
|
|
@ -300,6 +300,26 @@ def create(
|
|||
)
|
||||
else:
|
||||
custom_llm_provider = custom_llm_provider or "gemini"
|
||||
if agent and model and model != agent:
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"agent '{agent}' can only run on a deployment of that agent, but the model is '{model}'. "
|
||||
"Configure a deployment whose model is the agent and send its model group."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
if extra_body and (
|
||||
("agent" in extra_body and extra_body["agent"] != agent) or (agent and "model" in extra_body)
|
||||
):
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
"extra_body cannot set a different agent or add a model to an agent interaction. "
|
||||
"Send the agent and model as top-level fields."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
interactions_api_config: Final = get_provider_interactions_api_config(
|
||||
provider=custom_llm_provider,
|
||||
|
|
@ -340,7 +360,7 @@ def create(
|
|||
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model,
|
||||
model=agent or model,
|
||||
optional_params=dict(optional_params),
|
||||
litellm_params={"litellm_call_id": litellm_call_id},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,11 @@
|
|||
## Compat names
|
||||
|
||||
- A change here that removes a name outside code can import keeps the name only when one of these holds
|
||||
- It has no leading underscore and shipped in a stable release
|
||||
- It is private, code outside this repo calls it, and the compat keeps that code doing what it did
|
||||
- A private name whose old behavior cannot be kept stays removed. An `ImportError` names what changed, while a no-op that answers wrongly hides it
|
||||
- A vendored copy of litellm in another repo is not a caller
|
||||
|
||||
## get_model_cost_map.py
|
||||
|
||||
- This module decides when to fetch and install. Fetching, parsing, integrity checks and alias expansion run in Rust
|
||||
|
|
@ -9,6 +17,8 @@
|
|||
- `utils._register_on_current_map` reruns the whole pass on the new table after `NativeTableSwapped`, and its last attempt passes `rows=None`, so no entry is lost
|
||||
- Compat names exist only so code written against main keeps working. New code never calls them, and they never gain behavior of their own
|
||||
- `_finalize_model_cost_map`, use `model_cost_inputs.install`
|
||||
- `git_blob_id`, which Rust computes for `source_revision`
|
||||
- `FALLBACK_GENERALIZATIONS_KEY`, `RESERVED_TOP_LEVEL_KEYS` and `RETRYABLE_FETCH_STATUS_CODES`, public constants. Rust owns the retry rule
|
||||
- `GetModelCostMap.validate_model_cost_map` and `_check_*`, since Rust checks fetched documents
|
||||
- `ModelCostMapReloaded.model_cost_map`, use `GetModelCostMap.loaded_model_cost_map()`
|
||||
- the `sleep`, `rng` and `client` parameters, which are ignored with a `DeprecationWarning`
|
||||
|
|
@ -25,3 +35,4 @@
|
|||
- `utils._invalidate_model_cost_lowercase_map` forwards to `changed_in_place`
|
||||
- `utils.get_model_cost_mutation_generation` forwards to `model_cost_version().writes`
|
||||
- `utils._model_cost_lowercase_map` is always `None`
|
||||
- `utils._runtime_registered_model_cost` is always empty, since kept registrations live in `model_cost_inputs`
|
||||
|
|
|
|||
|
|
@ -141,6 +141,13 @@ def without(
|
|||
return {key: value for key, value in mapping.items() if key not in keys}
|
||||
|
||||
|
||||
def only(
|
||||
mapping: Mapping[_Key, _Value], *keys: object
|
||||
) -> dict[_Key, _Value]: # mutable-ok: a fresh copy the caller owns
|
||||
"""``mapping`` with only ``keys``, as a new dict in the same order; ``mapping`` is unchanged."""
|
||||
return {key: value for key, value in mapping.items() if key in keys}
|
||||
|
||||
|
||||
def decoded_json_object(value: object) -> Mapping[str, object]:
|
||||
mapping: Final = as_str_mapping(value)
|
||||
if mapping is not None:
|
||||
|
|
|
|||
|
|
@ -167,6 +167,15 @@ def _shift_months(moment: datetime, months: int) -> datetime:
|
|||
return moment.replace(year=year, month=month, day=min(moment.day, get_last_day_of_month(year, month)))
|
||||
|
||||
|
||||
def is_annual_budget_duration(duration: str) -> bool:
|
||||
return _parse_duration(normalize_duration(duration)) == (12, "mo")
|
||||
|
||||
|
||||
def get_annual_budget_window_start(reset_at: datetime, timezone_str: str) -> datetime:
|
||||
boundary: Final = _setup_timezone(reset_at, timezone_str)[0]
|
||||
return _shift_months(boundary, -12).astimezone(timezone.utc)
|
||||
|
||||
|
||||
def get_budget_window_start(duration: str, reset_at: datetime) -> datetime:
|
||||
"""Start of the budget period that ends at `reset_at`, under the same rules
|
||||
`get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar
|
||||
|
|
@ -220,8 +229,12 @@ def _parse_duration(duration: str) -> tuple[int | None, str | None]:
|
|||
if not match:
|
||||
return None, None
|
||||
|
||||
value, unit = match.groups()
|
||||
return int(value), unit
|
||||
value: Final = int(match.group(1))
|
||||
spelled_unit: Final = match.group(2)
|
||||
unit: Final = _extract_from_regex(duration)[1] or spelled_unit
|
||||
if unit == "mo" and spelled_unit != "mo" and value not in (1, 12):
|
||||
return value, spelled_unit
|
||||
return value, unit
|
||||
|
||||
|
||||
def _apply_time_of_day(dt: datetime, reset_time_of_day: time) -> datetime:
|
||||
|
|
@ -390,14 +403,22 @@ def _handle_month_reset(
|
|||
Args:
|
||||
current_time: Current datetime
|
||||
base_midnight: Midnight of current day
|
||||
value: Number of months (currently only supports 1 month resets)
|
||||
value: Number of months, 1 for monthly or 12 for annual resets
|
||||
reset_time_of_day: Wall-clock time the reset lands on
|
||||
|
||||
Returns:
|
||||
datetime: First day of the next reset month at `reset_time_of_day`
|
||||
"""
|
||||
if value == 12:
|
||||
first_of_year: Final = base_midnight.replace(month=1, day=1)
|
||||
annual_candidate: Final = _apply_time_of_day(first_of_year, reset_time_of_day)
|
||||
return (
|
||||
annual_candidate.replace(year=annual_candidate.year + 1)
|
||||
if annual_candidate <= current_time
|
||||
else annual_candidate
|
||||
)
|
||||
if value != 1:
|
||||
raise ValueError("Monthly resets currently only support 1 month intervals")
|
||||
raise ValueError("Calendar resets support 1 month or 12 month intervals")
|
||||
|
||||
first_of_this_month: Final = base_midnight.replace(day=1)
|
||||
candidate: Final = _apply_time_of_day(first_of_this_month, reset_time_of_day)
|
||||
|
|
|
|||
|
|
@ -8,9 +8,11 @@ Disable remote fetching entirely:
|
|||
export LITELLM_LOCAL_BLOG_POSTS=True
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Awaitable, Callable
|
||||
from email.utils import parsedate_to_datetime
|
||||
from importlib.resources import files
|
||||
from typing import Final
|
||||
|
|
@ -78,6 +80,7 @@ class GetBlogPosts:
|
|||
|
||||
_cached_posts: tuple[BlogPost, ...] | None = None
|
||||
_last_fetch_time: float = 0.0
|
||||
_refresh_lock: tuple[asyncio.AbstractEventLoop, asyncio.Lock] | None = None
|
||||
|
||||
@staticmethod
|
||||
def load_local_blog_posts() -> tuple[BlogPost, ...]:
|
||||
|
|
@ -98,7 +101,7 @@ class GetBlogPosts:
|
|||
return response.text
|
||||
|
||||
@staticmethod
|
||||
def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]:
|
||||
def parse_rss_to_posts(xml_text: str | bytes, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]:
|
||||
"""
|
||||
Parse RSS XML into blog posts.
|
||||
|
||||
|
|
@ -147,14 +150,52 @@ class GetBlogPosts:
|
|||
url,
|
||||
str(e),
|
||||
)
|
||||
return cls.load_local_blog_posts()
|
||||
return cls._cache_posts(cls.load_local_blog_posts())
|
||||
|
||||
if not cls.validate_blog_posts(posts):
|
||||
return cls._cache_posts(cls.load_local_blog_posts())
|
||||
|
||||
return cls._cache_posts(posts)
|
||||
|
||||
@classmethod
|
||||
def _cache_posts(cls, posts: tuple[BlogPost, ...]) -> tuple[BlogPost, ...]:
|
||||
cls._cached_posts = posts
|
||||
cls._last_fetch_time = time.time()
|
||||
return posts
|
||||
|
||||
@classmethod
|
||||
async def aget_blog_posts(cls, url: str, fetch: Callable[[str], Awaitable[bytes]]) -> tuple[BlogPost, ...]:
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
|
||||
return cls.load_local_blog_posts()
|
||||
|
||||
cls._cached_posts = posts
|
||||
cls._last_fetch_time = now
|
||||
return posts
|
||||
cached: Final = cls._cached_posts
|
||||
if cached is not None and (time.time() - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS:
|
||||
return cached
|
||||
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
state: Final = cls._refresh_lock
|
||||
lock: Final = state[1] if state is not None and state[0] is loop else asyncio.Lock()
|
||||
cls._refresh_lock = (loop, lock)
|
||||
async with lock:
|
||||
refreshed: Final = cls._cached_posts
|
||||
if refreshed is not None and (time.time() - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS:
|
||||
return refreshed
|
||||
return await cls._refresh_blog_posts(url, fetch)
|
||||
|
||||
@classmethod
|
||||
async def _refresh_blog_posts(cls, url: str, fetch: Callable[[str], Awaitable[bytes]]) -> tuple[BlogPost, ...]:
|
||||
try:
|
||||
xml_text: Final = await fetch(url)
|
||||
posts: Final = cls.parse_rss_to_posts(xml_text)
|
||||
except (httpx.HTTPError, ET.ParseError, ValueError) as e:
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Failed to fetch blog posts from %s: %s. Falling back to local backup.", url, str(e)
|
||||
)
|
||||
return cls._cache_posts(cls.load_local_blog_posts())
|
||||
|
||||
if not cls.validate_blog_posts(posts):
|
||||
return cls._cache_posts(cls.load_local_blog_posts())
|
||||
return cls._cache_posts(posts)
|
||||
|
||||
|
||||
def get_blog_posts(url: str) -> tuple[BlogPost, ...]:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
|
|||
```
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -51,6 +52,9 @@ from litellm.rust_bridge._native import (
|
|||
|
||||
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY})
|
||||
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
|
||||
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
|
||||
MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0
|
||||
MODEL_COST_MAP_FETCH_MIN_DELAY_SECONDS: Final = 2.0
|
||||
|
|
@ -456,18 +460,28 @@ def get_model_cost_map(
|
|||
return _rows_with_rules_set(installed)
|
||||
|
||||
|
||||
def registration_rows(body: bytes) -> dict[str, dict[str, object]]: # mutable-ok: register_model merges these rows
|
||||
"""The rows of a fetched ``register_model`` document, without its fallback rules block. Raises ``ValueError`` on
|
||||
a body that is not a JSON object of rows."""
|
||||
document: Final = _CATALOG_ADAPTER.validate_python(json.loads(body))
|
||||
return {key: entry for key, entry in document.items() if key != FALLBACK_GENERALIZATIONS_KEY}
|
||||
|
||||
|
||||
def fetch_registration_document(url: str) -> dict[str, dict[str, object]] | None:
|
||||
"""The document behind ``register_model(url)``, fetched in one attempt, or ``None`` when the fetch failed."""
|
||||
"""The document behind ``register_model(url)``, fetched in one attempt. ``None`` when offline mode forbids the
|
||||
fetch or it failed."""
|
||||
if is_offline_mode():
|
||||
verbose_logger.warning("LiteLLM: register_model did not fetch %s because LITELLM_OFFLINE_MODE is set", url)
|
||||
return None
|
||||
fetched: Final = cost_map_source(url, use_http_settings=False).fetch_blocking(fetch_policy(5, 1))
|
||||
if isinstance(fetched, NativeFetchFailed):
|
||||
verbose_logger.warning("LiteLLM: register_model could not fetch %s: %s", url, fetched.reason)
|
||||
return None
|
||||
try:
|
||||
document: Final = _CATALOG_ADAPTER.validate_json(fetched.body)
|
||||
return registration_rows(fetched.body)
|
||||
except ValueError as e:
|
||||
verbose_logger.warning("LiteLLM: register_model got an invalid cost map from %s: %s", url, e)
|
||||
return None
|
||||
return {key: entry for key, entry in document.items() if key != "fallback_generalizations"}
|
||||
|
||||
|
||||
def _finalize_model_cost_map( # pyright: ignore[reportUnusedFunction] # main's private name, downstream code calls it
|
||||
|
|
@ -478,15 +492,17 @@ def _finalize_model_cost_map( # pyright: ignore[reportUnusedFunction] # main's
|
|||
case NativeRejected(reason=reason):
|
||||
raise ValueError(reason)
|
||||
case installed:
|
||||
model_cost.pop("fallback_generalizations", None)
|
||||
model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None)
|
||||
return _rows_with_rules_set(installed)
|
||||
|
||||
|
||||
_RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
def git_blob_id(body: bytes) -> str:
|
||||
"""Compat for main's callers. ``git hash-object`` of the bytes, which ``source_revision`` reports."""
|
||||
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def _count_model_entries(model_cost: Mapping[str, object]) -> int:
|
||||
return sum(1 for key in model_cost if key not in _RESERVED_TOP_LEVEL_KEYS)
|
||||
return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
|
|
|
|||
|
|
@ -44,16 +44,16 @@ BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
|
|||
|
||||
ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
|
||||
OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
|
||||
ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
|
||||
OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
|
||||
ITPM_COUNTER_KEY: Final = "_litellm_itpm_counter"
|
||||
OTPM_COUNTER_KEY: Final = "_litellm_otpm_counter"
|
||||
IO_TOKEN_ATTEMPT_KEY: Final = "_litellm_io_token_attempt"
|
||||
IO_TOKEN_RESERVATION_ID_KEY: Final = "_litellm_io_token_reservation_id"
|
||||
IO_TOKEN_RESERVATION_KEYS: Final = frozenset(
|
||||
{
|
||||
ITPM_RESERVED_KEY,
|
||||
OTPM_RESERVED_KEY,
|
||||
ITPM_CACHE_KEY,
|
||||
OTPM_CACHE_KEY,
|
||||
ITPM_COUNTER_KEY,
|
||||
OTPM_COUNTER_KEY,
|
||||
IO_TOKEN_ATTEMPT_KEY,
|
||||
IO_TOKEN_RESERVATION_ID_KEY,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -428,6 +428,9 @@ class LoggingCallbackManager:
|
|||
|
||||
_get_all_callbacks = get_all_callbacks
|
||||
|
||||
def get_failure_callbacks(self) -> tuple[CustomLogger | Callable[..., object] | str, ...]:
|
||||
return (*litellm.failure_callback, *litellm._async_failure_callback)
|
||||
|
||||
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None:
|
||||
"""
|
||||
Remove a callback object from every callback list it may have been
|
||||
|
|
@ -489,7 +492,7 @@ class LoggingCallbackManager:
|
|||
"""
|
||||
# Get callback lists
|
||||
success_callbacks: Final = set(litellm.success_callback + litellm._async_success_callback)
|
||||
failure_callbacks: Final = set(litellm.failure_callback + litellm._async_failure_callback)
|
||||
failure_callbacks: Final = set(self.get_failure_callbacks())
|
||||
general_callbacks: Final = set(litellm.callbacks)
|
||||
|
||||
# Get all unique callbacks
|
||||
|
|
|
|||
|
|
@ -318,6 +318,7 @@ class _BodyPrefixReader:
|
|||
headers: Mapping[str, str] | None = None,
|
||||
follow_redirects: bool = False,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> Response:
|
||||
with self.client.stream(
|
||||
"GET",
|
||||
|
|
@ -342,6 +343,7 @@ class _ImageBodyReader:
|
|||
headers: Mapping[str, str] | None = None,
|
||||
follow_redirects: bool = False,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> Response:
|
||||
with self.client.stream(
|
||||
"GET",
|
||||
|
|
|
|||
|
|
@ -14,11 +14,13 @@ from litellm.rust_bridge import diagnostics
|
|||
|
||||
REDACTED: Final = "REDACTED"
|
||||
|
||||
_PEM_PRIVATE_KEY_BLOCK: Final = r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----"
|
||||
|
||||
|
||||
def _build_secret_patterns() -> "re.Pattern[str]":
|
||||
patterns: Final[list[str]] = [
|
||||
# PEM private key / certificate blocks
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
_PEM_PRIVATE_KEY_BLOCK,
|
||||
# GCP OAuth2 access tokens (ya29.*)
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
# Credential %s formatting (space separator, no key= prefix)
|
||||
|
|
@ -100,6 +102,57 @@ def redact_string(value: str) -> str:
|
|||
return diagnostics.run(lambda native: native.redact_text(value), lambda: python_redact_string(value))
|
||||
|
||||
|
||||
_TOKEN_SHAPES: Final = (
|
||||
r"ya29\.[A-Za-z0-9_.~+/-]{20,}",
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}\b",
|
||||
rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}",
|
||||
r"(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{24,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r"dapi[0-9a-f]{32}\b",
|
||||
r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}",
|
||||
r"github_pat_[A-Za-z0-9_]{82}",
|
||||
r"glpat-[A-Za-z0-9\-_]{20,}",
|
||||
r"lin_api_[A-Za-z0-9]{40}\b",
|
||||
r"xox[abposr]-[A-Za-z0-9-]{10,}",
|
||||
r"hf_[A-Za-z0-9]{34,}",
|
||||
r"npm_[A-Za-z0-9]{36}\b",
|
||||
r"pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,}",
|
||||
r"SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}(?![A-Za-z0-9_-])",
|
||||
r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+",
|
||||
)
|
||||
# Tokens start at a word boundary or right after a JSON escape like "\n", since tool output is often serialized JSON
|
||||
_TOKEN_START: Final = r"(?:(?<!\w)|(?<=\\[nrt])|(?<=\\u[0-9A-Fa-f]{4}))"
|
||||
_TOKEN_PEM_PRIVATE_KEY_BLOCK: Final = (
|
||||
r"-----BEGIN(?:[A-Z ]|-(?!----))*?PRIVATE KEY-----"
|
||||
r"(?:[^-]|-(?!----))*?-----END(?:[A-Z ]|-(?!----))*?PRIVATE KEY-----"
|
||||
)
|
||||
_NON_JWT_TOKEN_RE: Final = re.compile(
|
||||
f"{_TOKEN_PEM_PRIVATE_KEY_BLOCK}|{_TOKEN_START}(?:{'|'.join(_TOKEN_SHAPES[:-1])})"
|
||||
)
|
||||
_DOTTED_TOKEN: Final = f"{_TOKEN_START}(?:{_TOKEN_SHAPES[0]}|{_TOKEN_SHAPES[-2]})"
|
||||
# Consume failed JWT headers once, including dotted keys and their boundary so the second pass sees them whole.
|
||||
_UNMATCHED_JWT: Final = (
|
||||
rf"(?P<unmatched_jwt>eyJ(?:(?!-----BEGIN|{_DOTTED_TOKEN})[A-Za-z0-9_-])*"
|
||||
rf"(?:{_DOTTED_TOKEN})?[^A-Za-z0-9_-]?)"
|
||||
)
|
||||
_TOKEN_RE: Final = re.compile(f"{_NON_JWT_TOKEN_RE.pattern}|{_TOKEN_START}(?:{_TOKEN_SHAPES[-1]}|{_UNMATCHED_JWT})")
|
||||
|
||||
|
||||
def redact_tokens(value: str, replacement: str = REDACTED) -> str:
|
||||
"""Replace substrings shaped like a credential (provider key prefixes, JWTs, PEM private keys).
|
||||
|
||||
Unlike redact_string() this never matches on surrounding words such as ``password:`` or ``Basic``,
|
||||
so it is safe to run over free-form prompt and tool content.
|
||||
"""
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
if match.lastgroup == "unmatched_jwt":
|
||||
return _NON_JWT_TOKEN_RE.sub(replacement, match.group())
|
||||
return match.expand(replacement)
|
||||
|
||||
return _TOKEN_RE.sub(replace, value)
|
||||
|
||||
|
||||
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
|
||||
_WINDOWS_DRIVE_PATH: Final = r"[A-Za-z]:\\[^\s'\"\)\]}>,]+"
|
||||
_PRIVATE_OR_LOOPBACK_IPV4: Final = (
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from tiktoken.model import MODEL_TO_ENCODING
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
|
|
@ -31,6 +31,7 @@ from litellm.constants import (
|
|||
DEFAULT_IMAGE_HEIGHT,
|
||||
DEFAULT_IMAGE_TOKEN_COUNT,
|
||||
DEFAULT_IMAGE_WIDTH,
|
||||
DEFAULT_MAX_RECURSE_DEPTH,
|
||||
MAX_IMAGE_URL_DOWNLOAD_SIZE_MB,
|
||||
MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES,
|
||||
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
|
||||
|
|
@ -47,7 +48,7 @@ from litellm.constants import (
|
|||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
|
||||
ChatTemplateToolSection,
|
||||
chat_template_tool_section,
|
||||
|
|
@ -928,7 +929,6 @@ def token_counter(
|
|||
if litellm.disable_token_counter is True:
|
||||
return 0
|
||||
|
||||
verbose_logger.debug("messages in token_counter: %s, text in token_counter: %s", messages, text)
|
||||
if text is not None and messages is not None:
|
||||
raise ValueError("text and messages cannot both be set")
|
||||
if use_default_image_token_count is None:
|
||||
|
|
@ -1758,31 +1758,72 @@ def _count_file_tokens(
|
|||
)
|
||||
|
||||
|
||||
_ANTHROPIC_SERVER_TOOL_FIELDS: Final = MappingProxyType(
|
||||
{
|
||||
"server_tool_use": ("name", "input"),
|
||||
"mcp_tool_use": ("name", "input"),
|
||||
"web_search_tool_result": ("content",),
|
||||
"web_fetch_tool_result": ("content",),
|
||||
"code_execution_tool_result": ("content",),
|
||||
"bash_code_execution_tool_result": ("content",),
|
||||
"text_editor_code_execution_tool_result": ("content",),
|
||||
"mcp_tool_result": ("content",),
|
||||
"web_search_result": ("url", "title", "snippet"),
|
||||
"web_fetch_result": ("url", "content"),
|
||||
"code_execution_result": ("stdout", "stderr", "content"),
|
||||
"bash_code_execution_result": ("stdout", "stderr", "content"),
|
||||
"text_editor_code_execution_view_result": ("content",),
|
||||
"text_editor_code_execution_str_replace_result": ("lines",),
|
||||
"text_editor_code_execution_create_result": (),
|
||||
"encrypted_code_execution_result": ("stderr", "content"),
|
||||
"code_execution_output": (),
|
||||
"bash_code_execution_output": (),
|
||||
"web_search_tool_result_error": ("error_code",),
|
||||
"web_fetch_tool_result_error": ("error_code",),
|
||||
"code_execution_tool_result_error": ("error_code",),
|
||||
"bash_code_execution_tool_result_error": ("error_code",),
|
||||
"text_editor_code_execution_tool_result_error": ("error_code", "error_message"),
|
||||
}
|
||||
)
|
||||
_ANTHROPIC_DOCUMENT: Final = TypeAdapter(AnthropicMessagesDocumentParam)
|
||||
|
||||
|
||||
def _count_anthropic_content(
|
||||
content: Mapping[str, object],
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
*,
|
||||
_content_depth: int = 0,
|
||||
) -> int:
|
||||
"""
|
||||
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
|
||||
|
||||
Uses TypedDict definitions from litellm.types.llms.anthropic to determine
|
||||
what fields to count and how to handle nested structures.
|
||||
|
||||
Dynamically infers which fields to count based on the TypedDict definition,
|
||||
avoiding hardcoded field names.
|
||||
Uses readable server tool fields or the client tool TypedDict definitions.
|
||||
"""
|
||||
typeddict_cls: Final = _validate_anthropic_content(content)
|
||||
type_hints: Final[Mapping[str, object]] = getattr(typeddict_cls, "__annotations__", {})
|
||||
if _content_depth >= DEFAULT_MAX_RECURSE_DEPTH:
|
||||
raise UnsupportedContentTypeError("Anthropic tool content nesting exceeds the token counter limit")
|
||||
content_type: Final = content.get("type")
|
||||
if content_type == "document":
|
||||
return _count_document_tokens(
|
||||
_ANTHROPIC_DOCUMENT.validate_python(content),
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
server_fields: Final = _ANTHROPIC_SERVER_TOOL_FIELDS.get(content_type) if isinstance(content_type, str) else None
|
||||
fields: Final[Mapping[str, object]] = (
|
||||
dict.fromkeys(server_fields)
|
||||
if server_fields is not None
|
||||
else getattr(_validate_anthropic_content(content), "__annotations__", {})
|
||||
)
|
||||
tokens = 0
|
||||
|
||||
# Fields to skip (metadata/identifiers that don't contribute to prompt tokens)
|
||||
skip_fields: Final = {"type", "id", "tool_use_id", "cache_control", "is_error"}
|
||||
|
||||
# Iterate over all fields defined in the TypedDict
|
||||
for field_name, field_type in type_hints.items():
|
||||
for field_name in fields:
|
||||
if field_name in skip_fields:
|
||||
continue
|
||||
|
||||
|
|
@ -1801,7 +1842,21 @@ def _count_anthropic_content(
|
|||
pdf_pricing,
|
||||
)
|
||||
elif isinstance(field_value, dict):
|
||||
tokens += count_function(str(field_value))
|
||||
if (
|
||||
server_fields is not None
|
||||
and field_name == "content"
|
||||
and (field_mapping := as_str_mapping(content[field_name])) is not None
|
||||
):
|
||||
tokens += _count_anthropic_content(
|
||||
field_mapping,
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
_content_depth=_content_depth + 1,
|
||||
)
|
||||
else:
|
||||
tokens += count_function(str(field_value))
|
||||
except Exception as e:
|
||||
if default_token_count is not None:
|
||||
return default_token_count
|
||||
|
|
@ -1816,6 +1871,29 @@ _TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def _is_text_block(block: object) -> bool:
|
||||
mapping: Final = as_str_mapping(block)
|
||||
block_type: Final = mapping.get("type") if mapping is not None else None
|
||||
return isinstance(block, str) or (isinstance(block_type, str) and block_type in _TEXT_FIELD_BY_BLOCK_TYPE)
|
||||
|
||||
|
||||
def _has_only_text_content(message: object) -> bool:
|
||||
normalized: Final[object] = message.model_dump(exclude_none=True) if isinstance(message, BaseModel) else message
|
||||
mapping: Final = as_str_mapping(normalized)
|
||||
if mapping is None:
|
||||
return False
|
||||
content: Final = mapping.get("content")
|
||||
if content is None or isinstance(content, str):
|
||||
return True
|
||||
blocks: Final = as_object_list(content)
|
||||
return blocks is not None and all(_is_text_block(block) for block in blocks)
|
||||
|
||||
|
||||
def has_only_text_content(messages: Sequence[object]) -> bool:
|
||||
"""Whether counting ``messages`` stays on the tokenizer, never decoding images, files or documents."""
|
||||
return all(_has_only_text_content(message) for message in messages)
|
||||
|
||||
|
||||
def _count_content_list(
|
||||
count_function: TokenCounterFunction,
|
||||
content_list: str
|
||||
|
|
@ -1867,7 +1945,9 @@ def _count_content_list(
|
|||
use_default_image_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result", "search_result"):
|
||||
elif (
|
||||
c["type"] in ("tool_use", "tool_result", "search_result") or c["type"] in _ANTHROPIC_SERVER_TOOL_FIELDS
|
||||
):
|
||||
num_tokens += _count_anthropic_content(
|
||||
c,
|
||||
count_function,
|
||||
|
|
@ -1886,13 +1966,16 @@ def _count_content_list(
|
|||
tool_name = str(c.get("tool_name") or "")
|
||||
if tool_name:
|
||||
num_tokens += count_function(tool_name)
|
||||
elif (change := as_str_mapping(c)) is not None and change.get("type") in ("tool_addition", "tool_removal"):
|
||||
num_tokens += _tool_change_tokens(count_function, as_str_mapping(change.get("tool")))
|
||||
else:
|
||||
content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
|
||||
raise UnsupportedContentTypeError(
|
||||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"compaction, tool_reference, search_result)."
|
||||
f"compaction, tool_reference, tool_addition, tool_removal, search_result, "
|
||||
f"or Anthropic server tool blocks)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
@ -1904,6 +1987,24 @@ def _count_content_list(
|
|||
)
|
||||
|
||||
|
||||
def _tool_change_tokens(count_function: TokenCounterFunction, tool: Mapping[str, object] | None) -> int:
|
||||
match tool:
|
||||
case {"type": "tool_definition", "definition": definition} if (
|
||||
function := as_str_mapping(definition)
|
||||
) is not None:
|
||||
return count_function(_format_function_definitions((function,)))
|
||||
case {"type": "tool_reference", "name": str() as name}:
|
||||
return count_function(name)
|
||||
case {"type": "mcp_tool_reference", "server_name": str() as server_name, "name": str() as name}:
|
||||
return count_function(server_name) + count_function(name)
|
||||
case {"type": "mcp_toolset_reference", "server_name": str() as server_name}:
|
||||
return count_function(server_name)
|
||||
case _:
|
||||
raise UnsupportedContentTypeError(
|
||||
f"Invalid tool change target type: {None if tool is None else tool.get('type')}"
|
||||
)
|
||||
|
||||
|
||||
def _format_function_definitions(tools: Sequence[object]) -> str:
|
||||
"""Formats tool definitions in the format that OpenAI appears to use.
|
||||
Based on https://github.com/forestwanglin/openai-java/blob/main/jtokkit/src/main/java/xyz/felh/openai/jtokkit/utils/TikTokenUtils.java
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class _UrlFetcher(Protocol):
|
|||
headers: dict[str, str] | None = None,
|
||||
follow_redirects: bool = False,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response: ...
|
||||
|
||||
|
||||
|
|
@ -80,10 +81,11 @@ class _FetchOptions(TypedDict, total=False):
|
|||
headers: ReadOnly[dict[str, str]]
|
||||
follow_redirects: ReadOnly[bool]
|
||||
timeout: ReadOnly[float | httpx.Timeout | None]
|
||||
max_response_bytes: ReadOnly[int | None]
|
||||
|
||||
|
||||
class _AsyncFetchOptions(_FetchOptions, total=False):
|
||||
max_response_bytes: ReadOnly[int | None]
|
||||
pass
|
||||
|
||||
|
||||
# Cloud-internal IPs that ``ip.is_global`` can report as public. Everything
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
|
|
@ -20,6 +19,8 @@ from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_
|
|||
from litellm.llms.anthropic.messages.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
json_copy_or_original,
|
||||
object_mapping,
|
||||
openai_strict_json_schema,
|
||||
prompt_cache_key_from_user_id,
|
||||
)
|
||||
from litellm.responses.utils import auto_reasoning_summary
|
||||
|
|
@ -1017,66 +1018,23 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
Returns:
|
||||
OpenAI-compatible response_format dict, or None if invalid
|
||||
"""
|
||||
if not isinstance(output_format, dict):
|
||||
output_format_object: Final = object_mapping(output_format)
|
||||
if output_format_object is None or output_format_object.get("type") != "json_schema":
|
||||
return None
|
||||
|
||||
output_type: Final = output_format.get("type")
|
||||
if output_type != "json_schema":
|
||||
return None
|
||||
|
||||
schema = output_format.get("schema")
|
||||
schema: Final = output_format_object.get("schema")
|
||||
if not schema:
|
||||
return None
|
||||
|
||||
# Deep copy to avoid mutating the original schema
|
||||
schema = copy.deepcopy(schema)
|
||||
# OpenAI strict mode requires additionalProperties: false on every object
|
||||
self._add_additional_properties_false(schema)
|
||||
|
||||
# Convert to OpenAI response_format structure
|
||||
return {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "structured_output",
|
||||
"schema": schema,
|
||||
"schema": openai_strict_json_schema(schema),
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _add_additional_properties_false(schema: dict) -> None:
|
||||
"""
|
||||
Recursively ensure object schemas comply with OpenAI strict mode.
|
||||
|
||||
OpenAI's strict mode requires:
|
||||
1. 'additionalProperties': false at every object nesting level
|
||||
2. All property keys listed in 'required'
|
||||
"""
|
||||
if not isinstance(schema, dict):
|
||||
return
|
||||
|
||||
if schema.get("type") == "object" and "properties" in schema:
|
||||
schema["additionalProperties"] = False
|
||||
schema["required"] = list(schema["properties"].keys())
|
||||
for prop in schema["properties"].values():
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop)
|
||||
|
||||
# Handle array items
|
||||
if "items" in schema:
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"])
|
||||
|
||||
# Handle anyOf/oneOf/allOf
|
||||
for key in ("anyOf", "oneOf", "allOf"):
|
||||
if key in schema:
|
||||
for sub_schema in schema[key]:
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema)
|
||||
|
||||
# Handle $defs / definitions
|
||||
for key in ("$defs", "definitions"):
|
||||
if key in schema:
|
||||
for def_schema in schema[key].values():
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema)
|
||||
|
||||
def _translate_midturn_system_message_to_openai(
|
||||
self,
|
||||
message: AnthropicMessagesSystemMessageParam,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,19 @@ from litellm._logging import verbose_logger
|
|||
from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
|
||||
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2.enums import Hook
|
||||
from litellm.integrations.custom_logger_v2.legacy import (
|
||||
PROXY_REQUEST_KEYS,
|
||||
Admit,
|
||||
KwargsFields,
|
||||
hook_subscribers,
|
||||
kwargs_after,
|
||||
kwargs_patch,
|
||||
run_subscribers,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Patch
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -374,7 +387,7 @@ async def _execute_pre_request_hooks(
|
|||
pass
|
||||
|
||||
# Build complete request kwargs dict
|
||||
request_kwargs = {
|
||||
request_kwargs: dict[str, object] = { # mutable-ok: the hooks take and may replace the request dict
|
||||
"tools": tools,
|
||||
"stream": stream,
|
||||
"litellm_params": {
|
||||
|
|
@ -386,10 +399,21 @@ async def _execute_pre_request_hooks(
|
|||
if not litellm.callbacks:
|
||||
return request_kwargs
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger as _CustomLogger
|
||||
callbacks: Final = litellm.callbacks
|
||||
if (
|
||||
Hook.pre_deployment_call in CustomLoggerV2.declared_hooks
|
||||
and (subscribers := hook_subscribers(Hook.pre_deployment_call, callbacks, admit=Admit.every_logger)).has_v2
|
||||
):
|
||||
return kwargs_after( # pyright: ignore[reportReturnType] # whatever the last hook returned, as before
|
||||
await run_subscribers(
|
||||
subscribers,
|
||||
KwargsFields(request_kwargs, PROXY_REQUEST_KEYS),
|
||||
partial(_start_pre_request_hook, model, messages),
|
||||
)
|
||||
)
|
||||
|
||||
for callback in litellm.callbacks:
|
||||
if not isinstance(callback, _CustomLogger):
|
||||
for callback in callbacks:
|
||||
if not isinstance(callback, CustomLogger):
|
||||
continue
|
||||
|
||||
# Call the pre-request hook
|
||||
|
|
@ -402,6 +426,27 @@ async def _execute_pre_request_hooks(
|
|||
return request_kwargs
|
||||
|
||||
|
||||
def _start_pre_request_hook(
|
||||
model: str,
|
||||
messages: list[dict[str, object]], # mutable-ok: the hook takes the messages list
|
||||
callback: object,
|
||||
request_kwargs: object,
|
||||
) -> Awaitable[Patch | None] | None:
|
||||
if not isinstance(callback, CustomLogger):
|
||||
return None
|
||||
return _pre_request_hook(callback, model, messages, request_kwargs)
|
||||
|
||||
|
||||
async def _pre_request_hook(
|
||||
callback: CustomLogger,
|
||||
model: str,
|
||||
messages: list[dict[str, object]], # mutable-ok: see above
|
||||
request_kwargs: object,
|
||||
) -> Patch | None:
|
||||
modified_kwargs: Final = await callback.async_pre_request_hook(model, messages, request_kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownVariableType] # the request dict as the previous hook left it
|
||||
return None if modified_kwargs is None else kwargs_patch(request_kwargs, modified_kwargs) # pyright: ignore[reportUnknownArgumentType] # see above
|
||||
|
||||
|
||||
async def _try_websearch_short_circuit(
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ from litellm.llms.anthropic.messages.native.utils import (
|
|||
from litellm.llms.anthropic.messages.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
json_copy_or_original,
|
||||
object_mapping,
|
||||
openai_strict_json_schema,
|
||||
prompt_cache_key_from_user_id,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -588,10 +590,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_output_format_strict(strict: bool | None) -> bool:
|
||||
if strict is None:
|
||||
return True
|
||||
return strict
|
||||
def _resolve_output_format_strict(strict: object) -> bool:
|
||||
return True if strict is None else bool(strict)
|
||||
|
||||
def translate_request(
|
||||
self,
|
||||
|
|
@ -690,19 +690,23 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
# output_format / output_config.format -> text format
|
||||
# output_format: {"type": "json_schema", "schema": {...}}
|
||||
# output_config: {"format": {"type": "json_schema", "schema": {...}}}
|
||||
output_format: object = anthropic_request.get("output_format")
|
||||
output_config = anthropic_request.get("output_config")
|
||||
if not isinstance(output_format, dict) and isinstance(output_config, dict):
|
||||
output_format = output_config.get("format")
|
||||
if isinstance(output_format, dict) and output_format.get("type") == "json_schema":
|
||||
output_config_object: Final = object_mapping(anthropic_request.get("output_config"))
|
||||
top_level_output_format: Final = object_mapping(anthropic_request.get("output_format"))
|
||||
output_format: Final = (
|
||||
top_level_output_format
|
||||
if top_level_output_format is not None or output_config_object is None
|
||||
else object_mapping(output_config_object.get("format"))
|
||||
)
|
||||
if output_format is not None and output_format.get("type") == "json_schema":
|
||||
schema: Final = output_format.get("schema")
|
||||
if schema:
|
||||
output_strict: Final = self._resolve_output_format_strict(output_format.get("strict"))
|
||||
responses_kwargs["text"] = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "structured_output",
|
||||
"schema": schema,
|
||||
"strict": self._resolve_output_format_strict(output_format.get("strict")),
|
||||
"schema": openai_strict_json_schema(schema) if output_strict else schema,
|
||||
"strict": output_strict,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
|
|
@ -37,6 +38,61 @@ def json_copy_or_original(value: _JsonValue) -> _JsonValue:
|
|||
return value
|
||||
|
||||
|
||||
_SUBSCHEMA_LIST_KEYWORDS: Final = ("anyOf", "oneOf", "allOf")
|
||||
_SUBSCHEMA_MAP_KEYWORDS: Final = ("$defs", "definitions")
|
||||
|
||||
|
||||
_JSON_ARRAY: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _json_array(value: object) -> Sequence[object] | None:
|
||||
try:
|
||||
return _JSON_ARRAY.validate_python(value, strict=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def openai_strict_json_schema(schema: object) -> object:
|
||||
"""New copy of a JSON schema where every object with properties forbids extra keys and requires all of them.
|
||||
|
||||
OpenAI strict structured outputs rejects any schema that leaves an object open or a property optional.
|
||||
"""
|
||||
source: Final = object_mapping(schema)
|
||||
if source is None:
|
||||
return copy.deepcopy(schema)
|
||||
properties: Final = object_mapping(source.get("properties"))
|
||||
closing: Final[Mapping[str, object]] = (
|
||||
{"required": list(properties), "additionalProperties": False}
|
||||
if properties is not None and _is_object_type(source.get("type"))
|
||||
else {}
|
||||
)
|
||||
return {**{key: _strict_subschema(key, value) for key, value in source.items()}, **closing}
|
||||
|
||||
|
||||
def _is_object_type(schema_type: object) -> bool:
|
||||
"""Whether a ``type`` keyword names an object, alone or in a union such as ``["object", "null"]``."""
|
||||
types: Final = _json_array(schema_type)
|
||||
return schema_type == "object" if types is None else "object" in types
|
||||
|
||||
|
||||
def _strict_subschema(key: str, value: object) -> object:
|
||||
if key == "items":
|
||||
return openai_strict_json_schema(value)
|
||||
if key == "properties" or key in _SUBSCHEMA_MAP_KEYWORDS:
|
||||
named: Final = object_mapping(value)
|
||||
return (
|
||||
copy.deepcopy(value)
|
||||
if named is None
|
||||
else {name: openai_strict_json_schema(subschema) for name, subschema in named.items()}
|
||||
)
|
||||
if key in _SUBSCHEMA_LIST_KEYWORDS:
|
||||
variants: Final = _json_array(value)
|
||||
return (
|
||||
copy.deepcopy(value) if variants is None else [openai_strict_json_schema(variant) for variant in variants]
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
class _ClaudeCodeUserId(LiteLLMBaseModel):
|
||||
"""The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation."""
|
||||
|
||||
|
|
@ -136,7 +192,7 @@ def proxy_sub_call_logging_kwargs(parent_kwargs: Mapping[str, object]) -> ProxyS
|
|||
if disable_header is None
|
||||
else {**parent_params, "proxy_server_request": {"headers": {X_LITELLM_DISABLE_CALLBACKS: disable_header}}}
|
||||
)
|
||||
trusted_vars: Final = _object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD))
|
||||
trusted_vars: Final = object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD))
|
||||
if trusted_vars is None:
|
||||
return dynamic_params
|
||||
return {**dynamic_params, "litellm_trusted_callback_vars": trusted_vars}
|
||||
|
|
@ -154,13 +210,13 @@ def proxy_sub_call_trace_id(parent_kwargs: Mapping[str, object]) -> str | None:
|
|||
|
||||
|
||||
def _disable_callbacks_header(parent_kwargs: Mapping[str, object]) -> str | None:
|
||||
parent_request: Final = _object_mapping(parent_kwargs.get("proxy_server_request"))
|
||||
headers: Final = _object_mapping(parent_request.get("headers")) if parent_request is not None else None
|
||||
parent_request: Final = object_mapping(parent_kwargs.get("proxy_server_request"))
|
||||
headers: Final = object_mapping(parent_request.get("headers")) if parent_request is not None else None
|
||||
header: Final = headers.get(X_LITELLM_DISABLE_CALLBACKS) if headers is not None else None
|
||||
return header if isinstance(header, str) else None
|
||||
|
||||
|
||||
def _object_mapping(value: object) -> Mapping[str, object] | None:
|
||||
def object_mapping(value: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _OBJECT_MAPPING.validate_python(value)
|
||||
except ValidationError:
|
||||
|
|
|
|||
|
|
@ -145,7 +145,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params)
|
||||
|
||||
def get_stripped_model_name(self, model: str) -> str:
|
||||
return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
|
||||
return (
|
||||
model.replace("responses/", "")
|
||||
.replace("o_series/", "")
|
||||
.replace("azure_ai/", "")
|
||||
.replace(AzureOpenAIGPT5Config.GPT5_SERIES_ROUTE, "")
|
||||
)
|
||||
|
||||
def _validate_input_param(
|
||||
self, input: str | ResponseInputParam, *, drop_params: bool = False
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
||||
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
|
||||
|
|
@ -16,6 +18,24 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
_BODY_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def extra_body_fields(optional_params: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
"""The caller's ``extra_body`` fields, merged under the provider body after the transform so they add fields but never
|
||||
replace the input or a mapped param."""
|
||||
extra_body: Final = (optional_params or {}).get("extra_body")
|
||||
return _BODY_FIELDS.validate_python(extra_body) if isinstance(extra_body, Mapping) else {}
|
||||
|
||||
|
||||
def split_extra_body(
|
||||
optional_params: Mapping[str, object] | None,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""The params without ``extra_body``, and the ``extra_body`` fields to merge into the provider body."""
|
||||
params: Final = _BODY_FIELDS.validate_python(optional_params or {})
|
||||
return {k: v for k, v in params.items() if k != "extra_body"}, extra_body_fields(params)
|
||||
|
||||
|
||||
class BaseEmbeddingConfig(BaseConfig, ABC):
|
||||
@abstractmethod
|
||||
def transform_embedding_request(
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ class BaseImageGenerationConfig(ABC):
|
|||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
) -> dict[str, object]: # mutable-ok: inherited contract
|
||||
raise NotImplementedError(
|
||||
"ImageVariationConfig implementa 'transform_request_image_variation' for image variation models"
|
||||
)
|
||||
|
|
@ -100,6 +100,32 @@ class BaseImageGenerationConfig(ABC):
|
|||
"ImageVariationConfig implements 'transform_response_image_variation' for image variation models"
|
||||
)
|
||||
|
||||
async def async_transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict[str, object], # mutable-ok: inherited contract
|
||||
optional_params: dict[str, object], # mutable-ok: inherited contract
|
||||
litellm_params: dict[str, object], # mutable-ok: inherited contract
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
return self.transform_image_generation_response( # pyright: ignore[reportUnknownMemberType] # legacy sync hook has untyped dict params
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=logging_obj,
|
||||
request_data=request_data,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
def use_multipart_form_data(self) -> bool:
|
||||
"""
|
||||
Returns True if this provider requires multipart/form-data instead of JSON.
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import copy
|
|||
import json
|
||||
import urllib.parse
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, ClassVar, Final, get_args, overload
|
||||
|
||||
import httpx
|
||||
|
|
@ -15,6 +16,7 @@ import litellm
|
|||
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_param
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.embedding.transformation import extra_body_fields
|
||||
from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -103,7 +105,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
region_from_model: str | None = None,
|
||||
) -> tuple[Credentials | None, str]:
|
||||
auth_params: Final = pop_aws_auth_params(optional_params)
|
||||
explicit_region_name: Final = optional_params.pop("aws_region_name", None)
|
||||
explicit_region_name: Final = _OPTIONAL_STR.validate_python(optional_params.pop("aws_region_name", None))
|
||||
aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name
|
||||
|
||||
### SET REGION NAME ###
|
||||
|
|
@ -291,9 +293,10 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
||||
api_key: str | None = None,
|
||||
is_async_invoke: bool | None = False,
|
||||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
responses: Final[list[dict]] = []
|
||||
for data in batch_data:
|
||||
for data in ({**extra_body, **item} for item in batch_data):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
|
@ -360,9 +363,10 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
||||
api_key: str | None = None,
|
||||
is_async_invoke: bool | None = False,
|
||||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
responses: Final[list[dict]] = []
|
||||
for data in batch_data:
|
||||
for data in ({**extra_body, **item} for item in batch_data):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
|
@ -462,9 +466,12 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
)
|
||||
inference_params = copy.deepcopy(optional_params)
|
||||
inference_params = {
|
||||
k: v for k, v in inference_params.items() if k.lower() not in self.aws_authentication_params
|
||||
k: v
|
||||
for k, v in inference_params.items()
|
||||
if k.lower() not in self.aws_authentication_params and k != "extra_body"
|
||||
}
|
||||
inference_params.pop("user", None) # make sure user is not passed in for bedrock call
|
||||
extra_body: Final = extra_body_fields(optional_params)
|
||||
|
||||
data: CohereEmbeddingRequest | None = None
|
||||
batch_data: list | None = None
|
||||
|
|
@ -535,6 +542,8 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
)
|
||||
batch_data.append(nova_request)
|
||||
|
||||
request_data: Final = None if data is None else {**extra_body, **data}
|
||||
|
||||
### SET RUNTIME ENDPOINT ###
|
||||
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
|
|
@ -552,6 +561,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None),
|
||||
timeout=timeout,
|
||||
batch_data=batch_data,
|
||||
extra_body=extra_body,
|
||||
credentials=credentials,
|
||||
extra_headers=extra_headers,
|
||||
endpoint_url=endpoint_url,
|
||||
|
|
@ -566,6 +576,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
client=(client if client is not None and isinstance(client, HTTPHandler) else None),
|
||||
timeout=timeout,
|
||||
batch_data=batch_data,
|
||||
extra_body=extra_body,
|
||||
credentials=credentials,
|
||||
extra_headers=extra_headers,
|
||||
endpoint_url=endpoint_url,
|
||||
|
|
@ -579,7 +590,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
if returned_response is None:
|
||||
raise Exception("Unable to map Bedrock request to provider")
|
||||
return returned_response
|
||||
elif data is None:
|
||||
elif request_data is None:
|
||||
raise Exception("Unable to map Bedrock request to provider")
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
|
@ -591,7 +602,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
aws_region_name=aws_region_name,
|
||||
extra_headers=extra_headers,
|
||||
endpoint_url=endpoint_url,
|
||||
data=json.dumps(data),
|
||||
data=json.dumps(request_data),
|
||||
headers=headers,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
|
@ -607,7 +618,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
logging_obj=logging_obj,
|
||||
optional_params=optional_params,
|
||||
encoding=encoding,
|
||||
data=data,
|
||||
data=request_data,
|
||||
complete_api_base=prepped.url,
|
||||
api_key=None,
|
||||
aembedding=aembedding,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Optional, TypeVar, cast
|
|||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import AWS_ERROR_TYPE_HEADER
|
||||
|
|
@ -23,7 +23,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
|
|||
bedrock_invoke_guardrail_headers,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
from litellm.types.utils import Embedding, EmbeddingResponse, ModelResponseStream, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import (
|
||||
|
|
@ -376,6 +376,53 @@ def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]:
|
|||
return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)}
|
||||
|
||||
|
||||
class _TitanEmbeddingsByType(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
binary: tuple[int, ...] | None = None
|
||||
floats: tuple[float, ...] | None = Field(default=None, alias="float")
|
||||
|
||||
|
||||
class _TitanEmbeddingReply(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
embedding: tuple[float, ...] | None = None
|
||||
embeddings_by_type: _TitanEmbeddingsByType = Field(default=_TitanEmbeddingsByType(), alias="embeddingsByType")
|
||||
input_text_token_count: int = Field(alias="inputTextTokenCount")
|
||||
|
||||
@property
|
||||
def vector(self) -> tuple[float, ...] | None:
|
||||
if self.embedding is not None:
|
||||
return self.embedding
|
||||
if self.embeddings_by_type.floats is not None:
|
||||
return self.embeddings_by_type.floats
|
||||
return self.embeddings_by_type.binary
|
||||
|
||||
|
||||
def _invoke_embedding_response(model: str, httpx_response: Response, has_input_image: bool) -> EmbeddingResponse | None:
|
||||
reply: Final = relayed_json_object(httpx_response)
|
||||
if reply is None or reply.keys().isdisjoint(("embedding", "embeddingsByType")):
|
||||
return None
|
||||
try:
|
||||
parsed: Final = _TitanEmbeddingReply.model_validate(reply)
|
||||
except ValidationError:
|
||||
return None
|
||||
if parsed.vector is None:
|
||||
return None
|
||||
return EmbeddingResponse(
|
||||
model=model,
|
||||
usage=Usage(
|
||||
prompt_tokens=parsed.input_text_token_count,
|
||||
completion_tokens=0,
|
||||
total_tokens=parsed.input_text_token_count,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(image_count=1, text_tokens=parsed.input_text_token_count)
|
||||
if has_input_image
|
||||
else None,
|
||||
),
|
||||
data=[Embedding(embedding=list(parsed.vector), index=0, object="embedding")],
|
||||
)
|
||||
|
||||
|
||||
def _served_service_tier(response_headers: Mapping[str, str]) -> str | None:
|
||||
headers: Final = httpx.Headers(response_headers)
|
||||
if _BEDROCK_SERVICE_TIER_HEADER not in headers:
|
||||
|
|
@ -574,6 +621,14 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
|
|||
from litellm.types.utils import LlmProviders, ModelResponse
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
embedding_response: Final = (
|
||||
_invoke_embedding_response(model, httpx_response, has_input_image="inputImage" in request_data)
|
||||
if "invoke" in endpoint
|
||||
else None
|
||||
)
|
||||
if embedding_response is not None:
|
||||
return embedding_response
|
||||
|
||||
if "invoke" in endpoint:
|
||||
chat_config_model = "invoke/" + model
|
||||
elif "converse" in endpoint:
|
||||
|
|
|
|||
|
|
@ -1671,6 +1671,7 @@ class HTTPHandler:
|
|||
follow_redirects: bool | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
stream: bool = False,
|
||||
max_response_bytes: int | None = None,
|
||||
):
|
||||
# Set follow_redirects to UseClientDefault if None
|
||||
_follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
|
||||
|
|
@ -1681,6 +1682,16 @@ class HTTPHandler:
|
|||
url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout
|
||||
)
|
||||
|
||||
if max_response_bytes is not None:
|
||||
return self._get_with_response_limit(
|
||||
url,
|
||||
params=query,
|
||||
headers=headers,
|
||||
max_bytes=max_response_bytes,
|
||||
follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects,
|
||||
timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout),
|
||||
)
|
||||
|
||||
response: Final = self.client.get(
|
||||
url,
|
||||
params=query,
|
||||
|
|
@ -1691,6 +1702,67 @@ class HTTPHandler:
|
|||
|
||||
return response
|
||||
|
||||
def _get_with_response_limit(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
params: httpx.QueryParams,
|
||||
headers: Mapping[str, str] | None,
|
||||
timeout: httpx.Timeout,
|
||||
max_bytes: int,
|
||||
follow_redirects: bool,
|
||||
) -> httpx.Response:
|
||||
request: Final = self.client.build_request(
|
||||
"GET",
|
||||
url,
|
||||
headers=MappingProxyType({**(headers or {}), "accept-encoding": "identity"}),
|
||||
params=params,
|
||||
timeout=timeout,
|
||||
)
|
||||
response: Final = self.client.send(request, stream=True, follow_redirects=False)
|
||||
return self._read_with_response_limit(response, max_bytes=max_bytes, follow_redirects=follow_redirects)
|
||||
|
||||
def _next_redirect_hop(self, hop: httpx.Response) -> httpx.Response:
|
||||
if hop.next_request is None:
|
||||
return hop
|
||||
hop.close()
|
||||
return self.client.send(hop.next_request, auth=None, stream=True, follow_redirects=False)
|
||||
|
||||
def _last_redirect_hop(self, response: httpx.Response, *, follow_redirects: bool) -> httpx.Response:
|
||||
hop = response # rebind-ok: walks the redirect chain without recursion
|
||||
for _ in range(10):
|
||||
if hop.next_request is None or not follow_redirects:
|
||||
return hop
|
||||
hop = self._next_redirect_hop(hop)
|
||||
if hop.next_request is None or not follow_redirects:
|
||||
return hop
|
||||
hop.close()
|
||||
raise ValueError("Too many redirects")
|
||||
|
||||
def _read_with_response_limit(
|
||||
self, response: httpx.Response, *, max_bytes: int, follow_redirects: bool
|
||||
) -> httpx.Response:
|
||||
final: Final = self._last_redirect_hop(response, follow_redirects=follow_redirects)
|
||||
try:
|
||||
if final.is_redirect or final.is_error:
|
||||
return httpx.Response(final.status_code, headers=final.headers, request=final.request)
|
||||
content_encoding: Final = header_value(final.headers, "content-encoding")
|
||||
if ("identity" if content_encoding is None else content_encoding).lower() != "identity":
|
||||
raise HTTPResponseLimitError("Response size limits require an uncompressed response")
|
||||
content_length: Final = header_value(final.headers, "content-length")
|
||||
if int("0" if content_length is None else content_length) > max_bytes:
|
||||
raise HTTPResponseLimitError("Response exceeds the configured size limit")
|
||||
with BytesIO() as body:
|
||||
for chunk in final.iter_bytes(chunk_size=65536):
|
||||
if body.tell() + len(chunk) > max_bytes:
|
||||
raise HTTPResponseLimitError("Response exceeds the configured size limit")
|
||||
body.write(chunk)
|
||||
return httpx.Response(
|
||||
final.status_code, headers=final.headers, content=body.getvalue(), request=final.request
|
||||
)
|
||||
finally:
|
||||
final.close()
|
||||
|
||||
def _send_streaming_get(
|
||||
self,
|
||||
url: str,
|
||||
|
|
|
|||