diff --git a/.github/actions/cache-go-build/action.yml b/.github/actions/cache-go-build/action.yml new file mode 100644 index 00000000000..f29b144c934 --- /dev/null +++ b/.github/actions/cache-go-build/action.yml @@ -0,0 +1,39 @@ +name: "Cache the Go build" +description: >- + Restore the Go module and build caches on every run and save them only from + main, so pull requests reuse main's cache instead of evicting it with their + own copies. + + actions/setup-go's built-in cache saves from every ref, and a cache saved by a + pull request is readable only by that pull request. Callers must pass + `cache: false` to setup-go and run this action after it. + +inputs: + dependency-path: + description: "Path to the go.sum that keys the cache" + required: true + +runs: + using: composite + steps: + - name: Restore and save the Go module and build caches + if: github.ref == 'refs/heads/main' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Restore the Go module and build caches + if: github.ref != 'refs/heads/main' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles(inputs.dependency-path) }} + restore-keys: | + ${{ runner.os }}-go- diff --git a/.github/actions/restore-rust-bridge/action.yml b/.github/actions/restore-rust-bridge/action.yml new file mode 100644 index 00000000000..d050431dbc6 --- /dev/null +++ b/.github/actions/restore-rust-bridge/action.yml @@ -0,0 +1,37 @@ +name: "Restore the Rust bridge" +description: >- + Restore the prebuilt Rust bridge that the unit workflow's rust-bridge job saves + on main, keyed on every input of the Rust build. + + The bridge is abi3, so one build serves every supported Python version. A job + whose Rust inputs match main installs the restored wheel and native library + instead of compiling them inside `uv sync`. On a miss the caller builds from + source as before. + + The restored `rust-bridge-dist` directory holds the editable litellm wheel, the + litellm-proxy gateway wheel, and `litellm/rust_bridge/_native.abi3.so`. + +outputs: + key: + description: "Cache key for the bridge built from this checkout" + value: ${{ steps.key.outputs.key }} + cache-hit: + description: "'true' when the bridge for this exact key was restored" + value: ${{ steps.restore.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Define the Rust bridge cache key + id: key + shell: bash + env: + RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }} + run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" + + - name: Restore the Rust bridge + id: restore + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: rust-bridge-dist + key: ${{ steps.key.outputs.key }} diff --git a/.github/osv-scanner.toml b/.github/osv-scanner.toml index 7097188c3c7..6f14f3ef629 100644 --- a/.github/osv-scanner.toml +++ b/.github/osv-scanner.toml @@ -11,22 +11,27 @@ reason = "filelock WindowsFileLock ignores mode in every release from 3.10.0 bef [[IgnoredVulns]] id = "GHSA-pcg2-g6cj-q92c" ignoreUntil = 2027-02-01 -reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped" +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped" [[IgnoredVulns]] id = "GHSA-pw9p-v7r6-gx4p" ignoreUntil = 2027-02-01 -reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped" +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped" [[IgnoredVulns]] id = "GHSA-r4j5-9xpp-3985" ignoreUntil = 2027-02-01 -reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped" +reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped" [[IgnoredVulns]] id = "GHSA-45c3-73f7-pv4m" ignoreUntil = 2027-02-01 -reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" +reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.2; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" + +[[IgnoredVulns]] +id = "GHSA-mmhr-3c5m-f64f" +ignoreUntil = 2027-02-01 +reason = "soupsieve 4.0 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup in tests/code_coverage_tests/bedrock_pricing.py to parse AWS pricing HTML with html.parser and find table rows and cells, not CSS selectors; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5" [[PackageOverrides]] name = "pillow" @@ -35,3 +40,23 @@ ecosystem = "PyPI" vulnerability.ignore = true effectiveUntil = 2027-02-01 reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups and of the mlflow extra stays on 12.3.0; 3.11+ is on 13.0.0; pillow has never shipped an older-major release after the next major (no 10.x after 11.0.0, no 11.x after 12.0.0), so 13.x is the only fix line; extended to 2027-02-01 with maintainer approval, same date as every other Python 3.10 entry; remove once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-vvf4-9x63-8xvv" +ignoreUntil = 2027-02-01 +reason = "dnspython 2.9.2 is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.2; the advisory is memory and CPU use when dns.rdata.from_text parses a WKS record from attacker-controlled zone file text, and litellm only reaches dnspython through pydantic EmailStr, which never parses zone files or does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" + +[[IgnoredVulns]] +id = "GHSA-5cjw-f85j-cv6r" +ignoreUntil = 2027-02-01 +reason = "tornado 6.6.1 is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6.1; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped" + +[[IgnoredVulns]] +id = "GHSA-6655-6qjp-jx7x" +ignoreUntil = 2027-02-01 +reason = "soupsieve 3.0.2 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. The advisory is exponential regex backtracking when compiling an untrusted CSS selector. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup only in tests/code_coverage_tests/bedrock_pricing.py, which calls find and find_all and never compiles a CSS selector; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5" + +[[IgnoredVulns]] +id = "GHSA-7g44-g9rc-x53w" +ignoreUntil = 2027-02-01 +reason = "soupsieve 3.0.2 is the first fix and requires Python >= 3.11.5, so Python 3.10 and 3.11.0-3.11.4 resolve 2.10 while newer Python versions resolve 4.1. The advisory is unbounded recursion when parsing an untrusted CSS selector with deeply nested pseudo-classes. litellm declares beautifulsoup4 in its ci dependency group and uses BeautifulSoup only in tests/code_coverage_tests/bedrock_pricing.py, which calls find and find_all and never parses a CSS selector; no runtime litellm code imports soupsieve or beautifulsoup4. Remove once 3.10 is dropped and the minimum Python version is at least 3.11.5" diff --git a/.github/pr-assets/customer-hierarchy-settings/api-proof.md b/.github/pr-assets/customer-hierarchy-settings/api-proof.md new file mode 100644 index 00000000000..4b95e1c74ec --- /dev/null +++ b/.github/pr-assets/customer-hierarchy-settings/api-proof.md @@ -0,0 +1,210 @@ +## Captured API proof + +These are actual curl requests to a local proxy and a controlled HTTP provider, using synthetic records and keys. Addresses, header files and IDs below are placeholders. Response excerpts omit unrelated fields and credentials. + +### Permission change: verified before and after + +The baseline process ran clean relevant source at `6685525fa2a809e0d4bc2c13cb07aef8eec242c3`. The requests used the same bound key and org policy; the after request was rerun after the schema update: the org allowed only `gpt-4.1-mini`; the key allowed `usage-proof-model` and was bound with `metadata.customer_org_id`. No team/member was needed for this org-level comparison. + +```sh +curl --silent --show-error --request POST "$PROXY_BASE/v1/chat/completions" \ + --header "@$SCOPED_KEY_HEADERS" --header 'Content-Type: application/json' \ + --data-binary @request.json +``` + +`request.json`: +```json +{ + "model": "usage-proof-model", + "messages": [ + { + "role": "user", + "content": "Synthetic before-after permission check" + } + ] +} +``` + +Baseline: **HTTP 200**. +```json +{ + "object": "chat.completion", + "model": "usage-proof-model", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Controlled usage proof response.", + "role": "assistant" + }, + "provider_specific_fields": {} + } + ] +} +``` + +Current implementation: **HTTP 403**, with **zero controlled-provider calls**. +```json +{ + "error": { + "message": "The requested model 'usage-proof-model' is not in the allowed models for this customer. Check the models this customer can use and try again.", + "type": "customer_model_access_denied", + "param": "model", + "code": "403" + } +} +``` + +### Current hierarchy and key intersection + +Admin requests created the records below; each returned **201**. Member `budget_id` selects its template and `budget_override_id` identifies its optional override: + +`POST /management/v1/customer_organizations` +```json +{ + "customer_org_id": "", + "alias": "Permissions and deletion proof", + "budget": { + "max_budget": 500, + "allowed_models": [ + "gpt-4.1-mini", + "gpt-5.5" + ] + }, + "default_budgets": [ + { + "name": "Standard", + "is_default": true, + "max_budget": 100, + "allowed_models": [ + "gpt-4.1-mini" + ] + } + ] +} +``` + +`POST /management/v1/customer_teams` +```json +{ + "customer_team_id": "", + "parent_org_id": "", + "alias": "Proof customer team", + "budget": { + "max_budget": 250, + "allowed_models": [ + "gpt-4.1-mini", + "gpt-5.5" + ] + } +} +``` + +`POST /management/v1/customer_members` +```json +{ + "customer_member_id": "", + "parent_team_id": "", + "alias": "Proof customer member", + "budget_override": { + "max_budget": 100, + "allowed_models": [ + "gpt-4.1-mini" + ] + } +} +``` + +A synthetic `internal_user` was created with `auto_create_key: false`. The admin then created these keys through `POST /key/generate` (each **200**; returned credentials omitted): + +| Key | models | metadata / budget binding | +| --- | --- | --- | +| A | `["gpt-4.1-mini", "gpt-5.5"]` | `{"customer_org_id":"","customer_member_id":""}` | +| B | `["gpt-4.1-mini"]` | Same org/member binding | +| C | `["gpt-4.1-mini"]` | `budget_id: ""`; no customer binding | + +Inference curl uses the appropriate key header file. Captured request bodies: +```json +{ + "model": "gpt-4.1-mini", + "messages": [ + { + "role": "user", + "content": "Synthetic permission check" + } + ] +} +``` +```json +{ + "model": "gpt-4.1-mini", + "input": "Synthetic permission check" +} +``` + +| State / key | Requested model | Chat HTTP | Responses HTTP | Provider calls per request | +| --- | --- | --- | --- | --- | +| Member restriction / A | gpt-4.1-mini | 200 | 200 | 1 | +| Member restriction / A | gpt-5.5 | 403 | 403 | 0 | +| Member widened; key B remains narrow | gpt-5.5 | 403 | 403 | 0 | +| Member widened; org narrowed / A | gpt-5.5 | 403 | 403 | 0 | + +The widening PATCH was `{"budget_override":{"allowed_models":["gpt-4.1-mini","gpt-5.5"]}}` on the member. The org narrowing PATCH was `{"budget":{"allowed_models":["gpt-4.1-mini"]}}`. Key B still denied the model after the member widened. + +### Cascade, retained history and shared key budget + +**Same-revision comparison:** deleting the populated org without `cascade=true` returned **409**: +```json +{ + "type": "urn:litellm:error:customer-hierarchy-has-children", + "title": "Has children", + "status": 409, + "detail": "Delete the customer's children before deleting the parent." +} +``` + +The confirmed cascade request returned **204**, with an empty body: + +```sh +curl --silent --show-error --request DELETE \ + "$PROXY_BASE/management/v1/customer_organizations/$ORG_ID?cascade=true" \ + --header "@$ADMIN_HEADERS" +``` + +Subsequent GETs for that org, its team and its member each returned **404**. The existing demo organization still returned **200**. + +Key C referenced the org-owned budget. After PATCHing that budget to `max_budget: 0`, its inference returned **422 `budget_exceeded` before and after cascade**, with zero provider calls. Its authenticated `GET /key/info` retained the same relation and limit (selected fields): +```json +{ + "info": { + "budget_id": "", + "litellm_budget_table": { + "max_budget": 0.0 + } + } +} +``` + +The same admin daily query returned **200** before and after cascade: + +```sh +curl --silent --show-error \ + "$PROXY_BASE/management/v1/customer_organizations/daily/activity?customer_org_ids=$ORG_ID&start_date=$UTC_DATE&end_date=$UTC_DATE" \ + --header "@$ADMIN_HEADERS" +``` + +Selected captured metadata was identical before and after; API-key/entity breakdowns are omitted: +```json +{ + "metadata": { + "total_spend": 0.28, + "total_prompt_tokens": 200, + "total_completion_tokens": 40, + "total_tokens": 240, + "total_api_requests": 4, + "total_successful_requests": 2, + "total_failed_requests": 2 + } +} +``` diff --git a/.github/pr-assets/customer-hierarchy-settings/before.png b/.github/pr-assets/customer-hierarchy-settings/before.png new file mode 100644 index 00000000000..563c231cfbe Binary files /dev/null and b/.github/pr-assets/customer-hierarchy-settings/before.png differ diff --git a/.github/pr-assets/customer-hierarchy-settings/delete-confirmation.png b/.github/pr-assets/customer-hierarchy-settings/delete-confirmation.png new file mode 100644 index 00000000000..50744628be4 Binary files /dev/null and b/.github/pr-assets/customer-hierarchy-settings/delete-confirmation.png differ diff --git a/.github/pr-assets/customer-hierarchy-settings/member-settings.png b/.github/pr-assets/customer-hierarchy-settings/member-settings.png new file mode 100644 index 00000000000..fddb3b40a4d Binary files /dev/null and b/.github/pr-assets/customer-hierarchy-settings/member-settings.png differ diff --git a/.github/pr-assets/customer-hierarchy-settings/members.png b/.github/pr-assets/customer-hierarchy-settings/members.png new file mode 100644 index 00000000000..bc4b63ed1b3 Binary files /dev/null and b/.github/pr-assets/customer-hierarchy-settings/members.png differ diff --git a/.github/pr-assets/customer-hierarchy-settings/passthrough-api-proof.md b/.github/pr-assets/customer-hierarchy-settings/passthrough-api-proof.md new file mode 100644 index 00000000000..b575ffa7d20 --- /dev/null +++ b/.github/pr-assets/customer-hierarchy-settings/passthrough-api-proof.md @@ -0,0 +1,69 @@ +The OpenAI passthrough route now resolves customer organization, team and effective member model restrictions before forwarding the request. It uses the existing customer resolver and model matcher, then checks the model again after hooks. Gateway key permissions still apply. + +Actual curl proof used a controlled HTTP upstream. Before: PR head `43d55ab2b91e4cccccf014c5f813a2b70a4e89f9` merged with target `eb811913f1`, before the fix. After: the implementation linked from the current PR description. Counts below come from the upstream request log. + +Setup requests, using synthetic identifiers: + +```http +POST /management/v1/customer_organizations +{"customer_org_id":"proof-org","alias":"Synthetic passthrough permission proof","budget":{"max_budget":100,"allowed_models":["gpt-4.1-mini"]}} + +POST /user/new +{"user_id":"proof-user","user_role":"internal_user","auto_create_key":false} + +POST /key/generate +{"user_id":"proof-user","models":["gpt-4.1-mini","gpt-5.5"],"metadata":{"customer_org_id":"proof-org"}} + +POST /key/generate +{"user_id":"proof-user","models":["gpt-4.1-mini"],"metadata":{"customer_org_id":"proof-org"}} +``` + +The first key permits both models; the second permits only mini. Credentials from both responses are omitted. + +```bash +curl "$GATEWAY_BASE_URL/openai/v1/chat/completions" \ + --header "@$CUSTOMER_KEY_AB_HEADERS" \ + --header 'Content-Type: application/json' \ + --data '{"model":"gpt-5.5","messages":[{"role":"user","content":"Synthetic passthrough policy check"}]}' +``` + +Before: HTTP 200, one upstream call. Captured response excerpt: + +```json +{ + "model": "gpt-5.5", + "usage": { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120 + } +} +``` + +After: HTTP 403, zero upstream calls. Captured response: + +```json +{ + "error": { + "message": "The requested model 'gpt-5.5' is not in the allowed models for this customer. Check the models this customer can use and try again.", + "type": "customer_model_access_denied", + "param": "model", + "code": "403" + } +} +``` + +The same key still receives HTTP 200 for `gpt-4.1-mini`, with one upstream call. After widening the organization to permit both models, the mini-only key still receives HTTP 403 for `gpt-5.5`, with zero upstream calls: + +```json +{ + "error": { + "message": "The requested model 'gpt-5.5' is not available for this API key, or the model name is invalid. Check the models available to you and try again.", + "type": "key_model_access_denied", + "param": "model", + "code": "403" + } +} +``` + +The proof deletes its synthetic organization afterward. Registered-route regressions also cover team restrictions, member templates and overrides, hook model rewrites, model-less file retrieval, and the existing configured passthrough model contract. diff --git a/.github/pr-assets/customer-hierarchy-settings/teams.png b/.github/pr-assets/customer-hierarchy-settings/teams.png new file mode 100644 index 00000000000..24bf2a8cf60 Binary files /dev/null and b/.github/pr-assets/customer-hierarchy-settings/teams.png differ diff --git a/.github/pr-assets/customer-usage-detail/before-main-6685525.png b/.github/pr-assets/customer-usage-detail/before-main-6685525.png new file mode 100644 index 00000000000..563c231cfbe Binary files /dev/null and b/.github/pr-assets/customer-usage-detail/before-main-6685525.png differ diff --git a/.github/pr-assets/customer-usage-detail/final-overview.png b/.github/pr-assets/customer-usage-detail/final-overview.png new file mode 100644 index 00000000000..fb9438653a5 Binary files /dev/null and b/.github/pr-assets/customer-usage-detail/final-overview.png differ diff --git a/.github/pr-assets/customer-usage-detail/final-report.png b/.github/pr-assets/customer-usage-detail/final-report.png new file mode 100644 index 00000000000..7118992dfdc Binary files /dev/null and b/.github/pr-assets/customer-usage-detail/final-report.png differ diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index f401d022858..2358cbfc028 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -73,14 +73,14 @@ jobs: - name: Cache uv dependencies if: steps.changes.outputs.decision != 'skip' - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cache/uv .venv - key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }} + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-uv-lint- + ${{ runner.os }}-uv- - name: Clean Python cache if: steps.changes.outputs.decision != 'skip' @@ -396,7 +396,7 @@ jobs: - name: Cache uv dependencies if: steps.changes.outputs.relevant == 'true' - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cache/uv diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 6d6b2307700..89a1c972a7d 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -38,6 +38,16 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# These jobs check packaging, installation and imports, so they build the Rust +# workspace unoptimized. The release profile's fat LTO and opt-level 3 pushed +# every job to its timeout. The overrides go through Cargo's environment +# because a maturin `--profile ci` argument also reaches dependencies that +# build from source, such as jiter on Python 3.15, which have no ci profile +env: + CARGO_PROFILE_RELEASE_OPT_LEVEL: "0" + CARGO_PROFILE_RELEASE_LTO: "off" + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "256" + jobs: resolve: runs-on: ubuntu-latest diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 362d56c314e..8ed1876a081 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -3,6 +3,8 @@ name: smoke on: pull_request: branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"] + push: + branches: [main] workflow_dispatch: permissions: @@ -52,8 +54,26 @@ jobs: with: version: "0.10.9" + - name: Restore the Rust bridge + id: rust-bridge + uses: ./.github/actions/restore-rust-bridge + + - name: Restore the Cargo build cache + if: steps.rust-bridge.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies - run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }} + env: + RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" + exit 0 + fi + .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python "$PYTHON_VERSION" --no-install-project + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/litellm-*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so - name: Create the loopback-only network namespace run: | @@ -87,6 +107,7 @@ jobs: - name: Upload smoke diagnostics if: always() + continue-on-error: true uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: merge-smoke-diagnostics-py${{ matrix.python-version }} @@ -97,6 +118,10 @@ jobs: if: always() run: sudo ip netns delete smoke + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + all-extras: name: all-extras-py${{ matrix.python-version }} runs-on: ubuntu-24.04 @@ -123,12 +148,34 @@ jobs: with: version: "0.10.9" + - name: Restore the Rust bridge + id: rust-bridge + uses: ./.github/actions/restore-rust-bridge + + - name: Restore the Cargo build cache + if: steps.rust-bridge.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install every extra and dependency group - run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }} + env: + RUST_BRIDGE_RESTORED: ${{ steps.rust-bridge.outputs.cache-hit }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + if [ "$RUST_BRIDGE_RESTORED" != "true" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" + exit 0 + fi + .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python "$PYTHON_VERSION" --no-install-project --no-install-package litellm-proxy + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so - name: Import the SDK, the proxy server and the enterprise package run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise" + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + windows-sdk: name: windows-sdk runs-on: windows-2025 @@ -154,15 +201,16 @@ jobs: with: version: "0.10.9" - - name: Set up the Rust bridge - uses: ./.github/actions/rust-bridge - - name: Install dependencies run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11 - name: Run the Windows tests run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + proxy-build-from-pip: name: proxy-build-from-pip runs-on: ubuntu-24.04 @@ -298,6 +346,10 @@ jobs: if: always() run: cat /tmp/litellm-pip.log || true + - name: Keep only source-built wheels in the uv cache + if: always() && github.ref == 'refs/heads/main' + run: uv cache prune --ci + smoke-passed: name: smoke passed needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip] diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml index 6d4eba7b25e..116d6785287 100644 --- a/.github/workflows/test-rust-db.yml +++ b/.github/workflows/test-rust-db.yml @@ -42,10 +42,6 @@ concurrency: env: CARGO_TERM_COLOR: always - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" jobs: @@ -91,11 +87,7 @@ jobs: with: workspaces: litellm/rust cache-on-failure: true - - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Python dependencies working-directory: . diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 74a6957aba4..880e8ea785c 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -78,10 +78,6 @@ concurrency: env: CARGO_TERM_COLOR: always - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" jobs: @@ -108,11 +104,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - run: cargo clippy --workspace --all-targets --locked -- -D warnings rust-test: @@ -155,11 +146,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - name: Check generated trace contracts working-directory: . run: uv run litellm/scripts/generate_trace_types.py --check @@ -206,11 +192,6 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Set up sccache - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - run: uv build --wheel --out-dir dist - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 7dc61f52364..34958526c59 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -4,6 +4,7 @@ on: push: paths: - "litellm/deploy/terraform/provider/**" + - ".github/actions/cache-go-build/**" - ".github/workflows/test-terraform-provider.yml" pull_request: branches: @@ -13,6 +14,7 @@ on: - "litellm/deploy/terraform/provider/**" - "litellm/proxy/**" - "litellm/enterprise/**" + - ".github/actions/cache-go-build/**" - ".github/workflows/test-terraform-provider.yml" permissions: @@ -38,8 +40,12 @@ jobs: - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version-file: litellm/deploy/terraform/provider/go.mod - cache: true - cache-dependency-path: litellm/deploy/terraform/provider/go.sum + cache: false + + - name: Cache the Go build + uses: ./.github/actions/cache-go-build + with: + dependency-path: litellm/deploy/terraform/provider/go.sum - name: gofmt run: | @@ -120,8 +126,12 @@ jobs: - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: go-version-file: litellm/deploy/terraform/provider/go.mod - cache: true - cache-dependency-path: litellm/deploy/terraform/provider/go.sum + cache: false + + - name: Cache the Go build + uses: ./.github/actions/cache-go-build + with: + dependency-path: litellm/deploy/terraform/provider/go.sum - name: Audit provider endpoints against the schema working-directory: litellm/deploy/terraform/provider diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4c048ace26a..f27b58149ab 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -41,13 +41,6 @@ jobs: timeout-minutes: 2 uses: ./.github/actions/detect-changes - - name: Define editable Rust bridge cache key - id: rust-bridge-key - if: steps.changes.outputs.decision != 'skip' - env: - RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm/rust/Cargo.lock', 'litellm/rust/Cargo.toml', 'litellm/rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'litellm/packaging/litellm-proxy/pyproject.toml') }} - run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" - - name: Set up Python if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 @@ -66,30 +59,17 @@ jobs: id: rust-bridge-cache if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 - with: - path: rust-bridge-dist - key: ${{ steps.rust-bridge-key.outputs.key }} + uses: ./.github/actions/restore-rust-bridge - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 5 uses: ./.github/actions/cache-cargo-build - - name: Set up sccache - if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' - uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a - with: - version: "v0.18.0" - - name: Build editable Rust bridge if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 15 env: - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" run: | mkdir -p rust-bridge-dist @@ -103,10 +83,6 @@ jobs: timeout-minutes: 15 working-directory: litellm/packaging/litellm-proxy env: - RUSTC_WRAPPER: sccache - SCCACHE_GHA_ENABLED: "true" - SCCACHE_GHA_VERSION: litellm-ci-v1 - SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} CARGO_INCREMENTAL: "0" run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../../rust-bridge-dist @@ -125,7 +101,7 @@ jobs: uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: rust-bridge-dist - key: ${{ steps.rust-bridge-key.outputs.key }} + key: ${{ steps.rust-bridge-cache.outputs.key }} - name: Expose the Rust bridge artifact id: rust-bridge-artifact diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index e97d79a95bc..73e805cadcf 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,11 +36,6 @@ _billing_time: Final[ContextVar[datetime | None]] = ContextVar("billing_time", d _post_response: Final[ContextVar[bool]] = ContextVar("post_response", default=False) _service_target: Final[ContextVar[str | None]] = ContextVar("service_target", default=None) -# Event-metadata key under which a Redis pipeline reports the sorted, comma-joined families its ops were -# declared under when they span more than one. -REDIS_FAMILIES_METADATA_KEY: Final = "families" - -_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) @@ -140,22 +135,6 @@ def with_service_target( return decorate -@contextmanager -def service_caller(caller: str | None) -> Generator[None]: - """Name the litellm code a datastore call was issued for when its own frames cannot: an operation - declared in one task and run in another (a batch op retried on the flush) carries the chain captured - where it was declared.""" - token: Final = _service_caller.set(caller) - try: - yield - finally: - _restore(_service_caller, token) - - -def current_service_caller() -> str | None: - return _service_caller.get() - - @contextmanager def mirrored_call() -> Generator[None]: """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included.""" @@ -197,6 +176,20 @@ def current_request_parent_otel_span() -> Span | None: return _request_parent_otel_span.get() +@contextmanager +def request_parent_otel_span(span: Span | None) -> Generator[None]: + """Anchor the datastore service spans made inside this block, and by the tasks it starts, under the span an SDK + caller passed. A request that already anchored its own, as the proxy does at auth, keeps it.""" + if span is None or _request_parent_otel_span.get() is not None: + yield + return + token: Final = _request_parent_otel_span.set(span) + try: + yield + finally: + _restore(_request_parent_otel_span, token) + + @contextmanager def outside_mirrored_call() -> Generator[None]: """Logging a mirrored call: the calls its callbacks start are not mirrors themselves.""" diff --git a/litellm/accounting/delivery_projection.py b/litellm/accounting/delivery_projection.py index da63542d5b8..56b25f0580a 100644 --- a/litellm/accounting/delivery_projection.py +++ b/litellm/accounting/delivery_projection.py @@ -56,6 +56,9 @@ def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpda "model_access_group_list_transactions": dict( _entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable") ), + "customer_org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerOrganizationTable")), + "customer_team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerTeamTable")), + "customer_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_CustomerMemberTable")), } diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py index d1e461efca9..4d7341336ce 100644 --- a/litellm/accounting/projection.py +++ b/litellm/accounting/projection.py @@ -1,6 +1,6 @@ from collections.abc import Iterator, Mapping from dataclasses import dataclass, replace -from datetime import datetime, timezone +from datetime import datetime from itertools import chain from types import MappingProxyType from typing import Final @@ -14,9 +14,10 @@ from litellm.constants import ( MODEL_INSIGHTS_TASK_TAG_PREFIX, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks -from litellm.proxy.db.model_usage_rollup import date_from_start_time +from litellm.proxy.db.model_usage_rollup import date_from_start_time, utc_date_from_start_time from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens @@ -70,9 +71,32 @@ def _spend(table: str, key: tuple[tuple[str, str], ...], spend: float) -> Contri return Contribution(table, key, (("spend", spend),)) +_CUSTOMER_ID_KEYS: Final = ("customer_org_id", "customer_team_id", "customer_member_id") + + +def _customer_ids(metadata: Mapping[str, object]) -> Mapping[str, str | None]: + """The customer hierarchy a request is charged to: the member the request named, else the key's attribution.""" + entities: Final = _fields(metadata.get("accounting_entities")) + if isinstance(entities.get("customer_org_id"), str): + return {key: value if isinstance(value := entities.get(key), str) else None for key in _CUSTOMER_ID_KEYS} + serialized: Final = metadata.get(CUSTOMER_USAGE_METADATA_KEY) + if not isinstance(serialized, str): + return dict.fromkeys(_CUSTOMER_ID_KEYS) + try: + attribution: Final = CustomerUsageAttribution.model_validate_json(serialized) + except ValidationError: + return dict.fromkeys(_CUSTOMER_ID_KEYS) + return { + "customer_org_id": attribution.customer_org_id, + "customer_team_id": attribution.customer_team_id, + "customer_member_id": attribution.customer_member_id, + } + + def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: spend: Final = _number(row.get("spend")) entities: Final = _fields(metadata.get("accounting_entities")) + customer: Final = _customer_ids(metadata) identities: Final = ( ("LiteLLM_VerificationToken", "token", entities.get("api_key")), ("LiteLLM_UserTable", "user_id", entities.get("user")), @@ -80,6 +104,9 @@ def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, obje ("LiteLLM_TeamTable", "team_id", entities.get("team_id")), ("LiteLLM_OrganizationTable", "organization_id", entities.get("organization_id")), ("LiteLLM_EndUserTable", "user_id", entities.get("end_user")), + ("LiteLLM_CustomerOrganizationTable", "customer_org_id", customer["customer_org_id"]), + ("LiteLLM_CustomerTeamTable", "customer_team_id", customer["customer_team_id"]), + ("LiteLLM_CustomerMemberTable", "customer_member_id", customer["customer_member_id"]), ("LiteLLM_AgentsTable", "agent_id", entities.get("agent_id")), ("LiteLLM_ProjectTable", "project_id", entities.get("project_id")), ) @@ -159,7 +186,14 @@ def _daily_contributions(row: Mapping[str, object], metadata: Mapping[str, objec ) # The legacy daily organization writer uses the charged org argument, not # SpendLogs' normalized empty-string identity for a request with no org. - daily_row: Final = {**row, "organization_id": _fields(metadata.get("accounting_entities")).get("organization_id")} + entities: Final = _fields(metadata.get("accounting_entities")) + customer: Final = _customer_ids(metadata) + daily_row: Final = { + **row, + "organization_id": entities.get("organization_id"), + "customer_org_id": customer["customer_org_id"], + **{key: customer[key] or "" for key in ("customer_team_id", "customer_member_id")}, + } return tuple(_daily_rows(daily_row, shared_key, counters)) @@ -169,13 +203,26 @@ def _daily_rows( counters: tuple[tuple[str, int | float], ...], ) -> Iterator[Contribution]: for entity, table in DAILY_SPEND_TABLES.items(): + customer_date = _utc_date(row) if entity == "customer_member" else None + if entity == "customer_member" and customer_date is None: + continue + daily_key = tuple( + (column, customer_date if column == "date" and customer_date is not None else value) + for column, value in shared_key + ) + model_group = (("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),) for identity in daily_spend_entity_ids(row, entity): yield Contribution( table.name, - ((table.entity_id_column, _text(identity)), *shared_key), + ( + (table.entity_id_column, _text(identity)), + *((column, _text(row.get(column))) for column in table.parent_id_columns), + *daily_key, + *((("model_group", _text(row.get("model_group"))),) if table.hashed_identity else ()), + ), counters, ( - ("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))), + *(() if table.hashed_identity else model_group), *((("request_id", _text(row.get("request_id"))),) if table.carries_request_id else ()), ), ) @@ -259,18 +306,7 @@ def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, objec def _utc_date(row: Mapping[str, object]) -> str | None: start: Final = row.get("startTime") - try: - parsed: Final = datetime.fromisoformat(start.replace("Z", "+00:00")) if isinstance(start, str) else start - except ValueError: - return None - if not isinstance(parsed, datetime): - return None - return ( - (parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed) - .astimezone(timezone.utc) - .date() - .isoformat() - ) + return utc_date_from_start_time(start) if isinstance(start, (datetime, str)) else None def _guardrail_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]: diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py index 874b90110c1..86fbf94cb90 100644 --- a/litellm/accounting/snapshot.py +++ b/litellm/accounting/snapshot.py @@ -31,6 +31,7 @@ _METADATA_FIELDS: Final = frozenset( "compression_savings", "internal_call_origin", "billed_retry_attempt", + "user_api_key_customer_usage", } ) @@ -51,6 +52,9 @@ _ACCOUNTING_FIELDS: Final = frozenset( "team_id", "organization_id", "end_user", + "customer_member_id", + "customer_team_id", + "customer_org_id", "agent_id", "request_tags", "status", diff --git a/litellm/accounting/worker.py b/litellm/accounting/worker.py index 3c904615255..00e65dc1efe 100644 --- a/litellm/accounting/worker.py +++ b/litellm/accounting/worker.py @@ -10,8 +10,10 @@ from typing import ( from litellm._logging import verbose_proxy_logger from litellm.accounting.application import apply_deliveries +from litellm.accounting.delivery_projection import entity_transactions from litellm.accounting.metrics import record_backlog, record_storage from litellm.accounting.projection import Contribution +from litellm.proxy.spend_tracking.customer_budget_alerts import CustomerBudgetAlerting from litellm.repositories.accounting_delivery_repository import ( AccountingDeliveryRepository, DeliveryBatchResult, @@ -65,6 +67,9 @@ async def flush_accounting_deliveries( result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only) if result.applied: await writer.invalidate_accounting_entities(applied, proxy_logging_obj) + await CustomerBudgetAlerting.detect_committed_batch( + entity_transactions(applied), prisma_client, proxy_logging_obj.internal_usage_cache.dual_cache + ) if result.parked: verbose_proxy_logger.error( "Accounting delivery parked %d requests after isolated failures", len(result.parked) diff --git a/litellm/caching/affinity_cache.py b/litellm/caching/affinity_cache.py index 2b8dc67da7a..0b05e248a98 100644 --- a/litellm/caching/affinity_cache.py +++ b/litellm/caching/affinity_cache.py @@ -1,151 +1,42 @@ """Atomic affinity claims shared by deployment and tier-model selection.""" -import json from collections.abc import Mapping -from typing import ( - Final, - cast, # noqa: TID251 # Heterogeneous cache results are narrowed only to object, then validated -) - -from pydantic import JsonValue, TypeAdapter, ValidationError +from typing import Final from litellm._logging import verbose_router_logger -from litellm.caching.dual_cache import DualCache -from litellm.caching.redis_batch import active_request_redis_batch +from litellm.caching.redis_batch import Batch +from litellm.redis.stores.router_session_pins import ROUTER_SESSION_PINS_TARGET, PinKey, RouterSessionPinStore -ROUTER_SESSION_PINS_TARGET: Final = "router_session_pins" - -_PIN_JSON_ADAPTER: Final = TypeAdapter[JsonValue](JsonValue) - -_READ_PIN_SCRIPT: Final = """ -local current = redis.call('GET', KEYS[1]) -if current ~= false then - redis.call('EXPIRE', KEYS[1], ARGV[1]) -end -return current -""" - -_CLAIM_PIN_SCRIPT: Final = """ -local current = redis.call('GET', KEYS[1]) -if current == false then - redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) - return ARGV[1] -end -if ARGV[3] then - local decoded, stored = pcall(cjson.decode, current) - if ARGV[4] then - local legacy = current - if decoded and type(stored) == 'string' then legacy = stored end - for _, eligible in ipairs(cjson.decode(ARGV[3])) do - if eligible['model_id'] == legacy then - redis.call('EXPIRE', KEYS[1], ARGV[2]) - return current - end - end - end - if decoded and type(stored) == 'table' then - for _, eligible in ipairs(cjson.decode(ARGV[3])) do - local matches = true - for key, value in pairs(eligible) do - if stored[key] ~= value then matches = false; break end - end - for key, _ in pairs(stored) do - if eligible[key] == nil then matches = false; break end - end - if matches then - redis.call('EXPIRE', KEYS[1], ARGV[2]) - return current - end - end - end - redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) - return ARGV[1] -end -if current == ARGV[1] then - redis.call('EXPIRE', KEYS[1], ARGV[2]) -end -return current -""" +__all__ = [ + "ROUTER_SESSION_PINS_TARGET", + "claim_affinity_pin", + "read_affinity_pin", + "set_local_affinity_pin", +] -def set_local_affinity_pin(cache: DualCache, cache_key: str, value: object, ttl_seconds: int) -> None: - """Replace the entry because InMemoryCache.set_cache preserves a live key's expiry.""" - cache.in_memory_cache.delete_cache(cache_key) - cache.in_memory_cache.set_cache(cache_key, value, ttl=ttl_seconds) +def set_local_affinity_pin(pins: RouterSessionPinStore, cache_key: PinKey, value: object, ttl_seconds: int) -> None: + pins.set_local(cache_key, value, ttl_seconds) -def _legacy_pin_matches(stored: object, pin_value: Mapping[str, str]) -> bool: - if isinstance(stored, dict): - return all(stored.get(key) is not None and str(stored[key]) == value for key, value in pin_value.items()) - return isinstance(stored, str) and len(pin_value) == 1 and stored in pin_value.values() - - -def claim_affinity_pin_in_memory( - cache: DualCache, - cache_key: str, - pin_value: Mapping[str, str], - ttl_seconds: int, - *, - eligible_values: tuple[Mapping[str, str], ...] | None = None, - preserve_legacy_model_ids: bool = False, -) -> object: - """No await between read and write, so same-loop claims agree during a Redis outage.""" - existing: Final[object] = cache.in_memory_cache.get_cache(cache_key) - if existing is not None and eligible_values is None: - if _legacy_pin_matches(existing, pin_value): - set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds) - return existing - eligible_existing: Final = existing in (eligible_values or ()) or ( - preserve_legacy_model_ids - and isinstance(existing, str) - and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ())) - ) - winner: Final = existing if existing is not None and eligible_existing else pin_value - set_local_affinity_pin(cache, cache_key, winner, ttl_seconds) - return winner - - -def _decode_pin(value: str) -> object: - try: - return _PIN_JSON_ADAPTER.validate_json(value) - except ValidationError: - return value - - -async def read_affinity_pin(cache: DualCache, cache_key: str, ttl_seconds: int) -> object: +async def read_affinity_pin(pins: RouterSessionPinStore, cache_key: PinKey, ttl_seconds: int) -> object: """Refresh the authoritative pin without writing a previously observed value back.""" - redis_cache: Final = cache.redis_cache - if redis_cache is not None: + if pins.has_redis: try: - read_script: Final = redis_cache.async_register_script(_READ_PIN_SCRIPT) - batch: Final = active_request_redis_batch(redis_cache) - raw: Final = cast( # cast-ok: Redis script returns a serialized pin or a miss; validate before use - object, - await batch.script(_READ_PIN_SCRIPT, read_script, (cache_key,), (int(ttl_seconds),)) - if batch is not None and batch.pending - else await read_script(keys=(cache_key,), args=(int(ttl_seconds),)), - ) - decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw - value: Final = _decode_pin(decoded) if isinstance(decoded, str) else None - if value is None: - cache.in_memory_cache.delete_cache(cache_key) - else: - set_local_affinity_pin(cache, cache_key, value, ttl_seconds) + value: Final = await pins.read(cache_key, ttl_seconds, batch=Batch.PRE_CALL) except Exception as error: # noqa: BLE001 # Redis faults retain pod-local continuity, unlike a successful miss verbose_router_logger.debug("Affinity cache: Redis read failed, using pod-local pin. error=%s", error) else: return value - local_value: Final = cast( # cast-ok: the heterogeneous local cache is parsed by the affinity caller - object, cache.in_memory_cache.get_cache(cache_key) - ) + local_value: Final = pins.get_local(cache_key) if local_value is not None: - set_local_affinity_pin(cache, cache_key, local_value, ttl_seconds) + set_local_affinity_pin(pins, cache_key, local_value, ttl_seconds) return local_value async def claim_affinity_pin( - cache: DualCache, - cache_key: str, + pins: RouterSessionPinStore, + cache_key: PinKey, pin_value: Mapping[str, str], ttl_seconds: int, *, @@ -157,33 +48,20 @@ async def claim_affinity_pin( Eligible claims refresh the returned winner. Legacy deployment claims only refresh a matching candidate. Resolve Redis per call because the proxy attaches it lazily. """ - redis_cache: Final = cache.redis_cache - if redis_cache is not None: + if pins.has_redis: try: - claim_script: Final = redis_cache.async_register_script(_CLAIM_PIN_SCRIPT) - args: Final = ( - json.dumps(dict(pin_value)), - int(ttl_seconds), - *( - (json.dumps(tuple(dict(value) for value in eligible_values)),) - if eligible_values is not None - else () - ), - *(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()), + winner: Final = await pins.claim( + cache_key, + pin_value, + ttl_seconds, + eligible_values=eligible_values, + preserve_legacy_model_ids=preserve_legacy_model_ids, ) - raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here - object, await claim_script(keys=(cache_key,), args=args) - ) - decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw - if not isinstance(decoded, str): - return pin_value - winner: Final = _decode_pin(decoded) - set_local_affinity_pin(cache, cache_key, winner, ttl_seconds) - return winner except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error) - return claim_affinity_pin_in_memory( - cache, + else: + return pin_value if winner is None else winner + return pins.claim_local( cache_key, pin_value, ttl_seconds, diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index cbb01cae603..f0d400b7b01 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -8,11 +8,10 @@ Has 4 primary methods: - async_get_cache """ -import asyncio import itertools import logging import time -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from threading import Lock from typing import TYPE_CHECKING, Any, Final @@ -24,7 +23,6 @@ from litellm.types.caching import RedisPipelineIncrementOperation from .base_cache import BaseCache from .in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache -from .redis_batch import BatchResult, RedisBatch, active_post_call_redis_batch, active_request_redis_batch from .redis_cache import RedisCache, RedisCircuitBreakerOpenError, log_redis_failure if TYPE_CHECKING: @@ -59,14 +57,6 @@ class PendingBatchRead: previous_access_times: dict[str, float | None] -def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None: - if future.cancelled(): - return - failure: Final = future.exception() - if failure is not None: - log_redis_failure(verbose_logger, logging.WARNING, "post-call Redis increment failed", failure) - - class DualCache(BaseCache): """ DualCache is a cache implementation that updates both Redis and an in-memory cache simultaneously. @@ -269,9 +259,6 @@ class DualCache(BaseCache): result = in_memory_result if result is None and self.redis_cache is not None and local_only is False: - request_batch: Final = active_request_redis_batch(self.redis_cache) - if request_batch is not None and request_batch.read_as_missing(key): - return None # If not found in in-memory cache, try fetching from Redis redis_result: Final = await self.redis_cache.async_get_cache(key, parent_otel_span=parent_otel_span) @@ -316,20 +303,6 @@ class DualCache(BaseCache): return sublist_keys, previous_access_times - def reserve_redis_batch_reads(self, keys: Sequence[str]) -> tuple[list[str], dict[str, float | None]]: - """Reserve the memory-missed keys whose throttled Redis reads are due, as a batch read would.""" - if self.redis_cache is None: - return [], {} - key_list: Final = list(keys) - memory: Final = self.in_memory_cache - in_memory_result: Final = ( - None - if memory is None # pyright: ignore[reportUnnecessaryComparison] # handle an absent in-memory tier - else memory.batch_get_cache(key_list) - ) - result: Final = in_memory_result if in_memory_result is not None else tuple(None for _ in key_list) - return self._reserve_redis_batch_keys(time.time(), key_list, result) - def _rollback_redis_batch_key_reservations(self, previous_access_times: dict[str, float | None]) -> None: with self._last_redis_batch_access_time_lock: for key, previous_time in previous_access_times.items(): @@ -493,28 +466,6 @@ class DualCache(BaseCache): verbose_logger, logging.ERROR, "LiteLLM Cache: exception in async add_cache", e, with_traceback=True ) - async def async_set_cache_pre_call(self, key: str, value: object, ttl: float | None) -> BatchResult[None] | None: - """Memory now, the Redis SET on the request's pipeline, sent with the next read any caller awaits; None - when no pipeline is open, so the caller takes its direct path.""" - batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache) - return None if batch is None else await self._set_on_batch(batch, key, value, ttl) - - async def async_delete_cache_pre_call(self, key: str) -> Awaitable[None] | None: - """Memory now, the Redis DEL on the request's pipeline; None when no pipeline is open, so the caller - takes its direct path.""" - batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache) - if batch is None: - return None - if self.in_memory_cache is not None: - self.in_memory_cache.delete_cache(key) - return batch.delete(key) - - async def _set_on_batch(self, batch: RedisBatch, key: str, value: object, ttl: float | None) -> BatchResult[None]: - effective_ttl: Final = self.default_in_memory_ttl if ttl is None else ttl - if self.in_memory_cache is not None: - await self.in_memory_cache.async_set_cache(key, value, ttl=effective_ttl) - return batch.set(key, value, effective_ttl) - # async_batch_set_cache async def async_set_cache_pipeline( self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs @@ -582,41 +533,6 @@ class DualCache(BaseCache): ) return result - async def async_increment_cache_post_call( - self, - key: str, - value: float, - ttl: int | None, - parent_otel_span: Span | None = None, - ) -> None: - """Memory is incremented now; the Redis increment rides the request's post-call pipeline when one is - open, and runs on its own as ``async_increment_cache`` otherwise.""" - await self.async_increment_cache_pipeline_post_call( - (RedisPipelineIncrementOperation(key=key, increment_value=value, ttl=ttl),), parent_otel_span - ) - - async def async_increment_cache_pipeline_post_call( - self, - increment_list: Sequence["RedisPipelineIncrementOperation"], - parent_otel_span: Span | None = None, - ) -> None: - batch: Final = None if self.redis_cache is None else active_post_call_redis_batch(self.redis_cache) - operations: Final = list(increment_list) - if batch is None: - await self.async_increment_cache_pipeline(operations, parent_otel_span=parent_otel_span) - return - try: - if self.in_memory_cache is not None: - await self.in_memory_cache.async_increment_pipeline( - increment_list=operations, parent_otel_span=parent_otel_span - ) - except Exception as e: # noqa: BLE001 # same tolerance as async_increment_cache_pipeline - log_redis_failure(verbose_logger, logging.WARNING, "in-memory increment failed", e) - for operation in increment_list: - batch.increment(operation["key"], operation["increment_value"], operation["ttl"]).on_settled( - _log_deferred_increment_failure - ) - async def async_increment_cache_pipeline( self, increment_list: list["RedisPipelineIncrementOperation"], diff --git a/litellm/caching/redis_batch.py b/litellm/caching/redis_batch.py index d621d09b439..b8f14d7d711 100644 --- a/litellm/caching/redis_batch.py +++ b/litellm/caching/redis_batch.py @@ -1,453 +1,24 @@ -"""One Redis pipeline for several independent operations, each with its own result and its own failure. - -A ``RedisBatch`` collects MGETs, Lua scripts and increments declared by unrelated callers and sends them -in one ``pipeline(transaction=False)`` round trip. Every declaration returns an awaitable; awaiting one -flushes whatever has been declared so far, so callers keep their existing ``await`` shape and their own -error handling while sharing the wire. Redis Cluster clients run each operation on its own, as before: -a cluster pipeline is per node anyway and the existing per-operation paths already group by slot. -""" +"""The current request's native Redis batches: a pre-call batch for the reads it needs before answering and a +post-call batch for the writes nobody waits on, both owned by the native ``NativeRequestRedis``.""" from __future__ import annotations import asyncio -import hashlib -import json -import logging -import time import weakref -from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence -from contextvars import ContextVar, Token, copy_context -from dataclasses import dataclass, field -from datetime import timedelta +from collections.abc import Awaitable, Callable, Generator, Sequence +from contextvars import ContextVar, Token from enum import Enum, auto -from types import MappingProxyType, TracebackType +from types import TracebackType from typing import TYPE_CHECKING, Final, Generic, Literal, Protocol, TypeAlias, TypeVar -from litellm._internal_context import ( - REDIS_FAMILIES_METADATA_KEY, - current_service_target, - service_caller, - service_target, -) -from litellm._logging import verbose_logger -from litellm.caching.redis_cache import ( - RedisCache, - _get_call_stack_info, # pyright: ignore[reportPrivateUsage] # same caller chain every RedisCache method reports - _run_under_circuit_breaker, # pyright: ignore[reportPrivateUsage] # same health signal as every RedisCache method - log_redis_failure, -) -from litellm.caching.redis_cluster_cache import RedisClusterCache -from litellm.types.services import ServiceTypes - if TYPE_CHECKING: from litellm.rust_bridge._native import NativeRequestRedis, NativeSend _T = TypeVar("_T") _T_co: Final = TypeVar("_T_co", covariant=True) -_ScriptArg = str | bytes | int | float -SettledHook = Callable[[asyncio.Future[_T]], Awaitable[None] | None] POST_CALL_FLUSH_DEADLINE_SECONDS: Final = 1.0 -class RegisteredScript(Protocol): - def __call__(self, keys: Sequence[str], args: Sequence[_ScriptArg]) -> Awaitable[object]: ... - - -class _RedisPipeline(Protocol): - def mget(self, keys: Sequence[str]) -> object: ... - def evalsha(self, sha: str, numkeys: int, *keys_and_args: _ScriptArg) -> object: ... - def incrbyfloat(self, name: str, amount: float) -> object: ... - def expire(self, name: str, time: timedelta) -> object: ... - def set(self, name: str, value: str, ex: timedelta | None = None) -> object: ... - def delete(self, *names: str) -> object: ... - async def execute(self, raise_on_error: bool = True) -> list[object]: ... - - -class _Op(Generic[_T]): - """One declared operation: how many pipeline replies it consumes, how to turn them into a result, and - how to run on its own when the batch cannot pipeline (cluster client, or a reply the pipeline cannot - settle, like NOSCRIPT).""" - - __slots__ = ("caller", "future", "settled_hooks", "target") - - def __init__(self) -> None: - self.future: Final[asyncio.Future[_T]] = asyncio.get_running_loop().create_future() - self.future.add_done_callback(_mark_retrieved) - self.settled_hooks: Final[list[SettledHook[_T]]] = [] # mutable-ok: append-only registry - self.target: Final = current_service_target() - self.caller: Final = _get_call_stack_info() - - async def run_settled_hooks(self) -> None: - for hook in self.settled_hooks: - await self._run_settled_hook(hook) - - async def _run_settled_hook(self, hook: SettledHook[_T]) -> None: - try: - follow_up: Final = hook(self.future) - if follow_up is not None: - await follow_up - except Exception as e: # noqa: BLE001 # one owner's follow-up must not stop the others - verbose_logger.warning("redis batch settled hook failed: %s", e) - - def enqueue(self, pipe: _RedisPipeline) -> int: - raise NotImplementedError - - def resolve(self, replies: Sequence[object]) -> _T: - raise NotImplementedError - - async def run_alone(self) -> _T: - raise NotImplementedError - - def settle(self, replies: Sequence[object]) -> Awaitable[None] | None: - """Resolve from pipeline replies; return a coroutine when the op has to be retried on its own.""" - failure: Final = next((reply for reply in replies if isinstance(reply, Exception)), None) - if failure is None: - try: - self.future.set_result(self.resolve(replies)) - except Exception as e: # noqa: BLE001 # a reply this op cannot decode fails this op alone - self.future.set_exception(e) - return None - if _is_missing_script(failure): - return self._settle_alone() - self.future.set_exception(failure) - return None - - async def _settle_alone(self) -> None: - try: - with service_target(self.target), service_caller(self.caller): - self.future.set_result(await self.run_alone()) - except Exception as e: # noqa: BLE001 # the declaring caller owns the failure of its own operation - self.future.set_exception(e) - - -def _is_missing_script(failure: Exception) -> bool: - """Imported lazily: this module is reachable from a base ``import litellm`` while redis is not a base dependency.""" - from redis.exceptions import NoScriptError - - return isinstance(failure, NoScriptError) - - -def _mark_retrieved(future: asyncio.Future[object]) -> None: - """A caller that stops awaiting (cancelled request) must not leave an 'exception never retrieved' log.""" - if not future.cancelled(): - future.exception() - - -class _MGet(_Op[Mapping[str, object]]): - __slots__ = ("_keys", "_redis_cache") - - def __init__(self, redis_cache: RedisCache, keys: Sequence[str]) -> None: - super().__init__() - self._redis_cache: Final = redis_cache - self._keys: Final[tuple[str, ...]] = tuple(dict.fromkeys(keys)) - - def enqueue(self, pipe: _RedisPipeline) -> int: - pipe.mget(tuple(self._redis_cache.check_and_fix_namespace(key=key) for key in self._keys)) - return 1 - - def resolve(self, replies: Sequence[object]) -> Mapping[str, object]: - values: Final = replies[0] - if not isinstance(values, (list, tuple)): - raise TypeError(f"MGET reply is not a list: {type(values).__name__}") - return MappingProxyType( - {key: self._redis_cache._get_cache_logic(value) for key, value in zip(self._keys, values)} # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # shared decode with async_batch_get_cache - ) - - async def run_alone(self) -> Mapping[str, object]: - found: Mapping[str, object] = await self._redis_cache.async_batch_get_cache(key_list=list(self._keys)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped cache API - if any(key not in found for key in self._keys): - raise ConnectionError("batch get did not return every key") - return found - - -class _Script(_Op[object]): - __slots__ = ("_args", "_keys", "_redis_cache", "_run", "_sha") - - def __init__( - self, - redis_cache: RedisCache, - source: str, - run: RegisteredScript, - keys: Sequence[str], - args: Sequence[_ScriptArg], - ) -> None: - super().__init__() - self._redis_cache: Final = redis_cache - self._sha: Final = hashlib.sha1(source.encode()).hexdigest() # noqa: S324 # EVALSHA identifies scripts by SHA-1 - self._run: Final = run - self._keys: Final[tuple[str, ...]] = tuple(keys) - self._args: Final[tuple[_ScriptArg, ...]] = tuple(args) - - def enqueue(self, pipe: _RedisPipeline) -> int: - namespaced: Final = tuple(self._redis_cache.check_and_fix_namespace(key=key) for key in self._keys) - pipe.evalsha(self._sha, len(namespaced), *namespaced, *self._args) - return 1 - - def resolve(self, replies: Sequence[object]) -> object: - return replies[0] - - async def run_alone(self) -> object: - return await self._run(keys=self._keys, args=self._args) - - -class _Increment(_Op[float]): - __slots__ = ("_key", "_redis_cache", "_ttl", "_value") - - def __init__(self, redis_cache: RedisCache, key: str, value: float, ttl: int | None) -> None: - super().__init__() - self._redis_cache: Final = redis_cache - self._key: Final = key - self._value: Final = value - self._ttl: Final = ttl - - def enqueue(self, pipe: _RedisPipeline) -> int: - name: Final = self._redis_cache.check_and_fix_namespace(key=self._key) - pipe.incrbyfloat(name, self._value) - if self._ttl is None: - return 1 - pipe.expire(name, timedelta(seconds=self._ttl)) - return 2 - - def resolve(self, replies: Sequence[object]) -> float: - reply: Final = replies[0] - if not isinstance(reply, (int, float, str, bytes)): - raise TypeError(f"INCRBYFLOAT reply is not numeric: {type(reply).__name__}") - return float(reply) - - async def run_alone(self) -> float: - value: object = await self._redis_cache.async_increment(key=self._key, value=self._value, ttl=self._ttl) # pyright: ignore[reportUnknownMemberType] # untyped cache API - if not isinstance(value, (int, float)): - raise TypeError(f"increment did not return a number: {type(value).__name__}") - return float(value) - - -class _Set(_Op[None]): - """SET with the cache's TTL rules, same encoding as ``async_set_cache_pipeline_with_ttls``.""" - - __slots__ = ("_key", "_redis_cache", "_ttl", "_value") - - def __init__(self, redis_cache: RedisCache, key: str, value: object, ttl: float | None) -> None: - super().__init__() - self._redis_cache: Final = redis_cache - self._key: Final = key - self._value: Final = value - self._ttl: Final = ttl - - def enqueue(self, pipe: _RedisPipeline) -> int: - ttl: Final = self._redis_cache.get_ttl(ttl=self._ttl) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped cache API - pipe.set( - self._redis_cache.check_and_fix_namespace(key=self._key), - json.dumps(self._value), - ex=None if ttl is None else timedelta(seconds=ttl), - ) - return 1 - - def resolve(self, replies: Sequence[object]) -> None: - return None - - async def run_alone(self) -> None: - await self._redis_cache.async_set_cache_pipeline_with_ttls(((self._key, self._value, self._ttl),)) - - -class _Delete(_Op[None]): - """DEL of one key, the pipelined twin of ``async_delete_cache``.""" - - __slots__ = ("_key", "_redis_cache") - - def __init__(self, redis_cache: RedisCache, key: str) -> None: - super().__init__() - self._redis_cache: Final = redis_cache - self._key: Final = key - - def enqueue(self, pipe: _RedisPipeline) -> int: - pipe.delete(self._redis_cache.check_and_fix_namespace(key=self._key)) - return 1 - - def resolve(self, replies: Sequence[object]) -> None: - return None - - async def run_alone(self) -> None: - await self._redis_cache.async_delete_cache(self._key) - - -class BatchResult(Generic[_T]): - """Awaitable handle for one declared operation; awaiting it flushes the batch it belongs to.""" - - __slots__ = ("_batch", "_op") - - def __init__(self, batch: RedisBatch, op: _Op[_T]) -> None: - self._batch: Final = batch - self._op: Final = op - - def __await__(self) -> Generator[object, None, _T]: - return self._wait().__await__() - - async def _wait(self) -> _T: - if not self._op.future.done(): - await self._batch.flush() - return self._op.future.result() - - @property - def done(self) -> bool: - return self._op.future.done() - - def on_settled(self, hook: SettledHook[_T]) -> None: - """For an owner that does not await: runs inside the flush once this operation has its result or - failure (or was cancelled with the pipeline), so the flush completes with the follow-up done.""" - self._op.settled_hooks.append(hook) - - -@dataclass(slots=True) -class RedisBatch: - """Operations declared here go out in one pipeline the next time any of them is awaited or ``flush`` runs.""" - - redis_cache: RedisCache - name: str = "redis_batch" - _pending: list[_Op[object]] = field(default_factory=list) # mutable-ok: drained by flush - _flush_hooks: list[Callable[[], None]] = field(default_factory=list) # mutable-ok: append-only registry - _lock: asyncio.Lock = field(default_factory=asyncio.Lock) - _misses: set[str] = field(default_factory=set) # mutable-ok: keys an MGET of this request read as absent - flushes: int = 0 - - def mget(self, keys: Sequence[str]) -> BatchResult[Mapping[str, object]]: - op: Final = _MGet(self.redis_cache, keys) - op.future.add_done_callback(self._note_misses) - return self._declare(op) - - def _note_misses(self, future: asyncio.Future[Mapping[str, object]]) -> None: - if future.cancelled() or future.exception() is not None: - return - self._misses.update(key for key, value in future.result().items() if value is None) - - def read_as_missing(self, key: str) -> bool: - """True when an MGET on this batch already found no value under ``key`` and nothing has set it since, - so a per-key GET later in the same request can be answered without another round trip.""" - return key in self._misses - - def script( - self, source: str, run: RegisteredScript, keys: Sequence[str], args: Sequence[_ScriptArg] - ) -> BatchResult[object]: - return self._declare(_Script(self.redis_cache, source, run, keys, args)) - - def increment(self, key: str, value: float, ttl: int | None = None) -> BatchResult[float]: - return self._declare(_Increment(self.redis_cache, key, value, ttl)) - - def set(self, key: str, value: object, ttl: float | None = None) -> BatchResult[None]: - self._misses.discard(key) - return self._declare(_Set(self.redis_cache, key, value, ttl)) - - def delete(self, key: str) -> BatchResult[None]: - self._misses.add(key) - return self._declare(_Delete(self.redis_cache, key)) - - def add_flush_hook(self, hook: Callable[[], None]) -> None: - """Called at the start of every flush so lazily bound readers can declare their keys into the same trip.""" - self._flush_hooks.append(hook) - - @property - def pending(self) -> int: - return len(self._pending) - - def _declare(self, op: _Op[_T]) -> BatchResult[_T]: - self._pending.append(op) # pyright: ignore[reportArgumentType] # heterogeneous ops share the flush loop - return BatchResult(self, op) - - async def flush(self) -> None: - async with self._lock: - for hook in self._flush_hooks: - hook() - ops: Final = tuple(self._pending) - self._pending.clear() - if not ops: - return - self.flushes += 1 - try: - if isinstance(self.redis_cache, RedisClusterCache): - await asyncio.gather(*(op._settle_alone() for op in ops)) # pyright: ignore[reportPrivateUsage] # batch owns its ops - else: - await self._flush_pipeline(ops) - finally: - for op in ops: - if not op.future.done(): - op.future.cancel() - await asyncio.gather(*(op.run_settled_hooks() for op in ops)) - - async def _flush_pipeline(self, ops: Sequence[_Op[object]]) -> None: - start_time: Final = time.time() - target, metadata = _pipeline_service_event(ops) - widths: list[int] = [] # mutable-ok: filled while enqueuing - - async def run() -> list[object]: - client: Final = self.redis_cache.init_async_client() - async with client.pipeline(transaction=False) as pipe: - widths.extend(op.enqueue(pipe) for op in ops) - return await pipe.execute(raise_on_error=False) - - try: - replies: Final = await _run_under_circuit_breaker(self.redis_cache._health, self.name, run) # pyright: ignore[reportPrivateUsage] # same breaker as the cache's own methods - except Exception as e: # noqa: BLE001 # each declaring caller applies its own Redis fallback - log_redis_failure(verbose_logger, logging.WARNING, f"{self.name}: pipeline of {len(ops)} ops failed", e) - with service_target(target): - asyncio.create_task( - self.redis_cache.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, - duration=time.time() - start_time, - error=e, - call_type=self.name, - start_time=start_time, - end_time=time.time(), - event_metadata=metadata, - ) - ) - for op in ops: - op.future.set_exception(e) - return - with service_target(target): - asyncio.create_task( - self.redis_cache.service_logger_obj.async_service_success_hook( - service=ServiceTypes.REDIS, - duration=time.time() - start_time, - call_type=self.name, - start_time=start_time, - end_time=time.time(), - event_metadata=metadata, - ) - ) - retries: list[Awaitable[None]] = [] # mutable-ok: collected while slicing replies - offset = 0 - for op, width in zip(ops, widths): - retry = op.settle(replies[offset : offset + width]) - offset += width - if retry is not None: - retries.append(retry) - if retries: - await asyncio.gather(*retries) - - -MIXED_PIPELINE_TARGET: Final = "mixed" - - -def _pipeline_service_event(ops: Sequence[_Op[object]]) -> tuple[str | None, dict[str, int | str]]: - """The target and metadata of one pipeline flush: the one key family every op was declared under, or - ``"mixed"`` plus the sorted families when owners of several families share the trip.""" - families: Final = sorted({op.target for op in ops if op.target is not None}) - if len(families) > 1: - return MIXED_PIPELINE_TARGET, {"op_count": len(ops), REDIS_FAMILIES_METADATA_KEY: ",".join(families)} - return next(iter(families), None), {"op_count": len(ops)} - - -def _backend_key(redis_cache: RedisCache) -> object: - """Two ``RedisCache`` instances built from the same connection settings and namespace talk to the same server - under the same key prefix, so the proxy's cache and the router's cache share one pipeline (the router gets its - port as a string, hence the ``str`` comparison); a cache whose settings cannot be compared (a test double) gets - its own.""" - try: - settings: Final = tuple(sorted((str(k), str(v)) for k, v in redis_cache.redis_kwargs.items() if v is not None)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportUnknownArgumentType] # untyped cache API - except AttributeError: - return ("instance", id(redis_cache)) - return (type(redis_cache), redis_cache.namespace, settings) - - class NativePending(Protocol[_T_co]): def resolve(self) -> Awaitable[_T_co]: ... @@ -511,30 +82,22 @@ def _native_send(phase: RequestPhase, request: NativeRequestRedis) -> NativeSend _open_post_call: Final[weakref.WeakSet[RequestRedisBatches]] = weakref.WeakSet() -"""Requests whose post-call batch still holds declared ops, so a shutdown can send them before Redis goes away.""" +"""Requests whose post-call batch may still hold declared ops, so a shutdown can send them before Redis goes away.""" class RequestRedisBatches: - """One ``RedisBatch`` per Redis backend for the current request, so readers of different caches that - share a server (the proxy's and the router's) share the pipeline. - - The post-call batches hold the writes nothing waits on (counters, token scripts, the response cache). - They flush once, when the success or failure callbacks have all run, or at ``post_call_deadline`` - seconds after the first declaration when no callback phase closes them.""" + """The request's native batches. The post-call batch goes out once, when the success or failure callbacks have + all run, or ``post_call_deadline`` seconds after its first declaration when no callback phase closes it.""" __slots__ = ( "__weakref__", - "_batches", - "_deadline", - "_deadline_flush", "_native", - "_native_context", "_native_factory", "_native_flush_hooks", "_native_misses", "_native_send_factory", - "_post_call", "_post_call_settlements", + "_post_call_unsent", "post_call_deadline", "prefetched", ) @@ -545,54 +108,27 @@ class RequestRedisBatches: native_factory: NativeRequestFactory = _native_request, native_send_factory: NativeSendFactory = _native_send, ) -> None: - self._batches: Final[dict[object, RedisBatch]] = {} # mutable-ok: lazily filled per backend - self._post_call: Final[dict[object, RedisBatch]] = {} # mutable-ok: lazily filled per backend self._post_call_settlements: tuple[PostCallSettlement, ...] = () + self._post_call_unsent = False self.post_call_deadline: Final = post_call_deadline self._native_factory: Final = native_factory self._native_send_factory: Final = native_send_factory self._native: NativeRequestRedis | None = None - self._native_context: Final = copy_context() self._native_flush_hooks: Final[list[Callable[[], None]]] = [] # mutable-ok: append-only registry self._native_misses: frozenset[str] = frozenset() - self._deadline: asyncio.TimerHandle | None = None - self._deadline_flush: asyncio.Task[None] | None = None # Reads declared early for a consumer that runs later in the request, keyed by consumer name. self.prefetched: Final[dict[str, object]] = {} # mutable-ok: armed pre-admission, taken at use - def batch(self, redis_cache: RedisCache) -> RedisBatch: - key: Final = _backend_key(redis_cache) - batch = self._batches.get(key) - if batch is None: - batch = RedisBatch(redis_cache, name="request_redis_batch") - self._batches[key] = batch - return batch - - def post_call(self, redis_cache: RedisCache) -> RedisBatch: - key: Final = _backend_key(redis_cache) - existing: Final = self._post_call.get(key) - batch: Final = ( - existing - if existing is not None - else self._post_call.setdefault(key, RedisBatch(redis_cache, name="post_call_redis_batch")) - ) - if self._deadline is None: - self._deadline = asyncio.get_running_loop().call_later(self.post_call_deadline, self._flush_on_deadline) - _open_post_call.add(self) - return batch - def native(self) -> NativeRequestRedis: - """The request's native batches, built on first use so requests that touch no native store pay nothing. - - They keep a copy of the context they are built in, and the cycle collector cannot see through native - objects, so they are built in the context this scope was opened from, which cannot reach the scope.""" + """The request's native batches, built on first use so requests that touch no native store pay nothing.""" if self._native is None: - self._native = self._native_context.run(self._native_factory, self.post_call_deadline) + self._native = self._native_factory(self.post_call_deadline) return self._native def native_post_call(self) -> NativeRequestRedis: """``native`` for a post-call declaration, so a shutdown drain also flushes it.""" _open_post_call.add(self) + self._post_call_unsent = True return self.native() def native_send(self, phase: RequestPhase) -> NativeSend: @@ -612,7 +148,7 @@ class RequestRedisBatches: self._native_flush_hooks.append(hook) def note_native_misses(self, keys: Sequence[str]) -> None: - """Keys a native pre-call read found absent (or a native DEL removed), like ``RedisBatch.read_as_missing``.""" + """Keys a native pre-call read found absent, or a native DEL removed, so a later GET can skip Redis.""" self._native_misses = self._native_misses | frozenset(keys) def forget_native_miss(self, key: str) -> None: @@ -628,31 +164,21 @@ class RequestRedisBatches: hook() finally: _active_request_batches.reset(token) + await self.flush_pre_call() + + async def flush_pre_call(self) -> None: + """Send the pre-call ops declared so far, such as write-backs nobody awaits before the request scope closes. + The pre-call flush hooks are skipped: they declare reads for a reader about to await, and none is waiting.""" if self._native is not None: await self._native.flush_pre() - def _flush_on_deadline(self) -> None: - self._deadline = None - self._deadline_flush = asyncio.ensure_future(self.flush_post_call()) - - async def flush_all(self) -> None: - """Batch whatever is still declared (write-backs nobody awaits) before the request scope closes. The pre-call - flush hooks are skipped: they declare reads for later readers, and none are left.""" - await asyncio.gather( - *(batch.flush() for batch in self._batches.values() if batch.pending), - *(() if self._native is None else (self._native.flush_pre(),)), - ) - async def flush_post_call(self) -> None: - """One pipeline per backend for the post-call writes; the deadline is disarmed since this is that flush.""" - if self._deadline is not None: - self._deadline.cancel() - self._deadline = None - await asyncio.gather( - *(batch.flush() for batch in self._post_call.values() if batch.pending), - *(() if self._native is None else (self._native.flush_post(),)), - ) - if not any(batch.pending for batch in self._post_call.values()): + """Send the post-call writes. The request stays registered with the shutdown drain until they are back, so a + drain that overlaps this flush waits for it, and an op declared while it runs keeps the request registered.""" + self._post_call_unsent = False + if self._native is not None: + await self._native.flush_post() + if not self._post_call_unsent: _open_post_call.discard(self) def track_post_call_settlement(self, settlement: PostCallSettlement) -> None: @@ -663,36 +189,16 @@ class RequestRedisBatches: def post_call_settlements(self) -> tuple[PostCallSettlement, ...]: return self._post_call_settlements - @property - def batches(self) -> tuple[RedisBatch, ...]: - return tuple(self._batches.values()) - _active_request_batches: Final[ContextVar[RequestRedisBatches | None]] = ContextVar( "request_redis_batches", default=None ) -def active_request_redis_batch(redis_cache: RedisCache) -> RedisBatch | None: - """The request's batch for this backend, or None outside a ``request_redis_batch_scope``.""" - batches: Final = _active_request_batches.get() - if batches is None: - return None - return batches.batch(redis_cache) - - def active_request_redis_batches() -> RequestRedisBatches | None: return _active_request_batches.get() -def active_post_call_redis_batch(redis_cache: RedisCache) -> RedisBatch | None: - """The request's post-call batch for this backend, or None outside a ``request_redis_batch_scope``.""" - batches: Final = _active_request_batches.get() - if batches is None: - return None - return batches.post_call(redis_cache) - - async def flush_post_call_redis_batches() -> None: """Called where the success and failure callbacks of a request have all run.""" batches: Final = _active_request_batches.get() @@ -706,7 +212,7 @@ async def drain_post_call_redis_batches() -> None: class request_redis_batch_scope: - """Redis reads declared inside share one pipeline per backend; nested scopes join the outer one.""" + """Redis ops declared inside share the request's native batches; nested scopes join the outer one.""" __slots__ = ("_native_factory", "_native_send_factory", "_post_call_deadline", "_token") diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 3f047e6001d..620b9de3eb3 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -29,7 +29,6 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast from pydantic import TypeAdapter import litellm -from litellm._internal_context import current_service_caller from litellm._logging import print_verbose, verbose_logger from litellm.constants import ( DEFAULT_REDIS_MAJOR_VERSION, @@ -112,7 +111,6 @@ class _SupportsScriptLoad(Protocol): _GENERIC_CALLER_MODULES: Final = frozenset( { __name__, - "litellm.caching.redis_batch", "litellm.caching.dual_cache", "litellm.caching.caching", "litellm.rust_bridge.execution", @@ -125,8 +123,6 @@ _GENERIC_CALLER_FRAME_NAMES: Final = frozenset( "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync", - "run_alone", - "_settle_alone", "get_cache", "set_cache", "async_get_cache", @@ -184,11 +180,9 @@ def _get_call_stack_info(num_frames: int = 2) -> str: Get the function names of the nearest meaningful callers of the cache method. Frames that merely forward the call (this module's circuit-breaker guards, the - cache facades, the batch pipeline's retry path, generic cache verbs) are - skipped, and the walk stops at the event loop, so the chain names the litellm - code that wanted the call. When nothing but forwarding frames is found (the call - runs in a task of its own, like a batch op retried on the flush) the chain the - declaring code threaded through ``service_caller`` is reported, else ``unknown``. + cache facades, generic cache verbs) are skipped, and the walk stops at the event + loop, so the chain names the litellm code that wanted the call, or ``unknown`` + when nothing but forwarding frames is found. Args: num_frames: Number of previous frames to include (default: 2) @@ -219,9 +213,7 @@ def _get_call_stack_info(num_frames: int = 2) -> str: ) function_names: Final = leading_names + further_names - if function_names: - return " <- ".join(function_names) - return current_service_caller() or "unknown" + return " <- ".join(function_names) if function_names else "unknown" except Exception: return "unknown" diff --git a/litellm/constants.py b/litellm/constants.py index 066b682362d..276bc903680 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -163,6 +163,8 @@ DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COO DEFAULT_REDIS_BATCH_CACHE_EXPIRY_SECONDS: Final = 10 DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5) DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) +DEFAULT_REPLICATE_SYNC_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_SYNC_POLLING_RETRIES", 120) +REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES: Final = get_env_int("REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES", 512 * 1024 * 1024) DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range( @@ -448,6 +450,7 @@ MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body" MCP_ADMISSION_BODY_PEEK_TIMEOUT_SECONDS: Final = 5.0 REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged" REALTIME_METERED_USAGE_KEY: Final = "realtime_metered_usage" +FAILED_PROMPT_TOKEN_ESTIMATE_KEY: Final = "failed_prompt_token_estimate" OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS: Final = 30 OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS: Final = 5 OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset( @@ -493,6 +496,7 @@ REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_team_spend_upda REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_org_spend_update_buffer" REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_spend_update_buffer" REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer" +REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_customer_member_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer" REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer" REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer" @@ -1997,7 +2001,7 @@ BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = get_env_float( "BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", 60.0 ) BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = get_env_float( - "BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 3600.0 + "BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 7800.0 ) _background_interaction_cost_polling_env: Final = os.getenv( "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true" diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a37160b4528..171b731fac8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -101,6 +101,7 @@ from litellm.llms.openai.cost_calculation import ( cost_per_token as openai_cost_per_token, ) from litellm.llms.openai.cost_calculation import ( + video_clip_cost, video_output_cost_per_second, ) from litellm.llms.perplexity.cost_calculator import ( @@ -126,6 +127,7 @@ from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage from litellm.types.extraction import ExtractionResponse, ExtractionUsage +from litellm.types.interactions import InteractionsAPIResponse from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -1144,11 +1146,29 @@ def _served_or_priced_requested_model( if served in (requested, requested.partition("/")[2]): return served pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider) + profile_of_served: Final = _is_bedrock_cross_region_profile_of(requested_model, served_model, pricing_provider) + if profile_of_served and _resolves_to_rates(requested, pricing_provider): + return requested if _resolves_to_rates(served, pricing_provider): return served return requested if _resolves_to_rates(requested, pricing_provider) else served +def _is_bedrock_cross_region_profile_of(profile: str, served_model: str, custom_llm_provider: str | None) -> bool: + if custom_llm_provider != "bedrock": + return False + from litellm.llms.bedrock.common_utils import ( + get_bedrock_cross_region_inference_regions, + strip_bedrock_routing_prefix, + ) + + region, _, profiled_model = strip_bedrock_routing_prefix(profile).partition(".") + return ( + region in get_bedrock_cross_region_inference_regions() + and strip_bedrock_routing_prefix(served_model) == profiled_model + ) + + def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str: """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already @@ -1187,6 +1207,8 @@ def _get_response_model(completion_response: object) -> str | None: if completion_response is None: return None + if isinstance(completion_response, InteractionsAPIResponse): + return completion_response.model or completion_response.agent if isinstance(completion_response, BaseModel): return getattr(completion_response, "model", None) elif isinstance(completion_response, dict): @@ -2070,25 +2092,21 @@ def completion_cost( litellm_logging_obj, custom_pricing, router_model_id ) - usage_obj = getattr(completion_response, "usage", None) + _video_usage_obj: object = attribute_or_none(completion_response, "usage") duration_seconds: float | None = None video_resolution: str | None = None provider_reported_cost: float | None = None - if completion_response is not None and usage_obj: - # Handle both dict and Pydantic Usage object - if isinstance(usage_obj, dict): - duration_seconds = usage_obj.get("duration_seconds", None) - _vr = usage_obj.get("video_resolution", None) - provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None) - _vc = usage_obj.get("video_count", None) - else: - duration_seconds = getattr(usage_obj, "duration_seconds", None) - _vr = getattr(usage_obj, "video_resolution", None) - provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None) - _vc = getattr(usage_obj, "video_count", None) - if _vr is not None: - video_resolution = str(_vr).strip().lower() - video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1 + if completion_response is not None and _video_usage_obj: + duration_seconds = _VIDEO_USAGE_NUMBER.validate_python( + _video_usage_value(_video_usage_obj, "duration_seconds") + ) + provider_reported_cost = _VIDEO_USAGE_NUMBER.validate_python( + _video_usage_value(_video_usage_obj, "provider_reported_cost_usd") + ) + if ( + video_resolution_value := _video_usage_value(_video_usage_obj, "video_resolution") + ) is not None: + video_resolution = str(video_resolution_value).strip().lower() if _video_model_info is None and provider_reported_cost is not None: return float(provider_reported_cost) @@ -2105,7 +2123,7 @@ def completion_cost( custom_llm_provider=custom_llm_provider, model_info=_video_model_info, video_resolution=video_resolution, - ) * video_count + _video_input_image_cost( + ) * _video_count(_video_usage_obj) + _video_input_image_cost( requested_model or model, custom_llm_provider, _video_model_info, @@ -2118,7 +2136,7 @@ def completion_cost( custom_llm_provider=custom_llm_provider, model_info=_video_model_info, video_resolution=video_resolution, - ) + ) * _video_count(_video_usage_obj) elif call_type in _SPEECH_CALL_TYPES: prompt_characters = _speech_billed_characters( completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, @@ -2631,6 +2649,19 @@ def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) +_VIDEO_USAGE_NUMBER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) + + +def _video_usage_value(usage: object, field: str) -> object: + fields: Final = as_str_mapping(usage) + return fields.get(field) if fields is not None else attribute_or_none(usage, field) + + +def _video_count(usage: object) -> int: + count: Final = _video_usage_value(usage, "video_count") + return count if type(count) is int and count > 1 else 1 + + def _video_input_image_cost( model: str, custom_llm_provider: str | None, @@ -3122,6 +3153,9 @@ def default_image_cost_calculator( return cost +_VIDEO_COST_INFO: Final = TypeAdapter(Mapping[str, object]) + + def default_video_cost_calculator( model: str, duration_seconds: float, @@ -3184,6 +3218,14 @@ def default_video_cost_calculator( if cost_info is None: raise Exception(f"Model not found in cost map for model={model}") + clip_cost: Final = video_clip_cost(_VIDEO_COST_INFO.validate_python(cost_info), duration_seconds, video_resolution) + if clip_cost is not None: + return clip_cost + + video_cost: Final = cost_info.get("output_cost_per_video") + if video_cost is not None: + return video_cost + # Check for video-specific cost per second first video_cost_per_second: Final = cost_info.get("output_cost_per_video_per_second") if video_cost_per_second is not None: diff --git a/litellm/deploy/docker/Dockerfile b/litellm/deploy/docker/Dockerfile index 28161792dea..dadc3ef59bb 100644 --- a/litellm/deploy/docker/Dockerfile +++ b/litellm/deploy/docker/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:ae9575b121f5776e2f04e192064bc2c5665af9d1c425d620dc773aca31698712 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:ae9575b121f5776e2f04e192064bc2c5665af9d1c425d620dc773aca31698712 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/litellm/deploy/helm/litellm/templates/ingress.yaml b/litellm/deploy/helm/litellm/templates/ingress.yaml index 6c3670152ca..a7a977c10e3 100644 --- a/litellm/deploy/helm/litellm/templates/ingress.yaml +++ b/litellm/deploy/helm/litellm/templates/ingress.yaml @@ -96,25 +96,31 @@ /v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*, /v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix. */}} -{{- $gatewayPrefixes := list - "/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings" - "/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images" - "/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning" - "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads" - "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes" - "/v1/models" "/models" "/openai" "/engines" - "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" "/api/event_logging" - "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag" - "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search" - "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat" - "/v1beta" "/interactions" - "/anthropic" "/azure" "/azure_ai" "/azure_speech" "/aws" "/bedrock" "/comprehendmedical" "/transcribe" "/cohere" "/gemini" "/google" - "/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm" - "/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough" - "/toolset" - "/v1/realtime" "/realtime" - "/health" "/metrics" +{{- $gatewayPrefixGroups := list + (list "/chat" "/v1/chat") (list "/completions" "/v1/completions") (list "/embeddings" "/v1/embeddings") + (list "/moderations" "/v1/moderations") (list "/audio" "/v1/audio") (list "/images" "/v1/images") + (list "/files" "/v1/files") (list "/batches" "/v1/batches") (list "/fine_tuning" "/v1/fine_tuning") + (list "/fine-tuning" "/v1/fine-tuning") (list "/responses" "/v1/responses") (list "/threads" "/v1/threads") + (list "/assistants" "/v1/assistants") (list "/vector_stores" "/v1/vector_stores") (list "/models" "/v1/models") + (list "/messages" "/v1/messages") (list "/a2a" "/v1/a2a") (list "/rerank" "/v1/rerank") + (list "/decisions" "/v1/decisions") (list "/systemone" "/v1/systemone") (list "/extraction" "/v1/extraction") + (list "/ocr" "/v1/ocr") (list "/rag" "/v1/rag") (list "/video" "/v1/video") + (list "/videos" "/v1/videos") (list "/search" "/v1/search") (list "/containers" "/v1/containers") + (list "/realtime" "/v1/realtime") (list "/live/sessions" "/v1/live/sessions") (list "/v1/skills" "/v1/indexes") + (list "/v1/evals" "/v2/rerank") (list "/v1beta" "/v1/memory") (list "/openai" "/engines") + (list "/queue/chat" "/api/event_logging") (list "/toolset" "/interactions") (list "/azure" "/anthropic") + (list "/azure_ai" "/azure_speech") (list "/aws" "/bedrock") (list "/transcribe" "/comprehendmedical") + (list "/cohere" "/gemini") (list "/google" "/gigachat") (list "/vertex_ai" "/vertex-ai") + (list "/assemblyai" "/eu.assemblyai") (list "/fal_ai" "/deepgram") (list "/vllm" "/langfuse") + (list "/mistral" "/typesafe") (list "/openrouter" "/nvidia_nim") (list "/groq" "/voyage") + (list "/cursor" "/milvus") (list "/tinyfish" "/openai_passthrough") (list "/watsonx") + (list "/health" "/metrics") -}} +{{- $gatewayPrefixes := list -}} +{{- range $gatewayPrefixGroups -}} +{{- $gatewayPrefixes = concat $gatewayPrefixes . -}} +{{- end -}} +{{- $gatewayExactPaths := list "/test" "/debug/memory/summary" "/eu_jev/v1/systemone" -}} {{/* /test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its children /test/connection and /test/tools/list are MCP-server management @@ -131,7 +137,16 @@ at "/" Prefix would swallow the whole backend management API) instead of adding to it. */}} -{{- $builtinPathKeys := ternary (list "/|Prefix") (list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix") $monolith -}} +{{- $builtinPathKeys := list "/|Prefix" -}} +{{/* + An ALB allows 100 listener rules and the AWS Load Balancer Controller makes + one per Ingress path, so on alb each $gatewayPrefixGroups entry is one rule. + Its first, shorter prefix is the Ingress path, which keeps any longer child + path ahead of it in the controller's longest-first order, and a conditions + annotation adds the other prefix to the same rule. +*/}} +{{- $albGrouping := and (eq $controller "alb") (not $monolith) -}} +{{- $gatewayConditions := get (.Values.ingress.annotations | default dict) (printf "alb.ingress.kubernetes.io/conditions.%s" $gatewayName) | default "[]" | trim | trimPrefix "[" | trimSuffix "]" | trim -}} {{- if $hasComponentHosts }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -198,9 +213,20 @@ metadata: name: {{ include "litellm.fullname" . }} labels: {{- include "litellm.commonLabels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + {{- if or .Values.ingress.annotations $albGrouping }} annotations: + {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $albGrouping }} + {{- range $i, $group := $gatewayPrefixGroups }} + alb.ingress.kubernetes.io/actions.litellm-gateway-{{ $i }}: {{ printf `{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"%s","servicePort":"http"}]}}` $gatewayName | quote }} + {{- $conditions := compact (list $gatewayConditions (ternary (printf `{"field":"path-pattern","pathPatternConfig":{"values":["%s","%s/*"]}}` (last $group) (last $group)) "" (eq (len $group) 2))) }} + {{- with $conditions }} + alb.ingress.kubernetes.io/conditions.litellm-gateway-{{ $i }}: {{ printf "[%s]" (join "," .) | quote }} + {{- end }} + {{- end }} + {{- end }} {{- end }} spec: {{- with .Values.ingress.className }} @@ -234,21 +260,31 @@ spec: # /test/* MCP management endpoints fall to the backend catch-all. # Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory # gate); the rest of /debug/* stays on the backend. - - path: /test + {{- range $gatewayExactPaths }} + {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Exact" .) }} + - path: {{ . }} pathType: Exact backend: service: name: {{ $gatewayName }} port: number: {{ $gatewayPort }} - - path: /debug/memory/summary - pathType: Exact + {{- end }} + {{- if $albGrouping }} + {{- range $i, $group := $gatewayPrefixGroups }} + {{- range $group }} + {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }} + {{- end }} + - path: {{ first $group }} + pathType: Prefix backend: service: - name: {{ $gatewayName }} + name: litellm-gateway-{{ $i }} port: - number: {{ $gatewayPort }} - {{- range $gatewayPrefixes }} + name: use-annotation + {{- end }} + {{- end }} + {{- range (ternary list $gatewayPrefixes $albGrouping) }} {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }} {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }} - path: {{ . }} diff --git a/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml b/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml index aa30db3c9c1..1cd39c91546 100644 --- a/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml +++ b/litellm/deploy/helm/litellm/tests/ingress_controller_tests.yaml @@ -18,16 +18,8 @@ tests: name: RELEASE-NAME-litellm-ui port: number: 3000 - - contains: - path: spec.rules[0].http.paths - content: - path: /eu.assemblyai - pathType: Prefix - backend: - service: - name: RELEASE-NAME-litellm-gateway - port: - number: 4000 + - notExists: + path: spec.rules[0].http.paths[?(@.backend.service.port.name == "use-annotation" && @.pathType != "Prefix")] - contains: path: spec.rules[0].http.paths content: @@ -39,6 +31,39 @@ tests: port: number: 3000 + - it: fits the AWS Load Balancer Controller's one rule per path under the default 100 rules per ALB + set: + ingress.enabled: true + asserts: + - notExists: + path: spec.rules[0].http.paths[100] + + - it: routes each grouped gateway prefix to the gateway Service through a use-annotation action on alb + set: + ingress.enabled: true + asserts: + - equal: + path: metadata.annotations["alb.ingress.kubernetes.io/actions.litellm-gateway-0"] + value: '{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"RELEASE-NAME-litellm-gateway","servicePort":"http"}]}}' + - matchRegex: + path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"] + pattern: '^\[\{"field":"path-pattern","pathPatternConfig":\{"values":\["(/[^"*]+)","/[^"*]+/\*"\]\}\}\]$' + - notExists: + path: spec.rules[0].http.paths[?(@.pathType == "Prefix" && @.backend.service.name == "RELEASE-NAME-litellm-gateway")] + + - it: keeps an operator's conditions for the gateway Service on every grouped alb rule + set: + ingress.enabled: true + ingress.annotations: + alb.ingress.kubernetes.io/conditions.RELEASE-NAME-litellm-gateway: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]' + asserts: + - matchRegex: + path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-0"] + pattern: '^\[\{"field":"source-ip","sourceIpConfig":\{"values":\["10\.0\.0\.0/8"\]\}\},\{"field":"path-pattern",' + - equal: + path: metadata.annotations["alb.ingress.kubernetes.io/conditions.litellm-gateway-50"] + value: '[{"field":"source-ip","sourceIpConfig":{"values":["10.0.0.0/8"]}}]' + - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them set: ingress.enabled: true @@ -107,6 +132,26 @@ tests: name: RELEASE-NAME-litellm-gateway port: number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /eu_jev/v1/systemone + pathType: Exact + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - contains: + path: spec.rules[0].http.paths + content: + path: /openrouter + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 - equal: path: spec.rules[0].http.paths[-1] value: diff --git a/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml b/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml index dee46528d04..dd2a853323d 100644 --- a/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml +++ b/litellm/deploy/helm/litellm/tests/ingress_extra_paths_tests.yaml @@ -7,6 +7,7 @@ tests: - it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset set: ingress.enabled: true + ingress.controller: nginx asserts: - equal: path: spec.rules[0].http.paths[-1] @@ -33,12 +34,12 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm asserts: - equal: path: spec.rules[0].http.paths[-2] value: - path: /watsonx + path: /hosted_vllm pathType: Prefix backend: service: @@ -59,8 +60,9 @@ tests: - it: keeps every built-in path when extra paths are supplied set: ingress.enabled: true + ingress.controller: nginx ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm asserts: - contains: path: spec.rules[0].http.paths @@ -117,7 +119,7 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm service: gateway - path: /my-passthrough pathType: Exact @@ -129,7 +131,7 @@ tests: - equal: path: spec.rules[0].http.paths[-4] value: - path: /watsonx + path: /hosted_vllm pathType: Prefix backend: service: @@ -164,7 +166,7 @@ tests: backend.service.port: 8001 ui.service.port: 8080 ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm - path: /my-passthrough service: backend - path: /brand.txt @@ -184,21 +186,21 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm service: proxy asserts: - failedTemplate: - errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui' + errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown service "proxy", expected one of backend, gateway, ui' - it: rejects an entry whose pathType is not a kubernetes pathType set: ingress.enabled: true ingress.extraPaths: - - path: /watsonx + - path: /hosted_vllm pathType: prefix asserts: - failedTemplate: - errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix' + errorMessage: 'ingress.extraPaths[0] (path /hosted_vllm): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix' - it: rejects an entry with no path set: @@ -322,7 +324,7 @@ tests: set: ingress.enabled: true ingress.extraPaths: - - /watsonx + - /hosted_vllm asserts: - failedTemplate: errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key" @@ -372,7 +374,7 @@ tests: - path: /moyai serviceName: moyai-ingress servicePort: http - - path: /watsonx + - path: /hosted_vllm service: gateway asserts: - equal: diff --git a/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml b/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml index 08f7c598260..b3ace127ceb 100644 --- a/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml +++ b/litellm/deploy/helm/litellm/tests/ingress_monolith_tests.yaml @@ -41,6 +41,7 @@ tests: - it: componentized mode still splits traffic across ui, gateway and backend Services set: ingress.enabled: true + ingress.controller: nginx asserts: - equal: path: spec.rules[0].http.paths[0].backend.service.name diff --git a/litellm/deploy/helm/litellm/values.yaml b/litellm/deploy/helm/litellm/values.yaml index b4795cd9ca6..c5f63edf4d2 100644 --- a/litellm/deploy/helm/litellm/values.yaml +++ b/litellm/deploy/helm/litellm/values.yaml @@ -102,7 +102,7 @@ ingress: # The target component only answers paths its own route allowlist keeps, so # a path here still has to be one that component serves. extraPaths: [] - # - path: /watsonx + # - path: /hosted_vllm # pathType: Prefix # service: gateway diff --git a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py index a0c896b7e05..589e3e27c47 100644 --- a/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py +++ b/litellm/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py @@ -1,326 +1,334 @@ -""" -PagerDuty Alerting Integration - -Handles two types of alerts: -- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert. -- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert. - -Note: This is a Free feature on the regular litellm docker image. - -However, this is under the enterprise license -""" - -import asyncio -import os -from datetime import datetime, timedelta, timezone -from typing import List, Optional, Union - -from litellm._logging import verbose_logger -from litellm.caching import DualCache -from litellm.exceptions import MissingMessageRoleError -from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting -from litellm.llms.custom_httpx.http_handler import ( - AsyncHTTPHandler, - get_async_httpx_client, - httpxSpecialProvider, -) -from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.integrations.pagerduty import ( - AlertingConfig, - PagerDutyInternalEvent, - PagerDutyPayload, - PagerDutyRequestBody, -) -from litellm.types.utils import ( - CallTypesLiteral, - StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, -) - -PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60 -PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60 -PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60 -PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600 - - -class PagerDutyAlerting(SlackAlerting): - """ - Tracks failed requests and hanging requests separately. - If threshold is crossed for either type, triggers a PagerDuty alert. - """ - - def __init__( - self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs - ): - super().__init__() - _api_key = os.getenv("PAGERDUTY_API_KEY") - if not _api_key: - raise ValueError("PAGERDUTY_API_KEY is not set") - - self.api_key: str = _api_key - alerting_args = alerting_args or {} - self.pagerduty_alerting_args: AlertingConfig = AlertingConfig( - failure_threshold=alerting_args.get( - "failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD - ), - failure_threshold_window_seconds=alerting_args.get( - "failure_threshold_window_seconds", - PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS, - ), - hanging_threshold_seconds=alerting_args.get( - "hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS - ), - hanging_threshold_window_seconds=alerting_args.get( - "hanging_threshold_window_seconds", - PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS, - ), - ) - - # Separate storage for failures vs. hangs - self._failure_events: List[PagerDutyInternalEvent] = [] - self._hanging_events: List[PagerDutyInternalEvent] = [] - - # ------------------ MAIN LOGIC ------------------ # - - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - """ - Record a failure event. Only send an alert to PagerDuty if the - configured *failure* threshold is exceeded in the specified window. - """ - if isinstance(kwargs.get("exception"), MissingMessageRoleError): - return - now = datetime.now(timezone.utc) - standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( - "standard_logging_object" - ) - if not standard_logging_payload: - raise ValueError( - "standard_logging_object is required for PagerDutyAlerting" - ) - - # Extract error details - error_info: Optional[StandardLoggingPayloadErrorInformation] = ( - standard_logging_payload.get("error_information") or {} - ) - _meta = standard_logging_payload.get("metadata") or {} - - self._failure_events.append( - PagerDutyInternalEvent( - failure_event_type="failed_response", - timestamp=now, - error_class=error_info.get("error_class"), - error_code=error_info.get("error_code"), - error_llm_provider=error_info.get("llm_provider"), - user_api_key_hash=_meta.get("user_api_key_hash"), - user_api_key_alias=_meta.get("user_api_key_alias"), - user_api_key_spend=_meta.get("user_api_key_spend"), - user_api_key_max_budget=_meta.get("user_api_key_max_budget"), - user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"), - user_api_key_user_spend=_meta.get("user_api_key_user_spend"), - user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"), - user_api_key_team_spend=_meta.get("user_api_key_team_spend"), - user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"), - user_api_key_org_id=_meta.get("user_api_key_org_id"), - user_api_key_org_alias=_meta.get("user_api_key_org_alias"), - user_api_key_team_id=_meta.get("user_api_key_team_id"), - user_api_key_project_id=_meta.get("user_api_key_project_id"), - user_api_key_project_alias=_meta.get("user_api_key_project_alias"), - user_api_key_user_id=_meta.get("user_api_key_user_id"), - user_api_key_team_alias=_meta.get("user_api_key_team_alias"), - user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"), - user_api_key_user_email=_meta.get("user_api_key_user_email"), - user_api_key_request_route=_meta.get("user_api_key_request_route"), - user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"), - ) - ) - - # Prune + Possibly alert - window_seconds = self.pagerduty_alerting_args.get( - "failure_threshold_window_seconds", 60 - ) - threshold = self.pagerduty_alerting_args.get("failure_threshold", 1) - - # If threshold is crossed, send PD alert for failures - await self._send_alert_if_thresholds_crossed( - events=self._failure_events, - window_seconds=window_seconds, - threshold=threshold, - alert_prefix="High LLM API Failure Rate", - ) - - async def async_pre_call_hook( - self, - user_api_key_dict: UserAPIKeyAuth, - cache: DualCache, - data: dict, - call_type: CallTypesLiteral, - ) -> Optional[Union[Exception, str, dict]]: - """ - Example of detecting hanging requests by waiting a given threshold. - If the request didn't finish by then, we treat it as 'hanging'. - """ - verbose_logger.info("Inside Proxy Logging Pre-call hook!") - asyncio.create_task( - self.hanging_response_handler( - request_data=data, user_api_key_dict=user_api_key_dict - ) - ) - return None - - async def hanging_response_handler( - self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth - ): - """ - Checks if request completed by the time 'hanging_threshold_seconds' elapses. - If not, we classify it as a hanging request. - """ - verbose_logger.debug( - f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds" - ) - await asyncio.sleep( - self.pagerduty_alerting_args.get( - "hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS - ) - ) - - if await self._request_is_completed(request_data=request_data): - return # It's not hanging if completed - - # Otherwise, record it as hanging - self._hanging_events.append( - PagerDutyInternalEvent( - failure_event_type="hanging_response", - timestamp=datetime.now(timezone.utc), - error_class="HangingRequest", - error_code="HangingRequest", - error_llm_provider="HangingRequest", - user_api_key_hash=user_api_key_dict.api_key, - user_api_key_alias=user_api_key_dict.key_alias, - user_api_key_spend=user_api_key_dict.spend, - user_api_key_max_budget=user_api_key_dict.max_budget, - user_api_key_budget_reset_at=( - user_api_key_dict.budget_reset_at.isoformat() - if user_api_key_dict.budget_reset_at - else None - ), - user_api_key_user_spend=user_api_key_dict.user_spend, - user_api_key_user_max_budget=user_api_key_dict.user_max_budget, - user_api_key_team_spend=user_api_key_dict.team_spend, - user_api_key_team_max_budget=user_api_key_dict.team_max_budget, - user_api_key_org_id=user_api_key_dict.org_id, - user_api_key_org_alias=user_api_key_dict.organization_alias, - user_api_key_team_id=user_api_key_dict.team_id, - user_api_key_project_id=user_api_key_dict.project_id, - user_api_key_project_alias=user_api_key_dict.project_alias, - user_api_key_user_id=user_api_key_dict.user_id, - user_api_key_team_alias=user_api_key_dict.team_alias, - user_api_key_end_user_id=user_api_key_dict.end_user_id, - user_api_key_user_email=user_api_key_dict.user_email, - user_api_key_request_route=user_api_key_dict.request_route, - user_api_key_auth_metadata=user_api_key_dict.metadata, - ) - ) - - # Prune + Possibly alert - window_seconds = self.pagerduty_alerting_args.get( - "hanging_threshold_window_seconds", - PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS, - ) - threshold: int = self.pagerduty_alerting_args.get( - "hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS - ) - - # If threshold is crossed, send PD alert for hangs - await self._send_alert_if_thresholds_crossed( - events=self._hanging_events, - window_seconds=window_seconds, - threshold=threshold, - alert_prefix="High Number of Hanging LLM Requests", - ) - - # ------------------ HELPERS ------------------ # - - async def _send_alert_if_thresholds_crossed( - self, - events: List[PagerDutyInternalEvent], - window_seconds: int, - threshold: int, - alert_prefix: str, - ): - """ - 1. Prune old events - 2. If threshold is reached, build alert, send to PagerDuty - 3. Clear those events - """ - cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds) - pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff] - - # Update the reference list - events.clear() - events.extend(pruned) - - # Check threshold - verbose_logger.debug( - f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}" - ) - if len(events) >= threshold: - # Build short summary of last N events - error_summaries = self._build_error_summaries(events, max_errors=5) - alert_message = ( - f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds." - ) - custom_details = {"recent_errors": error_summaries} - - await self.send_alert_to_pagerduty( - alert_message=alert_message, - custom_details=custom_details, - ) - - # Clear them after sending an alert, so we don't spam - events.clear() - - def _build_error_summaries( - self, events: List[PagerDutyInternalEvent], max_errors: int = 5 - ) -> List[PagerDutyInternalEvent]: - """ - Build short text summaries for the last `max_errors`. - Example: "ValueError (code: 500, provider: openai)" - """ - recent = events[-max_errors:] - summaries = [] - for fe in recent: - # If any of these is None, show "N/A" to avoid messing up the summary string - fe.pop("timestamp") - summaries.append(fe) - return summaries - - async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict): - """ - Send [critical] Alert to PagerDuty - - https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api - """ - try: - verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}") - async_client: AsyncHTTPHandler = get_async_httpx_client( - llm_provider=httpxSpecialProvider.LoggingCallback - ) - payload: PagerDutyRequestBody = PagerDutyRequestBody( - payload=PagerDutyPayload( - summary=alert_message, - severity="critical", - source="LiteLLM Alert", - component="LiteLLM", - custom_details=custom_details, - ), - routing_key=self.api_key, - event_action="trigger", - ) - - return await async_client.post( - url="https://events.pagerduty.com/v2/enqueue", - json=dict(payload), - headers={"Content-Type": "application/json"}, - ) - except Exception as e: - verbose_logger.exception(f"Error sending alert to PagerDuty: {e}") +""" +PagerDuty Alerting Integration + +Handles two types of alerts: +- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert. +- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert. + +Note: This is a Free feature on the regular litellm docker image. + +However, this is under the enterprise license +""" + +import asyncio +import os +from datetime import datetime, timedelta, timezone +from typing import Final, Literal + +from litellm._logging import verbose_logger +from litellm.caching import DualCache +from litellm.exceptions import MissingMessageRoleError +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.integrations.pagerduty import ( + AlertingConfig, + PagerDutyInternalEvent, + PagerDutyPayload, + PagerDutyRequestBody, +) +from litellm.types.utils import ( + CallTypesLiteral, + StandardLoggingMetadata, + StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, +) + +PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60 +PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60 +PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60 +PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600 + + +def _metadata_of(payload: StandardLoggingPayload) -> StandardLoggingMetadata: + return payload["metadata"] + + +_CustomerIdKey = Literal[ + "user_api_key_customer_member_id", "user_api_key_customer_team_id", "user_api_key_customer_org_id" +] + + +def _customer_id(metadata: StandardLoggingMetadata, key: _CustomerIdKey) -> str | None: + return metadata[key] if key in metadata else None + + +class PagerDutyAlerting(SlackAlerting): + """ + Tracks failed requests and hanging requests separately. + If threshold is crossed for either type, triggers a PagerDuty alert. + """ + + def __init__(self, alerting_args: AlertingConfig | dict | None = None, **kwargs): + super().__init__() + _api_key = os.getenv("PAGERDUTY_API_KEY") + if not _api_key: + raise ValueError("PAGERDUTY_API_KEY is not set") + + self.api_key: str = _api_key + alerting_args = alerting_args or {} + self.pagerduty_alerting_args: AlertingConfig = AlertingConfig( + failure_threshold=alerting_args.get("failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD), + failure_threshold_window_seconds=alerting_args.get( + "failure_threshold_window_seconds", + PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS, + ), + hanging_threshold_seconds=alerting_args.get( + "hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS + ), + hanging_threshold_window_seconds=alerting_args.get( + "hanging_threshold_window_seconds", + PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS, + ), + ) + + # Separate storage for failures vs. hangs + self._failure_events: list[PagerDutyInternalEvent] = [] + self._hanging_events: list[PagerDutyInternalEvent] = [] + + # ------------------ MAIN LOGIC ------------------ # + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + """ + Record a failure event. Only send an alert to PagerDuty if the + configured *failure* threshold is exceeded in the specified window. + """ + if isinstance(kwargs.get("exception"), MissingMessageRoleError): + return + now = datetime.now(timezone.utc) + standard_logging_payload: StandardLoggingPayload | None = kwargs.get("standard_logging_object") + if not standard_logging_payload: + raise ValueError("standard_logging_object is required for PagerDutyAlerting") + + # Extract error details + error_info: StandardLoggingPayloadErrorInformation | None = ( + standard_logging_payload.get("error_information") or {} + ) + _meta: Final = _metadata_of(standard_logging_payload) + + self._failure_events.append( + PagerDutyInternalEvent( + failure_event_type="failed_response", + timestamp=now, + error_class=error_info.get("error_class"), + error_code=error_info.get("error_code"), + error_llm_provider=error_info.get("llm_provider"), + user_api_key_hash=_meta.get("user_api_key_hash"), + user_api_key_alias=_meta.get("user_api_key_alias"), + user_api_key_spend=_meta.get("user_api_key_spend"), + user_api_key_max_budget=_meta.get("user_api_key_max_budget"), + user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"), + user_api_key_user_spend=_meta.get("user_api_key_user_spend"), + user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"), + user_api_key_team_spend=_meta.get("user_api_key_team_spend"), + user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"), + user_api_key_org_id=_meta.get("user_api_key_org_id"), + user_api_key_org_alias=_meta.get("user_api_key_org_alias"), + user_api_key_team_id=_meta.get("user_api_key_team_id"), + user_api_key_project_id=_meta.get("user_api_key_project_id"), + user_api_key_project_alias=_meta.get("user_api_key_project_alias"), + user_api_key_user_id=_meta.get("user_api_key_user_id"), + user_api_key_team_alias=_meta.get("user_api_key_team_alias"), + user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"), + user_api_key_customer_member_id=_customer_id(_meta, "user_api_key_customer_member_id"), + user_api_key_customer_team_id=_customer_id(_meta, "user_api_key_customer_team_id"), + user_api_key_customer_org_id=_customer_id(_meta, "user_api_key_customer_org_id"), + user_api_key_user_email=_meta.get("user_api_key_user_email"), + user_api_key_request_route=_meta.get("user_api_key_request_route"), + user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"), + ) + ) + + # Prune + Possibly alert + window_seconds = self.pagerduty_alerting_args.get("failure_threshold_window_seconds", 60) + threshold = self.pagerduty_alerting_args.get("failure_threshold", 1) + + # If threshold is crossed, send PD alert for failures + await self._send_alert_if_thresholds_crossed( + events=self._failure_events, + window_seconds=window_seconds, + threshold=threshold, + alert_prefix="High LLM API Failure Rate", + ) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: CallTypesLiteral, + ) -> Exception | str | dict | None: + """ + Example of detecting hanging requests by waiting a given threshold. + If the request didn't finish by then, we treat it as 'hanging'. + """ + verbose_logger.info("Inside Proxy Logging Pre-call hook!") + asyncio.create_task(self.hanging_response_handler(request_data=data, user_api_key_dict=user_api_key_dict)) + return None + + async def hanging_response_handler(self, request_data: dict | None, user_api_key_dict: UserAPIKeyAuth): + """ + Checks if request completed by the time 'hanging_threshold_seconds' elapses. + If not, we classify it as a hanging request. + """ + verbose_logger.debug( + f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds" + ) + await asyncio.sleep( + self.pagerduty_alerting_args.get("hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS) + ) + + if await self._request_is_completed(request_data=request_data): + return # It's not hanging if completed + + # Otherwise, record it as hanging + self._hanging_events.append( + PagerDutyInternalEvent( + failure_event_type="hanging_response", + timestamp=datetime.now(timezone.utc), + error_class="HangingRequest", + error_code="HangingRequest", + error_llm_provider="HangingRequest", + user_api_key_hash=user_api_key_dict.api_key, + user_api_key_alias=user_api_key_dict.key_alias, + user_api_key_spend=user_api_key_dict.spend, + user_api_key_max_budget=user_api_key_dict.max_budget, + user_api_key_budget_reset_at=( + user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None + ), + user_api_key_user_spend=user_api_key_dict.user_spend, + user_api_key_user_max_budget=user_api_key_dict.user_max_budget, + user_api_key_team_spend=user_api_key_dict.team_spend, + user_api_key_team_max_budget=user_api_key_dict.team_max_budget, + user_api_key_org_id=user_api_key_dict.org_id, + user_api_key_org_alias=user_api_key_dict.organization_alias, + user_api_key_team_id=user_api_key_dict.team_id, + user_api_key_project_id=user_api_key_dict.project_id, + user_api_key_project_alias=user_api_key_dict.project_alias, + user_api_key_user_id=user_api_key_dict.user_id, + user_api_key_team_alias=user_api_key_dict.team_alias, + user_api_key_end_user_id=user_api_key_dict.end_user_id, + user_api_key_customer_member_id=( + user_api_key_dict.customer_hierarchy.customer_member_id + if user_api_key_dict.customer_hierarchy + else None + ), + user_api_key_customer_team_id=( + user_api_key_dict.customer_hierarchy.customer_team_id + if user_api_key_dict.customer_hierarchy + else None + ), + user_api_key_customer_org_id=( + user_api_key_dict.customer_hierarchy.customer_org_id + if user_api_key_dict.customer_hierarchy + else None + ), + user_api_key_user_email=user_api_key_dict.user_email, + user_api_key_request_route=user_api_key_dict.request_route, + user_api_key_auth_metadata=user_api_key_dict.metadata, + ) + ) + + # Prune + Possibly alert + window_seconds = self.pagerduty_alerting_args.get( + "hanging_threshold_window_seconds", + PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS, + ) + threshold: int = self.pagerduty_alerting_args.get( + "hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS + ) + + # If threshold is crossed, send PD alert for hangs + await self._send_alert_if_thresholds_crossed( + events=self._hanging_events, + window_seconds=window_seconds, + threshold=threshold, + alert_prefix="High Number of Hanging LLM Requests", + ) + + # ------------------ HELPERS ------------------ # + + async def _send_alert_if_thresholds_crossed( + self, + events: list[PagerDutyInternalEvent], + window_seconds: int, + threshold: int, + alert_prefix: str, + ): + """ + 1. Prune old events + 2. If threshold is reached, build alert, send to PagerDuty + 3. Clear those events + """ + cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds) + pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff] + + # Update the reference list + events.clear() + events.extend(pruned) + + # Check threshold + verbose_logger.debug( + f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}" + ) + if len(events) >= threshold: + # Build short summary of last N events + error_summaries = self._build_error_summaries(events, max_errors=5) + alert_message = f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds." + custom_details = {"recent_errors": error_summaries} + + await self.send_alert_to_pagerduty( + alert_message=alert_message, + custom_details=custom_details, + ) + + # Clear them after sending an alert, so we don't spam + events.clear() + + def _build_error_summaries( + self, events: list[PagerDutyInternalEvent], max_errors: int = 5 + ) -> list[PagerDutyInternalEvent]: + """ + Build short text summaries for the last `max_errors`. + Example: "ValueError (code: 500, provider: openai)" + """ + recent = events[-max_errors:] + summaries = [] + for fe in recent: + # If any of these is None, show "N/A" to avoid messing up the summary string + fe.pop("timestamp") + summaries.append(fe) + return summaries + + async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict): + """ + Send [critical] Alert to PagerDuty + + https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api + """ + try: + verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}") + async_client: AsyncHTTPHandler = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + payload: PagerDutyRequestBody = PagerDutyRequestBody( + payload=PagerDutyPayload( + summary=alert_message, + severity="critical", + source="LiteLLM Alert", + component="LiteLLM", + custom_details=custom_details, + ), + routing_key=self.api_key, + event_action="trigger", + ) + + return await async_client.post( + url="https://events.pagerduty.com/v2/enqueue", + json=dict(payload), + headers={"Content-Type": "application/json"}, + ) + except Exception as e: + verbose_logger.exception(f"Error sending alert to PagerDuty: {e}") diff --git a/litellm/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/litellm/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bbb09e75b47..9c594c70634 100644 --- a/litellm/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/litellm/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -82,6 +82,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( TeamProviderDeployment, _is_base64_encoded_unified_file_id, apply_unified_file_ids, + canonical_provider_file_id, decode_model_from_file_id, ensure_batch_response_managed_file_ids, get_batch_id_from_unified_batch_id, @@ -91,6 +92,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( is_litellm_executed_batch, map_raw_file_ids_to_unified, normalize_mime_type_for_provider, + provider_file_id_spellings, resolve_managed_output_file_model_name, validate_file_list_limit, validate_file_list_purpose, @@ -597,14 +599,27 @@ async def _scoped_file_rows( return _SCOPED_FILE_ROWS.validate_python(rows) -def _iter_provider_file_id_pairs( - rows: Sequence[_ManagedFileRow], - requested_provider_file_ids: frozenset[str], -) -> Iterator[tuple[str, str]]: +def _iter_canonical_provider_file_ids(rows: Sequence[_ManagedFileRow]) -> Iterator[tuple[str, str]]: for row in rows: for provider_file_id in row.flat_model_file_ids: - if provider_file_id in requested_provider_file_ids: - yield provider_file_id, row.unified_file_id + yield canonical_provider_file_id(provider_file_id), row.unified_file_id + + +def _iter_requested_provider_file_id_spellings(provider_file_ids: Sequence[str]) -> Iterator[str]: + for provider_file_id in provider_file_ids: + yield from provider_file_id_spellings(provider_file_id) + + +def _iter_provider_file_id_pairs( + rows: Sequence[_ManagedFileRow], + requested_provider_file_ids: Sequence[str], +) -> Iterator[tuple[str, str]]: + unified_by_canonical_id: Final = dict(_iter_canonical_provider_file_ids(rows)) + for requested_provider_file_id in requested_provider_file_ids: + if ( + unified_file_id := unified_by_canonical_id.get(canonical_provider_file_id(requested_provider_file_id)) + ) is not None: + yield requested_provider_file_id, unified_file_id def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedObjectTableActions: @@ -767,12 +782,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) -> None: verbose_logger.info(f"Storing LiteLLM Managed File object with id={file_id} in cache") storage_metadata: Final = _storage_metadata_of(file_object) + flat_model_file_ids: Final = [ + canonical_provider_file_id(provider_file_id) for provider_file_id in model_mappings.values() + ] if file_object is not None: litellm_managed_file_object = LiteLLM_ManagedFileTable( unified_file_id=file_id, file_object=file_object, model_mappings=model_mappings, - flat_model_file_ids=list(model_mappings.values()), + flat_model_file_ids=flat_model_file_ids, created_by=resolve_resource_owner_id(user_api_key_dict), team_id=user_api_key_dict.team_id, updated_by=user_api_key_dict.user_id, @@ -790,14 +808,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): db_data = { "unified_file_id": file_id, "model_mappings": json.dumps(model_mappings), - "flat_model_file_ids": list(model_mappings.values()), + "flat_model_file_ids": flat_model_file_ids, "created_by": resolve_resource_owner_id(user_api_key_dict), "team_id": user_api_key_dict.team_id, "updated_by": user_api_key_dict.user_id, } update_data = { "model_mappings": json.dumps(model_mappings), - "flat_model_file_ids": list(model_mappings.values()), + "flat_model_file_ids": flat_model_file_ids, "updated_by": user_api_key_dict.user_id, } @@ -1094,8 +1112,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ if self.prisma_client is None: return + spellings: Final = list(provider_file_id_spellings(file_id)) managed_file = await _managed_file_table(self.prisma_client).find_first( - where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]} + where={ + "OR": [ + {"unified_file_id": {"in": spellings}}, # bounded-ok: at most four spellings of one file id + {"flat_model_file_ids": {"hasSome": spellings}}, + ] + } ) if managed_file is None: return @@ -1139,7 +1163,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) for file_id in provider_file_ids: model_name = decode_model_from_file_id(file_id) - raw_file_id = get_original_file_id(file_id) + raw_file_id = canonical_provider_file_id(get_original_file_id(file_id)) await self.store_batch_output_file( unified_file_id=file_id, provider_file_id=raw_file_id, @@ -1629,7 +1653,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return MappingProxyType({}) provider_file_ids_list: Final = [ # mutable-ok: Prisma hasSome requires a list - provider_file_id for provider_file_id in unique_provider_file_ids + *dict.fromkeys(_iter_requested_provider_file_id_spellings(unique_provider_file_ids)) ] rows: Final = await _managed_file_table(self.prisma_client).find_many( where={ # mutable-ok: Prisma requires a plain dictionary for where @@ -1641,10 +1665,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) return MappingProxyType( dict( - _iter_provider_file_id_pairs( - rows, - frozenset(unique_provider_file_ids), - ) + _iter_provider_file_id_pairs(rows, unique_provider_file_ids) ) ) diff --git a/litellm/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/litellm/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index b17498a370a..0278e8c8c0c 100644 --- a/litellm/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/litellm/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -43,6 +43,7 @@ from litellm.proxy.management_helpers.model_quota_policy import ( ) from litellm.proxy.management_helpers.utils import ( management_endpoint_wrapper, + validate_scalar_budget_attachment, ) from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy from litellm.repositories.base_repository import record_to_dict @@ -637,6 +638,7 @@ async def new_project( ) # Create budget if not provided + await validate_scalar_budget_attachment(prisma_client, data.budget_id) if data.budget_id is None: data.budget_id = await _create_budget_for_project( data=data, @@ -869,6 +871,7 @@ async def update_project( ) # Prepare update data + await validate_scalar_budget_attachment(prisma_client, data.budget_id) update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"})) update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name diff --git a/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py b/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py index bb3b712d50b..678c5c09cf5 100644 --- a/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py +++ b/litellm/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py @@ -35,6 +35,7 @@ class SendKeyRotatedEmailEvent(WebhookEvent): class EmailEvent(str, enum.Enum): model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT + performance_alerts = "Performance Alerts" virtual_key_created = "Virtual Key Created" new_user_invitation = "New User Invitation" virtual_key_rotated = "Virtual Key Rotated" @@ -60,7 +61,8 @@ class DefaultEmailSettings(LiteLLMBaseModel): settings: dict[EmailEvent, bool] = Field( default_factory=lambda: { - EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders. + EmailEvent.model_deprecations: False, + EmailEvent.performance_alerts: False, EmailEvent.virtual_key_created: True, # On by default EmailEvent.new_user_invitation: True, # On by default EmailEvent.virtual_key_rotated: True, # On by default diff --git a/litellm/enterprise/pyproject.toml b/litellm/enterprise/pyproject.toml index 7a8056e8f15..352aaa5d4dc 100644 --- a/litellm/enterprise/pyproject.toml +++ b/litellm/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.83" +version = "0.1.85" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.83" +version = "0.1.85" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-enterprise==", diff --git a/litellm/images/main.py b/litellm/images/main.py index b2da69bb452..e1ecfc35a02 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -465,6 +465,7 @@ def image_generation( litellm.LlmProviders.AIML, litellm.LlmProviders.GEMINI, litellm.LlmProviders.FAL_AI, + litellm.LlmProviders.REPLICATE, litellm.LlmProviders.STABILITY, litellm.LlmProviders.RUNWAYML, litellm.LlmProviders.VERTEX_AI, @@ -496,6 +497,7 @@ def image_generation( logging_obj=litellm_logging_obj, timeout=timeout, client=client, + _is_async=aimg_generation is True and custom_llm_provider == litellm.LlmProviders.REPLICATE, ) elif custom_llm_provider == "black_forest_labs": # Route to BFL-specific handler (polling required) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index dd611947228..f728d054c0d 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -2,10 +2,11 @@ # Class for sending Slack Alerts # import asyncio import datetime +import json import os import random import time -from collections.abc import Callable +from collections.abc import Awaitable, Callable from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -38,6 +39,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import InvitationLinkRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository @@ -87,6 +89,12 @@ def _proxy_llm_router() -> Router | None: _DAILY_REPORT_TARGET: Final = "daily_report_schedule" +async def _model_deprecations_enabled() -> bool: + from litellm.proxy.proxy_server import prisma_client + + return prisma_client is None or await ConfigRepository(prisma_client, use_writer=True).model_deprecations_enabled() + + class SlackAlerting(CustomBatchLogger): """ Class for sending Slack Alerts @@ -97,12 +105,13 @@ class SlackAlerting(CustomBatchLogger): self, internal_usage_cache: DualCache | None = None, alerting_threshold: float | None = None, # threshold for slow / hanging llm responses (in seconds) - alerting: list | None = [], + alerting: list[str] | None = [], alert_types: list[AlertType] = DEFAULT_ALERT_TYPES, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, # if user wants to separate alerts to diff channels alerting_args={}, default_webhook_url: str | None = None, + model_deprecations_enabled: Callable[[], Awaitable[bool]] = _model_deprecations_enabled, alert_type_config: dict[str, dict] | None = None, async_http_handler: AsyncHTTPHandler | None = None, **kwargs, @@ -120,6 +129,7 @@ class SlackAlerting(CustomBatchLogger): self.is_running = False self.alerting_args = SlackAlertingArgs(**alerting_args) self.default_webhook_url = default_webhook_url + self.model_deprecations_enabled: Final = model_deprecations_enabled self.flush_lock = asyncio.Lock() self.periodic_started = False self._periodic_flush_task: asyncio.Task[None] | None = None @@ -142,7 +152,7 @@ class SlackAlerting(CustomBatchLogger): def update_values( self, - alerting: list | None = None, + alerting: list[str] | None = None, alerting_threshold: float | None = None, alert_types: list[AlertType] | None = None, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, @@ -1114,7 +1124,7 @@ Model Info: The daily lock is claimed only once there is something to say, so an empty pass never blocks a later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking """ - if not self._deprecation_alerts_enabled(): + if not self._deprecation_alerts_enabled() or not await self.model_deprecations_enabled(): return False from litellm.proxy.common_utils.model_deprecation import ( @@ -1194,6 +1204,26 @@ Model Info: continue await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) + def performance_alert_webhooks(self) -> tuple[str, ...]: + if not self.alerting or "slack" not in self.alerting or AlertType.performance_alerts not in self.alert_types: + return () + configured: Final = (self.alert_to_webhook_url or {}).get(AlertType.performance_alerts) + destination: Final = ( + configured + or self.default_webhook_url + or os.getenv("SLACK_WEBHOOK_URL") + or os.getenv("ALERTING_WEBHOOK_URL") + ) + if isinstance(destination, list): + return tuple(dict.fromkeys(destination)) + return (destination,) if destination else () + + async def send_performance_alert_summary(self, webhook: str, message: str) -> bool: + response: Final = await self.async_http_handler.post( + url=webhook, headers={"Content-Type": "application/json"}, data=json.dumps({"text": message}) + ) + return response.status_code == 200 + async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ Sends structured alert to webhook, if set. diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index b582d6b8c4f..16a68bc52e0 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -3,6 +3,8 @@ import re import traceback from collections.abc import AsyncGenerator, Mapping, Sequence +from functools import cache +from itertools import takewhile from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional @@ -36,6 +38,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) + from litellm.types.integrations.customer_budget import CustomerBudgetEvent from litellm.types.mcp import ( MCPPostCallResponseObject, MCPPreCallRequestObject, @@ -66,6 +69,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile( class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class # Class variables or attributes server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset() + wants_failed_prompt_token_estimate: ClassVar[bool] = False enforces_request_content: bool = False """ @@ -196,6 +200,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac async def async_log_audit_log_event(self, audit_log: "StandardAuditLogPayload"): """Called when an audit log is created. Override in subclasses to handle.""" + async def async_on_customer_budget_event(self, event: "CustomerBudgetEvent") -> None: + pass + async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None: """Called once per part of a finished batch's lines, when this logger is listed in batch_line_item_callbacks.""" @@ -1115,3 +1122,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac msg[key] = self._redact_base64(value=val, max_depth=max_depth) filtered_messages.append(msg) return filtered_messages + + +@cache +def subclass_overrides_hook(cls: type, hook_name: str, base: type = CustomLogger) -> bool: + below_base: Final = takewhile(lambda klass: klass is not base, cls.__mro__) + return any(hook_name in vars(klass) for klass in below_base) diff --git a/litellm/integrations/custom_logger_v2/__init__.py b/litellm/integrations/custom_logger_v2/__init__.py new file mode 100644 index 00000000000..91896830abd --- /dev/null +++ b/litellm/integrations/custom_logger_v2/__init__.py @@ -0,0 +1,74 @@ +from litellm.integrations.custom_logger_v2.call import CallContext, CallFacts, CallIds, CallKind, CallTarget, CallTiming +from litellm.integrations.custom_logger_v2.dispatcher import Dispatcher, HookResult, RequestScope +from litellm.integrations.custom_logger_v2.enforcement import ContractViolation, UndeclaredReadError +from litellm.integrations.custom_logger_v2.enums import Activation, Field, FieldTier, Hook, OptOutBy +from litellm.integrations.custom_logger_v2.hooks import ( + HOOK_SPECS, + HookSpec, + InvalidDeclarationError, + Subscription, +) +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on +from litellm.integrations.custom_logger_v2.manifest import CallbackManifest, build_manifest +from litellm.integrations.custom_logger_v2.offpath import InProcessSink, OffPathSink +from litellm.integrations.custom_logger_v2.outcomes import ( + Abort, + InvalidPatchError, + JsonValue, + MergeField, + Outcome, + Patch, + Reroute, + Respond, + SetField, +) +from litellm.integrations.custom_logger_v2.patches import apply_patch, apply_patches +from litellm.integrations.custom_logger_v2.planner import HookPlan, plan_hook +from litellm.integrations.custom_logger_v2.registry import CallbackRegistry +from litellm.integrations.custom_logger_v2.selection import RequestSelection, Selection, is_selected + +__all__ = [ + "HOOK_SPECS", + "Abort", + "Activation", + "CallContext", + "CallFacts", + "CallIds", + "CallKind", + "CallTarget", + "CallTiming", + "CallbackManifest", + "CallbackRegistry", + "ContractViolation", + "CustomLoggerV2", + "Dispatcher", + "Field", + "FieldTier", + "Hook", + "HookPlan", + "HookResult", + "HookSpec", + "InProcessSink", + "InvalidDeclarationError", + "InvalidPatchError", + "JsonValue", + "MergeField", + "OffPathSink", + "OptOutBy", + "Outcome", + "Patch", + "RequestScope", + "RequestSelection", + "Reroute", + "Respond", + "Selection", + "SetField", + "Subscription", + "UndeclaredReadError", + "apply_patch", + "apply_patches", + "build_manifest", + "is_selected", + "on", + "plan_hook", +] diff --git a/litellm/integrations/custom_logger_v2/call.py b/litellm/integrations/custom_logger_v2/call.py new file mode 100644 index 00000000000..0d96a86faa3 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/call.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from typing import Final, Generic, TypeVar + +from litellm.integrations.custom_logger_v2.enums import Hook + +StateT: Final = TypeVar("StateT") + + +@dataclass(frozen=True, slots=True) +class CallIds: + call_id: str + trace_id: str | None = None + session_id: str | None = None + parent_call_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class CallKind: + call_type: str + route: str | None = None + stream: bool = False + + +@dataclass(frozen=True, slots=True) +class CallTarget: + model: str + model_group: str | None = None + provider: str | None = None + api_base: str | None = None + deployment_id: str | None = None + attempt: int | None = None + + +@dataclass(frozen=True, slots=True) +class CallTiming: + start: float + provider_start: float | None = None + first_token: float | None = None + end: float | None = None + + +@dataclass(frozen=True, slots=True) +class CallFacts: + ids: CallIds + kind: CallKind + target: CallTarget + timing: CallTiming + cache_hit: bool | None = None + + +@dataclass(frozen=True, slots=True) +class CallContext(Generic[StateT]): + hook: Hook + call: CallFacts | None + state: StateT diff --git a/litellm/integrations/custom_logger_v2/dispatcher.py b/litellm/integrations/custom_logger_v2/dispatcher.py new file mode 100644 index 00000000000..058a30efe9e --- /dev/null +++ b/litellm/integrations/custom_logger_v2/dispatcher.py @@ -0,0 +1,400 @@ +import asyncio +import inspect +import logging +from collections import Counter +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from operator import itemgetter +from types import TracebackType +from typing import Final, Protocol, TypeAlias, TypeVar + +from typing_extensions import Self + +from litellm.integrations.custom_logger_v2.call import CallContext, CallFacts +from litellm.integrations.custom_logger_v2.enforcement import ContractViolation, Enforced, Policy, Unchecked +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS +from litellm.integrations.custom_logger_v2.manifest import LegacyBinding +from litellm.integrations.custom_logger_v2.offpath import InProcessSink, OffPathSink +from litellm.integrations.custom_logger_v2.outcomes import Abort, Outcome, Patch, Reroute, Respond +from litellm.integrations.custom_logger_v2.patches import apply_patch, apply_patches +from litellm.integrations.custom_logger_v2.planner import EMPTY_PLAN, Entry, HookPlan +from litellm.integrations.custom_logger_v2.registry import CallbackRegistry, RegistrySnapshot +from litellm.integrations.custom_logger_v2.selection import RequestSelection, is_selected + +LegacyT: Final = TypeVar("LegacyT") + +Between: TypeAlias = Callable[[Mapping[Field, object]], Awaitable[Mapping[Field, object]]] +SyncBinding: TypeAlias = Callable[[object, object], object] + +SWALLOWING_HOOKS: Final = frozenset({Hook.response_headers, Hook.post_call_failure}) + +_LOGGER: Final = logging.getLogger("LiteLLM") + + +class _Handler(Protocol): + def __call__(self, view: object, ctx: CallContext[object], /) -> object: ... + + +@dataclass(frozen=True, slots=True) +class HookResult: + fields: Mapping[Field, object] + patches: tuple[Patch, ...] = () + outcome: Outcome | None = None + + +@dataclass(frozen=True, slots=True) +class _Chain: + fields: Mapping[Field, object] + patches: tuple[tuple[int, Patch], ...] = () + reroute: Reroute | None = None + terminal: Abort | Respond | None = None + + @property + def ordered_patches(self) -> tuple[Patch, ...]: + return tuple(patch for _, patch in sorted(self.patches, key=itemgetter(0))) + + +def _handler(logger: object, name: str) -> _Handler: + found: Final[object] = getattr(logger, name, None) + if not callable(found): + raise ContractViolation(f"{name} is not callable") + return found + + +async def _awaited(returned: object) -> object: + return await returned if inspect.isawaitable(returned) else returned + + +def _decides(hook: Hook, result: object) -> bool: + match result: + case Abort() | Respond(): + return True + case ContractViolation(): + return True + case Exception(): + return hook not in SWALLOWING_HOOKS + case BaseException(): + return True + case _: + return False + + +def _fatal(error: BaseException) -> bool: + return isinstance(error, ContractViolation) or not isinstance(error, Exception) + + +def _split( + stages: tuple[tuple[Entry, ...], ...], +) -> tuple[tuple[tuple[Entry, ...], ...], tuple[tuple[Entry, ...], ...]]: + if stages and stages[-1][0].subscription.parallel: + return stages[:-1], stages[-1:] + return stages, () + + +def _legacy_start( + entry: Entry, binding: Callable[[object, object], LegacyT] | None, fields: Mapping[Field, object] +) -> LegacyT | None: + call: Final = entry.manifest.legacy_call + if call is None or binding is None: + raise ContractViolation( + f"{entry.manifest.selection.name} is a legacy entry, so the hook point must pass a binding" + ) + return call(binding, fields) + + +async def _run_legacy(entry: Entry, binding: LegacyBinding | None, fields: Mapping[Field, object]) -> object: + started: Final = _legacy_start(entry, binding, fields) + return None if started is None else await started + + +async def _gather_started(started: tuple[Awaitable[object] | None, ...], fail_fast: bool) -> tuple[object, ...]: + pending: Final = tuple(run for run in started if run is not None) + if not pending: + return tuple(None for _ in started) + done: Final = iter( + await asyncio.gather(*pending) if fail_fast else await asyncio.gather(*pending, return_exceptions=True) + ) + return tuple(None if run is None else next(done) for run in started) + + +def _fail_fast(hook: Hook, stage: tuple[Entry, ...]) -> bool: + return HOOK_SPECS[hook].fail_fast_parallel and all(entry.subscription.parallel for entry in stage) + + +async def run_legacy_plan( + hook: Hook, + plan: HookPlan, + fields: Mapping[Field, object], + binding: LegacyBinding, + between: Between | None = None, +) -> Mapping[Field, object]: + serial, parallel = _split(plan.stages) + current: Mapping[Field, object] = fields # rebind-ok: each legacy entry may replace the kwargs + for stage in serial: + for entry in stage: + result = await _run_legacy(entry, binding, current) + if isinstance(result, Patch): + current = apply_patch(current, result) + if between is not None: + current = await between(current) + for stage in parallel: + started = tuple(_legacy_start(entry, binding, current) for entry in stage) + results = await _gather_started(started, _fail_fast(hook, stage)) + failure = next((result for result in results if isinstance(result, BaseException)), None) + if failure is not None: + raise failure + for entry in plan.inline: + _ = await _run_legacy(entry, binding, current) + return current + + +def run_legacy_plan_sync( + plan: HookPlan, fields: Mapping[Field, object], binding: SyncBinding +) -> Mapping[Field, object]: + staged: Final = tuple(chain.from_iterable(plan.stages)) + if any(entry.subscription.parallel for entry in staged): + raise ContractViolation("a synchronous hook cannot run parallel legacy entries") + current: Mapping[Field, object] = fields # rebind-ok: each legacy entry may replace the kwargs + for entry in (*staged, *plan.inline): + result = _legacy_start(entry, binding, current) + if isinstance(result, Patch): + current = apply_patch(current, result) + return current + + +class Dispatcher: + def __init__( + self, + registry: CallbackRegistry, + sink: OffPathSink | None = None, + enforce: bool = False, + ) -> None: + self.registry: Final = registry + self.sink: Final[OffPathSink] = sink or InProcessSink() + self.policy: Final[Policy] = Enforced() if enforce else Unchecked() + self.failures: Final[Counter[tuple[str, Hook]]] = Counter() # mutable-ok: process-wide failure counts + + def open_scope(self, selection: RequestSelection) -> "RequestScope": + return RequestScope(self, self.registry.snapshot, selection) + + def record_failure(self, entry: Entry, hook: Hook, error: BaseException) -> None: + self.failures[(entry.manifest.selection.name, hook)] += 1 + _LOGGER.warning("custom logger v2 %s failed on %s: %r", entry.manifest.selection.name, hook.value, error) + + +class RequestScope: + def __init__(self, dispatcher: Dispatcher, snapshot: RegistrySnapshot, selection: RequestSelection) -> None: + self._dispatcher: Final = dispatcher + self._snapshot: Final = snapshot + self._selection: Final = selection + self._states: dict[int, object] = {} # mutable-ok: request_state instances are created lazily per logger + self.reroute: Reroute | None = None + + def __enter__(self) -> Self: + return self + + def __exit__( + self, kind: type[BaseException] | None, error: BaseException | None, trace: TracebackType | None + ) -> None: + self.close() + + def close(self) -> None: + self._states.clear() + + def selects(self, entry: Entry) -> bool: + return is_selected(entry.manifest.selection, entry.activation, self._selection) + + def plan(self, hook: Hook) -> HookPlan: + entries: Final = self._snapshot.entries[hook] + if not entries: + return EMPTY_PLAN + selected: Final = tuple(index for index, entry in enumerate(entries) if self.selects(entry)) + return self._snapshot.plan(hook, selected) if selected else EMPTY_PLAN + + def reads(self, hook: Hook) -> frozenset[Field]: + return self.plan(hook).reads + + async def run_hook( + self, + hook: Hook, + fields: Mapping[Field, object], + call: CallFacts | None, + ) -> HookResult: + plan: Final = self.plan(hook) + if plan is EMPTY_PLAN: + return HookResult(fields=fields) + chain: Final = await self._run_stages(hook, plan.stages, _Chain(fields=fields), call, None) + patches: Final = chain.ordered_patches + if chain.terminal is not None: + return HookResult(fields=chain.fields, patches=patches, outcome=chain.terminal) + await self._run_awaited(hook, plan.awaited, chain.fields, call) + if plan.off_path: + self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, chain.fields, call)) + if chain.reroute is not None: + self.reroute = chain.reroute + return HookResult(fields=chain.fields, patches=patches, outcome=chain.reroute) + + async def run_plan( + self, + hook: Hook, + plan: HookPlan, + fields: Mapping[Field, object], + call: CallFacts | None, + binding: LegacyBinding, + between: Between | None = None, + ) -> HookResult: + serial, parallel = _split(plan.stages) + chain: Final = await self._run_stages(hook, serial, _Chain(fields=fields), call, binding) + if chain.terminal is not None: + return HookResult(fields=chain.fields, patches=chain.ordered_patches, outcome=chain.terminal) + joined: Final = ( + chain + if between is None + else _Chain(fields=await between(chain.fields), patches=chain.patches, reroute=chain.reroute) + ) + done: Final = await self._run_stages(hook, parallel, joined, call, binding) + if done.terminal is not None: + return HookResult(fields=done.fields, patches=done.ordered_patches, outcome=done.terminal) + for entry in plan.inline: + _ = await _run_legacy(entry, binding, done.fields) + await self._run_awaited(hook, plan.awaited, done.fields, call) + if plan.off_path: + self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, done.fields, call)) + if done.reroute is not None: + self.reroute = done.reroute + return HookResult(fields=done.fields, patches=done.ordered_patches, outcome=done.reroute) + + def run_plan_sync( + self, + hook: Hook, + plan: HookPlan, + fields: Mapping[Field, object], + call: CallFacts | None, + binding: SyncBinding, + ) -> Mapping[Field, object]: + v2_entries: Final = tuple(entry for entry in chain.from_iterable(plan.stages) if not entry.subscription.legacy) + if v2_entries or plan.awaited: + raise ContractViolation( + f"{hook.value} runs synchronously here, so only off-path V2 callbacks may subscribe" + ) + result: Final = run_legacy_plan_sync(plan, fields, binding) + if plan.off_path: + self._dispatcher.sink.submit(self._deliver_off_path(hook, plan.off_path, result, call)) + return result + + def _state(self, entry: Entry) -> object: + factory: Final = entry.manifest.request_state + if factory is None: + return None + key: Final = id(entry.manifest.logger) + if key not in self._states: + self._states[key] = factory() + return self._states[key] + + async def _invoke( + self, hook: Hook, entry: Entry, fields: Mapping[Field, object], call: CallFacts | None, state: object + ) -> object: + policy: Final = self._dispatcher.policy + handler: Final = _handler(entry.manifest.logger, entry.subscription.handler) + context: Final = CallContext[object](hook=hook, call=call, state=state) + view: Final = policy.view(entry.subscription, fields) + result: Final = ( + await _awaited(handler(view, context)) + if inspect.iscoroutinefunction(handler) + else await asyncio.to_thread(handler, view, context) + ) + policy.check(entry.subscription, result) + return result + + def _start( + self, + hook: Hook, + entry: Entry, + fields: Mapping[Field, object], + call: CallFacts | None, + live: bool, + binding: LegacyBinding | None, + ) -> Awaitable[object] | None: + if entry.subscription.legacy: + return _legacy_start(entry, binding, fields) + return self._invoke(hook, entry, fields, call, self._state(entry) if live else None) + + async def _gather( + self, + hook: Hook, + entries: Sequence[Entry], + fields: Mapping[Field, object], + call: CallFacts | None, + live: bool, + binding: LegacyBinding | None = None, + ) -> tuple[object, ...]: + if len(entries) == 1 and entries[0].subscription.legacy: + return (await _run_legacy(entries[0], binding, fields),) + started: Final = tuple(self._start(hook, entry, fields, call, live, binding) for entry in entries) + return await _gather_started(started, _fail_fast(hook, tuple(entries))) + + async def _run_stages( + self, + hook: Hook, + stages: tuple[tuple[Entry, ...], ...], + chain: _Chain, + call: CallFacts | None, + binding: LegacyBinding | None, + ) -> _Chain: + if not stages: + return chain + stage: Final = stages[0] + results: Final = await self._gather(hook, stage, chain.fields, call, live=True, binding=binding) + folded: Final = self._fold(hook, stage, results, chain) + if folded.terminal is not None: + return folded + return await self._run_stages(hook, stages[1:], folded, call, binding) + + def _fold(self, hook: Hook, stage: tuple[Entry, ...], results: tuple[object, ...], chain: _Chain) -> _Chain: + pairs: Final = tuple(zip(stage, results, strict=True)) + decisive: Final = next((pair for pair in pairs if _decides(hook, pair[1])), None) + self._count_failures(hook, tuple(pair for pair in pairs if pair is not decisive)) + match decisive: + case (_, BaseException() as error): + raise error + case (_, Abort() | Respond() as outcome): + return _Chain(fields=chain.fields, patches=chain.patches, reroute=chain.reroute, terminal=outcome) + case _: + return self._merge(pairs, chain) + + def _merge(self, pairs: tuple[tuple[Entry, object], ...], chain: _Chain) -> _Chain: + stage_patches: Final = tuple((entry.position, result) for entry, result in pairs if isinstance(result, Patch)) + reroutes: Final = tuple(result for _, result in pairs if isinstance(result, Reroute)) + return _Chain( + fields=apply_patches(chain.fields, (patch for _, patch in stage_patches)), + patches=(*chain.patches, *stage_patches), + reroute=chain.reroute or next(iter(reroutes), None), + ) + + def _count_failures(self, hook: Hook, pairs: Sequence[tuple[Entry, object]]) -> None: + for entry, result in pairs: + if isinstance(result, BaseException) and not entry.subscription.legacy: + self._dispatcher.record_failure(entry, hook, result) + + def _swallow(self, hook: Hook, entries: Sequence[Entry], results: tuple[object, ...]) -> None: + pairs: Final = tuple(zip(entries, results, strict=True)) + fatal: Final = next( + (result for _, result in pairs if isinstance(result, BaseException) and _fatal(result)), + None, + ) + self._count_failures(hook, tuple(pair for pair in pairs if pair[1] is not fatal)) + if fatal is not None: + raise fatal + + async def _run_awaited( + self, hook: Hook, entries: tuple[Entry, ...], fields: Mapping[Field, object], call: CallFacts | None + ) -> None: + if entries: + self._swallow(hook, entries, await self._gather(hook, entries, fields, call, live=True)) + + async def _deliver_off_path( + self, hook: Hook, entries: tuple[Entry, ...], fields: Mapping[Field, object], call: CallFacts | None + ) -> None: + self._swallow(hook, entries, await self._gather(hook, entries, fields, call, live=False)) diff --git a/litellm/integrations/custom_logger_v2/enforcement.py b/litellm/integrations/custom_logger_v2/enforcement.py new file mode 100644 index 00000000000..8e1c709c741 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/enforcement.py @@ -0,0 +1,95 @@ +from collections.abc import Mapping +from typing import Final, Protocol + +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS, Subscription +from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch, Reroute, Respond + + +class ContractViolation(Exception): + pass + + +class UndeclaredReadError(ContractViolation, AttributeError): + pass + + +def _field_named(name: str) -> Field | None: + member: Final = Field._value2member_map_.get(name) + return member if isinstance(member, Field) else None + + +class FieldView: + __slots__ = ("_fields",) + + def __init__(self, fields: Mapping[Field, object]) -> None: + self._fields = fields + + def __getattr__(self, name: str) -> object: + field: Final = _field_named(name) + if field is None or field not in self._fields: + raise AttributeError(name) + return self._fields[field] + + +class DeclaredView: + __slots__ = ("_fields", "_reads") + + def __init__(self, fields: Mapping[Field, object], reads: frozenset[Field]) -> None: + self._fields = fields + self._reads = reads + + def __getattr__(self, name: str) -> object: + field: Final = _field_named(name) + if field is None or field not in self._reads: + raise UndeclaredReadError(f"{name} is not among the declared reads") + if field not in self._fields: + raise AttributeError(name) + return self._fields[field] + + +class Policy(Protocol): + def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object: ... + + def check(self, subscription: Subscription, result: object) -> None: ... + + +class Unchecked: + def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object: + return FieldView(fields) + + def check(self, subscription: Subscription, result: object) -> None: + return None + + +def _outcome_problem(hook: Hook, subscription: Subscription, outcome: Abort | Respond | Reroute) -> str | None: + if not subscription.may_abort: + return f"returned {type(outcome).__name__} without may_abort" + if type(outcome) not in HOOK_SPECS[hook].outcomes: + return f"returned {type(outcome).__name__}, which {hook.value} does not allow" + return None + + +def _result_problem(subscription: Subscription, result: object) -> str | None: + match result: + case None: + return None + case Patch(): + undeclared: Final = result.fields - subscription.writes + if not undeclared: + return None + return f"patched {', '.join(sorted(field.value for field in undeclared))} outside its declared writes" + case Abort() | Respond() | Reroute(): + return _outcome_problem(subscription.hook, subscription, result) + case _: + return f"returned {type(result).__name__}, which is neither None, a Patch nor an outcome" + + +class Enforced: + def view(self, subscription: Subscription, fields: Mapping[Field, object]) -> object: + return DeclaredView(fields, subscription.reads) + + def check(self, subscription: Subscription, result: object) -> None: + problem: Final = _result_problem(subscription, result) + if problem is not None: + raise ContractViolation(f"{subscription.handler} on {subscription.hook.value} {problem}") diff --git a/litellm/integrations/custom_logger_v2/enums.py b/litellm/integrations/custom_logger_v2/enums.py new file mode 100644 index 00000000000..fcc63ecc4db --- /dev/null +++ b/litellm/integrations/custom_logger_v2/enums.py @@ -0,0 +1,97 @@ +from enum import Enum, IntEnum +from typing import Final + + +class Hook(str, Enum): + prompt_resolve = "prompt_resolve" + pre_route = "pre_route" + filter_deployments = "filter_deployments" + pre_call = "pre_call" + during_call = "during_call" + pre_deployment_call = "pre_deployment_call" + post_deployment_success = "post_deployment_success" + post_deployment_failure = "post_deployment_failure" + post_call_success = "post_call_success" + post_call_failure = "post_call_failure" + response_headers = "response_headers" + stream_chunk = "stream_chunk" + list_models = "list_models" + mcp_tool_result = "mcp_tool_result" + log_prepare = "log_prepare" + provider_request_sent = "provider_request_sent" + provider_response_received = "provider_response_received" + attempt_ended = "attempt_ended" + call_ended = "call_ended" + router_event = "router_event" + audit_event = "audit_event" + + +class FieldTier(IntEnum): + small_record = 1 + content = 2 + wire = 3 + compatibility = 4 + + +class Field(str, Enum): + _value_: str + bit: int + tier: FieldTier + + def __new__(cls, value: str, bit: int, tier: FieldTier) -> "Field": + member: Final = str.__new__(cls, value) + member._value_ = value + member.bit = bit + member.tier = tier + return member + + model = ("model", 0, FieldTier.small_record) + key_identity = ("key_identity", 1, FieldTier.small_record) + deployment = ("deployment", 2, FieldTier.small_record) + candidate_deployments = ("candidate_deployments", 3, FieldTier.small_record) + request_tags = ("request_tags", 4, FieldTier.small_record) + dynamic_params = ("dynamic_params", 5, FieldTier.small_record) + guardrail_info = ("guardrail_info", 6, FieldTier.small_record) + outcome = ("outcome", 7, FieldTier.small_record) + error = ("error", 8, FieldTier.small_record) + usage = ("usage", 9, FieldTier.small_record) + cost = ("cost", 10, FieldTier.small_record) + metadata = ("metadata", 11, FieldTier.content) + messages = ("messages", 12, FieldTier.content) + input = ("input", 13, FieldTier.content) + tools = ("tools", 14, FieldTier.content) + optional_params = ("optional_params", 15, FieldTier.content) + request_headers = ("request_headers", 16, FieldTier.content) + response = ("response", 17, FieldTier.content) + response_headers = ("response_headers", 18, FieldTier.content) + stream_chunk = ("stream_chunk", 19, FieldTier.content) + assembled_stream = ("assembled_stream", 20, FieldTier.content) + tool_result = ("tool_result", 21, FieldTier.content) + model_list = ("model_list", 22, FieldTier.content) + request_body = ("request_body", 23, FieldTier.wire) + provider_request_body = ("provider_request_body", 24, FieldTier.wire) + provider_response_raw = ("provider_response_raw", 25, FieldTier.wire) + provider_response_headers = ("provider_response_headers", 26, FieldTier.wire) + standard_logging_payload = ("standard_logging_payload", 27, FieldTier.compatibility) + deprecated_kwargs = ("deprecated_kwargs", 28, FieldTier.compatibility) + deprecated_response_obj = ("deprecated_response_obj", 29, FieldTier.compatibility) + + +class Activation(str, Enum): + required = "required" + default_on = "default_on" + opt_in = "opt_in" + + +class OptOutBy(str, Enum): + admin = "admin" + caller = "caller" + + +ALL_FIELDS_MASK: Final = sum(1 << field.bit for field in Field) + + +def field_mask(fields: frozenset[Field]) -> int: + if Field.deprecated_kwargs in fields: + return ALL_FIELDS_MASK + return sum(1 << field.bit for field in fields) diff --git a/litellm/integrations/custom_logger_v2/hook_point_fields.py b/litellm/integrations/custom_logger_v2/hook_point_fields.py new file mode 100644 index 00000000000..67f987df83d --- /dev/null +++ b/litellm/integrations/custom_logger_v2/hook_point_fields.py @@ -0,0 +1,98 @@ +from collections.abc import Iterator, Mapping +from functools import reduce +from typing import Final + +from litellm.integrations.custom_logger_v2.enums import Field +from litellm.integrations.custom_logger_v2.legacy import KwargsFields +from litellm.integrations.custom_logger_v2.outcomes import InvalidPatchError, Patch, PatchOp, SetField +from litellm.integrations.custom_logger_v2.patches import LiveFields + +_CANDIDATE_FIELDS: Final = (Field.deprecated_kwargs, Field.candidate_deployments, Field.model) + + +class CandidateFields(LiveFields): + __slots__ = ("_deployments", "_model") + + def __init__(self, deployments: object, model: str) -> None: + self._deployments: Final = deployments + self._model: Final = model + + @property + def deployments(self) -> object: + return self._deployments + + def __getitem__(self, field: Field) -> object: + if field is Field.deprecated_kwargs: + return self._deployments + if field is Field.model: + return self._model + if field is Field.candidate_deployments and isinstance(self._deployments, list | tuple): + return tuple(self._deployments) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts + raise KeyError(field) + + def __iter__(self) -> Iterator[Field]: + return iter(tuple(field for field in _CANDIDATE_FIELDS if self._has(field))) + + def __len__(self) -> int: + return sum(1 for field in _CANDIDATE_FIELDS if self._has(field)) + + def _has(self, field: Field) -> bool: + return field is not Field.candidate_deployments or isinstance(self._deployments, list | tuple) + + def apply(self, patch: Patch) -> Mapping[Field, object]: + return reduce(_narrow, patch.ops, self) + + +def _narrow(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]: + if not isinstance(fields, CandidateFields) or not isinstance(op, SetField): + raise InvalidPatchError("candidate deployments are replaced whole, never merged") + if op.field is Field.deprecated_kwargs: + return CandidateFields(op.value, str(fields[Field.model])) + if op.field is not Field.candidate_deployments: + raise InvalidPatchError(f"{op.field.value} has no place in filter_deployments") + return CandidateFields(_narrowed(fields.deployments, op.value), str(fields[Field.model])) + + +def _narrowed(current: object, proposed: object) -> list[object]: # mutable-ok: legacy filters take a list + if not isinstance(current, list | tuple) or not isinstance(proposed, list | tuple): + raise InvalidPatchError("candidate_deployments must stay a sequence of the current candidates") + allowed: Final = frozenset(map(id, current)) # pyright: ignore[reportUnknownArgumentType] # untyped deployment dicts + picked: Final = tuple(map(id, proposed)) # pyright: ignore[reportUnknownArgumentType] # see above + if not allowed.issuperset(picked) or len(frozenset(picked)) != len(picked): + raise InvalidPatchError("filter_deployments may only narrow the candidates, never add or repeat one") + return list(proposed) # pyright: ignore[reportUnknownArgumentType] # see above + + +class ResultFields(LiveFields): + __slots__ = ("_request", "_slot", "value") + + def __init__(self, request: object, keys: Mapping[Field, str], slot: Field, value: object) -> None: + self._request: Final = KwargsFields(request, keys) + self._slot: Final = slot + self.value = value + + def __getitem__(self, field: Field) -> object: + if field is self._slot or field is Field.deprecated_response_obj: + return self.value + return self._request[field] + + def __iter__(self) -> Iterator[Field]: + return iter((*self._request, self._slot, Field.deprecated_response_obj)) + + def __len__(self) -> int: + return len(self._request) + 2 + + def apply(self, patch: Patch) -> Mapping[Field, object]: + values: Final = tuple(_slot_value(op, self._slot) for op in patch.ops) + if values: + self.value = values[-1] + return self + + +def _slot_value(op: PatchOp, slot: Field) -> object: + if not isinstance(op, SetField) or op.field not in (slot, Field.deprecated_response_obj): + raise InvalidPatchError(f"{op.field.value} cannot be patched here, only {slot.value}") + return op.value + + +__all__ = ["CandidateFields", "ResultFields"] diff --git a/litellm/integrations/custom_logger_v2/hooks.py b/litellm/integrations/custom_logger_v2/hooks.py new file mode 100644 index 00000000000..b3fe1b18671 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/hooks.py @@ -0,0 +1,211 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook +from litellm.integrations.custom_logger_v2.outcomes import Abort, Reroute, Respond + +LegacyGuardrailEvent: TypeAlias = Literal["pre_call", "during_call", "post_call", "post_mcp_call"] +OutcomeType: TypeAlias = type[Abort] | type[Respond] | type[Reroute] + + +class InvalidDeclarationError(TypeError): + pass + + +@dataclass(frozen=True, slots=True) +class HookSpec: + legacy_methods: tuple[str, ...] + request_path: bool + writes: frozenset[Field] = frozenset() + outcomes: frozenset[OutcomeType] = frozenset() + legacy_guardrail_events: frozenset[LegacyGuardrailEvent] = frozenset() + has_call: bool = True + subscribable: bool = True + fail_fast_parallel: bool = False + + +_NO_OUTCOME: Final[frozenset[OutcomeType]] = frozenset() +_ABORT: Final[frozenset[OutcomeType]] = frozenset({Abort}) +_ABORT_OR_RESPOND: Final[frozenset[OutcomeType]] = frozenset({Abort, Respond}) +_ANY_OUTCOME: Final[frozenset[OutcomeType]] = frozenset({Abort, Respond, Reroute}) + +HOOK_SPECS: Final[Mapping[Hook, HookSpec]] = MappingProxyType( + { + Hook.prompt_resolve: HookSpec( + legacy_methods=("get_chat_completion_prompt", "async_get_chat_completion_prompt"), + request_path=True, + writes=frozenset({Field.messages, Field.optional_params, Field.model}), + outcomes=_ABORT_OR_RESPOND, + ), + Hook.pre_route: HookSpec( + legacy_methods=("async_pre_routing_hook",), + request_path=True, + writes=frozenset({Field.model, Field.messages}), + outcomes=_ABORT_OR_RESPOND, + ), + Hook.filter_deployments: HookSpec( + legacy_methods=("async_filter_deployments",), + request_path=True, + writes=frozenset({Field.candidate_deployments}), + outcomes=_ABORT, + ), + Hook.pre_call: HookSpec( + legacy_methods=("async_pre_call_hook",), + request_path=True, + writes=frozenset( + { + Field.messages, + Field.optional_params, + Field.metadata, + Field.request_headers, + Field.guardrail_info, + Field.tools, + Field.model, + Field.input, + } + ), + outcomes=_ANY_OUTCOME, + legacy_guardrail_events=frozenset({"pre_call"}), + ), + Hook.during_call: HookSpec( + legacy_methods=("async_moderation_hook",), + request_path=True, + outcomes=_ABORT_OR_RESPOND, + legacy_guardrail_events=frozenset({"during_call"}), + fail_fast_parallel=True, + ), + Hook.pre_deployment_call: HookSpec( + legacy_methods=("async_pre_call_deployment_hook", "async_pre_request_hook"), + request_path=True, + writes=frozenset({Field.provider_request_body, Field.request_headers}), + outcomes=_ABORT_OR_RESPOND, + ), + Hook.post_deployment_success: HookSpec( + legacy_methods=("async_post_call_success_deployment_hook", "async_post_call_streaming_deployment_hook"), + request_path=True, + writes=frozenset({Field.response}), + outcomes=_ABORT_OR_RESPOND, + ), + Hook.post_deployment_failure: HookSpec( + legacy_methods=("async_post_call_failure_deployment_hook",), + request_path=True, + writes=frozenset({Field.error}), + outcomes=_ABORT, + ), + Hook.post_call_success: HookSpec( + legacy_methods=("async_post_call_success_hook",), + request_path=True, + writes=frozenset({Field.response, Field.guardrail_info}), + outcomes=_ABORT_OR_RESPOND, + legacy_guardrail_events=frozenset({"post_call"}), + ), + Hook.post_call_failure: HookSpec( + legacy_methods=("async_post_call_failure_hook",), + request_path=True, + writes=frozenset({Field.error}), + outcomes=_ABORT, + ), + Hook.response_headers: HookSpec( + legacy_methods=("async_post_call_response_headers_hook",), + request_path=True, + writes=frozenset({Field.response_headers}), + outcomes=_NO_OUTCOME, + ), + Hook.stream_chunk: HookSpec( + legacy_methods=( + "async_post_call_streaming_hook", + "async_post_call_streaming_iterator_hook", + "log_stream_event", + ), + request_path=True, + writes=frozenset({Field.stream_chunk}), + outcomes=_ABORT, + subscribable=False, + ), + Hook.list_models: HookSpec( + legacy_methods=("async_filter_listed_models",), + request_path=True, + writes=frozenset({Field.model_list}), + outcomes=_ABORT, + has_call=False, + ), + Hook.mcp_tool_result: HookSpec( + legacy_methods=("async_post_mcp_tool_call_hook",), + request_path=True, + writes=frozenset({Field.tool_result}), + outcomes=_ABORT_OR_RESPOND, + legacy_guardrail_events=frozenset({"post_mcp_call"}), + ), + Hook.log_prepare: HookSpec( + legacy_methods=( + "logging_hook", + "async_logging_hook", + "redact_standard_logging_payload_from_model_call_details", + ), + request_path=False, + writes=frozenset({Field.standard_logging_payload}), + ), + Hook.provider_request_sent: HookSpec(legacy_methods=("log_pre_api_call",), request_path=False), + Hook.provider_response_received: HookSpec(legacy_methods=("log_post_api_call",), request_path=False), + Hook.attempt_ended: HookSpec( + legacy_methods=( + "log_success_event", + "log_failure_event", + "async_log_success_event", + "async_log_failure_event", + ), + request_path=False, + ), + Hook.call_ended: HookSpec(legacy_methods=(), request_path=False), + Hook.router_event: HookSpec( + legacy_methods=( + "log_success_fallback_event", + "log_failure_fallback_event", + "log_model_group_rate_limit_error", + ), + request_path=False, + ), + Hook.audit_event: HookSpec( + legacy_methods=("async_log_audit_log_event", "async_log_batch_line_items"), + request_path=False, + has_call=False, + ), + } +) + + +@dataclass(frozen=True, slots=True) +class Subscription: + hook: Hook + handler: str + reads: frozenset[Field] = frozenset() + writes: frozenset[Field] = frozenset() + may_abort: bool = False + parallel: bool = False + awaited: bool = False + activation: Activation | None = None + legacy: bool = False + + def __post_init__(self) -> None: + problem: Final = subscription_problem(self) + if problem is not None: + raise InvalidDeclarationError(f"{self.handler} on {self.hook.value}: {problem}") + + +def subscription_problem(subscription: Subscription) -> str | None: + spec: Final = HOOK_SPECS[subscription.hook] + if not spec.subscribable: + return "this hook point has no V2 subscription support yet" + legacy_writes: Final = frozenset({Field.deprecated_kwargs}) if subscription.legacy else frozenset[Field]() + forbidden: Final = subscription.writes - spec.writes - legacy_writes + if forbidden: + return f"writes {', '.join(sorted(field.value for field in forbidden))}, which this hook point cannot apply" + if subscription.may_abort and not spec.outcomes: + return "may_abort is set, but this hook point allows no outcome" + if subscription.awaited and (subscription.writes or subscription.may_abort): + return "an awaited subscription must neither write nor abort" + if subscription.parallel and (subscription.writes or not subscription.may_abort): + return "parallel is only for subscriptions that may abort and write nothing" + return None diff --git a/litellm/integrations/custom_logger_v2/host.py b/litellm/integrations/custom_logger_v2/host.py new file mode 100644 index 00000000000..efa0f44d026 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/host.py @@ -0,0 +1,62 @@ +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Final + +from litellm.integrations.custom_logger_v2.dispatcher import Dispatcher, RequestScope +from litellm.integrations.custom_logger_v2.outcomes import Outcome, Patch +from litellm.integrations.custom_logger_v2.registry import CallbackRegistry +from litellm.integrations.custom_logger_v2.selection import RequestSelection + +_NO_SELECTION: Final = RequestSelection() +_CURRENT_SCOPE: Final[ContextVar[RequestScope | None]] = ContextVar("custom_logger_v2_scope", default=None) + + +class V2Outcome(Exception): + def __init__(self, outcome: Outcome | None, patches: tuple[Patch, ...]) -> None: + super().__init__(f"a V2 callback ended the hook with {type(outcome).__name__}") + self.outcome: Final = outcome + self.patches: Final = patches + + +class _Host: + def __init__(self) -> None: + self._dispatcher: Dispatcher | None = None + + def dispatcher(self) -> Dispatcher: + if self._dispatcher is None: + self._dispatcher = Dispatcher(CallbackRegistry()) + return self._dispatcher + + def reset(self) -> None: + self._dispatcher = None + + +_HOST: Final = _Host() + + +def get_dispatcher() -> Dispatcher: + return _HOST.dispatcher() + + +def reset_host_for_tests() -> None: + _HOST.reset() + + +def current_scope() -> RequestScope | None: + return _CURRENT_SCOPE.get() + + +@contextmanager +def bind_scope(scope: RequestScope) -> Generator[RequestScope]: + token: Final = _CURRENT_SCOPE.set(scope) + try: + yield scope + finally: + _CURRENT_SCOPE.reset(token) + + +@contextmanager +def request_scope(selection: RequestSelection = _NO_SELECTION) -> Generator[RequestScope]: + with _HOST.dispatcher().open_scope(selection) as scope, bind_scope(scope): + yield scope diff --git a/litellm/integrations/custom_logger_v2/legacy.py b/litellm/integrations/custom_logger_v2/legacy.py new file mode 100644 index 00000000000..242301eb432 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/legacy.py @@ -0,0 +1,434 @@ +import weakref +from collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from enum import Enum +from functools import lru_cache, reduce +from itertools import count +from types import MappingProxyType +from typing import Final, TypeAlias, TypeVar + +from typing_extensions import TypeIs # noqa: TID251 # narrows the untyped legacy kwargs without copying them + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger, subclass_overrides_hook +from litellm.integrations.custom_logger_v2.call import CallFacts +from litellm.integrations.custom_logger_v2.dispatcher import ( + Between, + HookResult, + RequestScope, + run_legacy_plan, + run_legacy_plan_sync, +) +from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook +from litellm.integrations.custom_logger_v2.hooks import HOOK_SPECS, Subscription +from litellm.integrations.custom_logger_v2.host import V2Outcome, current_scope, get_dispatcher +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.manifest import CallbackManifest, LegacyBinding, build_manifest +from litellm.integrations.custom_logger_v2.outcomes import Abort, InvalidPatchError, MergeField, Patch, PatchOp, Respond +from litellm.integrations.custom_logger_v2.patches import LiveFields +from litellm.integrations.custom_logger_v2.planner import Entry, HookPlan, plan_hook +from litellm.integrations.custom_logger_v2.selection import RequestSelection, Selection + +ResultT: Final = TypeVar("ResultT") + +SyncLegacyBinding: TypeAlias = Callable[[object, object], object] + +# These hook points are the only ones where v1 custom logger subclasses are +# checked to see if they run in parallel +PARALLEL_GUARDRAIL_HOOKS: Final = frozenset({Hook.pre_call, Hook.post_call_success}) +LEGACY_READS: Final = frozenset({Field.deprecated_kwargs, Field.deprecated_response_obj}) +PROXY_REQUEST_KEYS: Final[Mapping[Field, str]] = MappingProxyType( + { + Field.model: "model", + Field.messages: "messages", + Field.input: "input", + Field.tools: "tools", + } +) +SUBSCRIBERS_CACHE_SIZE: Final = 128 +PLAN_CACHE_SIZE: Final = 64 + +_SERIALS: Final = count() + + +class Admit(str, Enum): + overriders = "overriders" + every_logger = "every_logger" + every_entry = "every_entry" + + +_ABSENT: Final = object() +_NO_SELECTION: Final = RequestSelection() + + +@lru_cache(maxsize=None) +def legacy_subscription(hook: Hook, parallel: bool) -> Subscription: + spec: Final = HOOK_SPECS[hook] + may_abort: Final = bool(spec.outcomes) + gathered: Final = may_abort and (parallel or hook is Hook.during_call) + return Subscription( + hook=hook, + handler="call", + reads=LEGACY_READS, + writes=frozenset({Field.deprecated_kwargs}) if spec.writes and not gathered else frozenset(), + may_abort=may_abort, + parallel=gathered, + legacy=True, + ) + + +def _subscribes(cls: type, guardrail: bool, hook: Hook) -> bool: + spec: Final = HOOK_SPECS[hook] + if not spec.subscribable: + return False + if guardrail and spec.legacy_guardrail_events: + return True + return any(subclass_overrides_hook(cls, method) for method in spec.legacy_methods) + + +@lru_cache(maxsize=512) +def _derived_subscriptions(cls: type, guardrail: bool, parallel: bool) -> Mapping[Hook, Subscription]: + return MappingProxyType( + { + hook: legacy_subscription(hook, parallel and hook in PARALLEL_GUARDRAIL_HOOKS) + for hook in Hook + if _subscribes(cls, guardrail, hook) + } + ) + + +class LegacyLogger(CustomLoggerV2): + def __init__(self, inner: CustomLogger) -> None: + guardrail: Final = isinstance(inner, CustomGuardrail) + self.parallel = guardrail and bool(getattr(inner, "run_in_parallel", False)) + self.serial: Final = next(_SERIALS) + self._inner: Final = weakref.ref(inner) + self.manifest: Final = CallbackManifest( + logger=self, + selection=Selection(name=type(inner).__name__, activation=Activation.required), + subscriptions=_derived_subscriptions(type(inner), guardrail, self.parallel), + request_state=None, + legacy_call=self.call, + ) + + @property + def inner(self) -> CustomLogger | None: + return self._inner() + + def subscription(self, hook: Hook, admit: Admit) -> Subscription | None: + derived: Final = self.manifest.subscriptions.get(hook) + if derived is not None or admit is Admit.overriders or not HOOK_SPECS[hook].subscribable: + return derived + return legacy_subscription(hook, self.parallel and hook in PARALLEL_GUARDRAIL_HOOKS) + + def call(self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object]) -> ResultT | None: + inner: Final = self._inner() + return None if inner is None else binding(inner, fields[Field.deprecated_kwargs]) + + +_ADAPTERS: Final[dict[int, LegacyLogger]] = {} # mutable-ok: adapters live as long as the logger they wrap + + +def adapter_for(inner: CustomLogger) -> LegacyLogger: + key: Final = id(inner) + cached: Final = _ADAPTERS.get(key) + if cached is not None and cached.inner is inner: + return cached + adapter: Final = LegacyLogger(inner) + _ADAPTERS[key] = adapter + _ = weakref.finalize(inner, _ADAPTERS.pop, key, None) + return adapter + + +def _readable(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: legacy kwargs are an untyped dict + return isinstance(value, Mapping) + + +def _writable( + value: object, +) -> TypeIs[MutableMapping[str, object]]: # guard-ok: legacy kwargs are an untyped dict # mutable-ok: written in place + return isinstance(value, MutableMapping) + + +class OpaqueLegacy(CustomLoggerV2): + """Holds a callback list entry that is not a CustomLogger (a string name or a plain function) at its list + position, for hook points that admit every entry. There is no class to derive subscriptions from, so it + subscribes to the one hook it was built for and runs serially.""" + + def __init__(self, inner: object, hook: Hook) -> None: + self.inner: Final = inner + self.serial: Final = next(_SERIALS) + self.manifest: Final = CallbackManifest( + logger=self, + selection=Selection(name=type(inner).__name__, activation=Activation.required), + subscriptions=MappingProxyType({hook: legacy_subscription(hook, False)}), + request_state=None, + legacy_call=self.call, + ) + + def call(self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object]) -> ResultT | None: + return binding(self.inner, fields[Field.deprecated_kwargs]) + + +class KwargsFields(LiveFields): + """A live Field view over a legacy hook's untyped kwargs dict, so the dispatcher can run over it without copying. + + Two fields cover the whole call: deprecated_kwargs is the dict itself, and deprecated_response_obj is the + response, when the hook point passes one. Patching either one swaps in the new value. + + Every other field maps to one key in the dict through the hook point's key table (PROXY_REQUEST_KEYS or + LOGGING_KEYS), and is present only when that key is. Patching one writes straight into the dict, so later + callbacks and the request see the change.""" + + __slots__ = ("_keys", "_kwargs", "_response") + + def __init__(self, kwargs: object, keys: Mapping[Field, str], response: object = _ABSENT) -> None: + self._kwargs: Final = kwargs + self._keys: Final = keys + self._response: Final = response + + def _key(self, field: Field) -> str | None: + key: Final = self._keys.get(field) + return key if key is not None and _readable(self._kwargs) and key in self._kwargs else None + + def __getitem__(self, field: Field) -> object: + if field is Field.deprecated_kwargs: + return self._kwargs + if field is Field.deprecated_response_obj and self._response is not _ABSENT: + return self._response + key: Final = self._key(field) + if key is None or not _readable(self._kwargs): + raise KeyError(field) + return self._kwargs[key] + + def __iter__(self) -> Iterator[Field]: + return iter(tuple(field for field in Field if self._has(field))) + + def __len__(self) -> int: + return sum(1 for field in Field if self._has(field)) + + def _has(self, field: Field) -> bool: + if field is Field.deprecated_kwargs: + return True + if field is Field.deprecated_response_obj: + return self._response is not _ABSENT + return self._key(field) is not None + + def apply(self, patch: Patch) -> Mapping[Field, object]: + return reduce(_apply_op, patch.ops, self) + + def replaced(self, field: Field, value: object) -> "KwargsFields": + if field is Field.deprecated_kwargs: + return KwargsFields(value, self._keys, self._response) + return KwargsFields(self._kwargs, self._keys, value) + + def write(self, field: Field, value: object) -> None: + target: Final = self._target(field) + target[self._keys[field]] = value + + def merge(self, field: Field, keys: Mapping[str, object]) -> None: + target: Final = self._target(field) + current: Final = target.get(self._keys[field]) + if current is None: + target[self._keys[field]] = dict(keys) + return + if not _writable(current): + raise InvalidPatchError(f"{field.value} holds no mapping here, so keys cannot be merged into it") + current.update(keys) + + def _target(self, field: Field) -> MutableMapping[str, object]: # mutable-ok: legacy kwargs are the live request + if field not in self._keys: + raise InvalidPatchError(f"{field.value} has no place in the legacy kwargs of this hook") + if not _writable(self._kwargs): + raise InvalidPatchError(f"{field.value} cannot be written because the legacy kwargs are not a mapping") + return self._kwargs + + +def _apply_op(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]: + if not isinstance(fields, KwargsFields): + raise InvalidPatchError("legacy kwargs patches need KwargsFields") + if op.field in (Field.deprecated_kwargs, Field.deprecated_response_obj): + if isinstance(op, MergeField): + raise InvalidPatchError(f"{op.field.value} is replaced whole, never merged") + return fields.replaced(op.field, op.value) + if isinstance(op, MergeField): + fields.merge(op.field, op.keys) + return fields + fields.write(op.field, op.value) + return fields + + +def kwargs_patch(before: object, after: object) -> Patch | None: + return None if after is before else Patch().set(Field.deprecated_kwargs, after) + + +@dataclass(frozen=True, slots=True, eq=False) +class HookSubscribers: + hook: Hook + entries: tuple[Entry, ...] + plan: HookPlan + has_v2: bool + plans: Callable[[tuple[int, ...]], HookPlan] + + @property + def legacy(self) -> tuple[object, ...]: + return tuple(inner for entry in self.entries if (inner := _wrapped(entry.manifest.logger)) is not None) + + def plan_for(self, scope: RequestScope) -> HookPlan: + selected: Final = tuple( + index for index, entry in enumerate(self.entries) if entry.subscription.legacy or scope.selects(entry) + ) + return self.plan if len(selected) == len(self.entries) else self.plans(selected) + + +def _wrapped(logger: CustomLoggerV2) -> object: + if isinstance(logger, LegacyLogger | OpaqueLegacy): + return logger.inner + return None + + +def _v2_entry(v2_callback: CustomLoggerV2, hook: Hook) -> tuple[CallbackManifest, Subscription] | None: + manifest: Final = build_manifest(v2_callback) + subscription: Final = manifest.subscriptions.get(hook) + return None if subscription is None else (manifest, subscription) + + +def _legacy_entry( + inner: CustomLogger, hook: Hook, admit: Admit, parallel: bool +) -> tuple[CallbackManifest, Subscription] | None: + adapter: Final = adapter_for(inner) + subscription: Final = adapter.subscription(hook, admit) + if subscription is None: + return None + if subscription.parallel and not parallel: + return adapter.manifest, legacy_subscription(hook, False) + return adapter.manifest, subscription + + +def _opaque_entry(item: object, hook: Hook) -> tuple[CallbackManifest, Subscription] | None: + if not HOOK_SPECS[hook].subscribable: + return None + opaque: Final = OpaqueLegacy(item, hook) + return opaque.manifest, opaque.manifest.subscriptions[hook] + + +def _subscriber( + item: object, hook: Hook, admit: Admit, include_v2: bool, parallel: bool +) -> tuple[CallbackManifest, Subscription] | None: + if isinstance(item, CustomLoggerV2): + return _v2_entry(item, hook) if include_v2 else None + if isinstance(item, CustomLogger): + return _legacy_entry(item, hook, admit, parallel) + return _opaque_entry(item, hook) if admit is Admit.every_entry else None + + +def _build_subscribers( + hook: Hook, callbacks: Sequence[object], admit: Admit, include_v2: bool, parallel: bool +) -> HookSubscribers: + members: Final = tuple( + member for item in callbacks if (member := _subscriber(item, hook, admit, include_v2, parallel)) is not None + ) + entries: Final = tuple( + Entry(position, manifest, subscription) for position, (manifest, subscription) in enumerate(members) + ) + + def subset_plan(selected: tuple[int, ...]) -> HookPlan: + return plan_hook(tuple(entries[index] for index in selected)) + + return HookSubscribers( + hook=hook, + entries=entries, + plan=plan_hook(entries), + has_v2=any(not entry.subscription.legacy for entry in entries), + plans=lru_cache(maxsize=PLAN_CACHE_SIZE)(subset_plan), + ) + + +@dataclass(frozen=True, slots=True, eq=False) +class _CachedSubscribers: + subscribers: HookSubscribers + pinned: tuple[object, ...] + + +_SUBSCRIBERS: Final[dict[tuple[object, ...], _CachedSubscribers]] = {} # mutable-ok: bounded subscribers cache + + +def _pinned(item: object) -> bool: + return not isinstance(item, CustomLogger) or isinstance(item, CustomLoggerV2) + + +def _cache_subscribers(key: tuple[object, ...], subscribers: HookSubscribers, callbacks: Sequence[object]) -> None: + if len(_SUBSCRIBERS) >= SUBSCRIBERS_CACHE_SIZE: + _SUBSCRIBERS.clear() + _SUBSCRIBERS[key] = _CachedSubscribers( + subscribers=subscribers, pinned=tuple(item for item in callbacks if _pinned(item)) + ) + for item in callbacks: + if not _pinned(item): + _ = weakref.finalize(item, _SUBSCRIBERS.pop, key, None) + + +def hook_subscribers( + hook: Hook, + callbacks: Sequence[object], + *, + admit: Admit = Admit.overriders, + include_v2: bool = True, + parallel: bool = True, +) -> HookSubscribers: + key: Final = (hook, admit, include_v2, parallel, *map(id, callbacks)) + cached: Final = _SUBSCRIBERS.get(key) + if cached is not None: + return cached.subscribers + subscribers: Final = _build_subscribers(hook, callbacks, admit, include_v2, parallel) + _cache_subscribers(key, subscribers, callbacks) + return subscribers + + +def clear_subscribers_cache_for_tests() -> None: + _SUBSCRIBERS.clear() + + +async def run_subscribers( + subscribers: HookSubscribers, + fields: Mapping[Field, object], + binding: LegacyBinding, + *, + between: Between | None = None, + call: CallFacts | None = None, + scope: RequestScope | None = None, +) -> HookResult: + if not subscribers.has_v2: + return HookResult(fields=await run_legacy_plan(subscribers.hook, subscribers.plan, fields, binding, between)) + bound: Final = scope or current_scope() + if bound is not None: + return await bound.run_plan(subscribers.hook, subscribers.plan_for(bound), fields, call, binding, between) + with get_dispatcher().open_scope(_NO_SELECTION) as transient: + return await transient.run_plan( + subscribers.hook, subscribers.plan_for(transient), fields, call, binding, between + ) + + +def run_subscribers_sync( + subscribers: HookSubscribers, + fields: Mapping[Field, object], + binding: SyncLegacyBinding, + *, + call: CallFacts | None = None, + scope: RequestScope | None = None, +) -> Mapping[Field, object]: + if not subscribers.has_v2: + return run_legacy_plan_sync(subscribers.plan, fields, binding) + bound: Final = scope or current_scope() + if bound is not None: + return bound.run_plan_sync(subscribers.hook, subscribers.plan_for(bound), fields, call, binding) + with get_dispatcher().open_scope(_NO_SELECTION) as transient: + return transient.run_plan_sync(subscribers.hook, subscribers.plan_for(transient), fields, call, binding) + + +def kwargs_after(result: HookResult) -> object: + match result.outcome: + case Abort() | Respond() as terminal: + raise V2Outcome(terminal, result.patches) + case _: + return result.fields[Field.deprecated_kwargs] diff --git a/litellm/integrations/custom_logger_v2/logger.py b/litellm/integrations/custom_logger_v2/logger.py new file mode 100644 index 00000000000..ce8b9f163e0 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/logger.py @@ -0,0 +1,109 @@ +import inspect +from collections.abc import Callable, Mapping, Sequence, Set +from dataclasses import replace +from types import MappingProxyType +from typing import ClassVar, Final, TypeVar + +from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook +from litellm.integrations.custom_logger_v2.hooks import InvalidDeclarationError, Subscription +from litellm.integrations.custom_logger_v2.selection import Selection + +HandlerT: Final = TypeVar("HandlerT", bound=Callable[..., object]) + +_SUBSCRIPTION_ATTR: Final = "__custom_logger_v2_subscription__" + + +def on( + hook: Hook, + *, + reads: Set[Field] = frozenset(), + writes: Set[Field] = frozenset(), + may_abort: bool = False, + parallel: bool | None = None, + awaited: bool = False, +) -> Callable[[HandlerT], HandlerT]: + def decorate(handler: HandlerT) -> HandlerT: + subscription: Final = Subscription( + hook=hook, + handler=handler.__name__, + reads=frozenset(reads), + writes=frozenset(writes), + may_abort=may_abort, + parallel=(hook is Hook.during_call and may_abort) if parallel is None else parallel, + awaited=awaited, + ) + setattr(handler, _SUBSCRIPTION_ATTR, subscription) + return handler + + return decorate + + +def _declared_subscription(value: object) -> Subscription | None: + marker: Final[object] = getattr(value, _SUBSCRIPTION_ATTR, None) + return marker if isinstance(marker, Subscription) else None + + +def _namespace(cls: type) -> Mapping[str, object]: + return vars(cls) + + +def _declared_in(cls: type) -> Mapping[Hook, Subscription]: + declared: Final = tuple( + replace(subscription, handler=name) + for name, value in _namespace(cls).items() + if (subscription := _declared_subscription(value)) is not None + ) + hooks: Final = tuple(subscription.hook for subscription in declared) + duplicated: Final = sorted({hook.value for hook in hooks if hooks.count(hook) > 1}) + if duplicated: + raise InvalidDeclarationError(f"{cls.__name__} declares more than one handler for {', '.join(duplicated)}") + return MappingProxyType({subscription.hook: subscription for subscription in declared}) + + +def _nearest(layers: Sequence[Mapping[Hook, Subscription]], hook: Hook) -> Subscription | None: + return next((layer[hook] for layer in layers if hook in layer), None) + + +def _handler_problem(cls: type, subscription: Subscription) -> str | None: + handler: Final[object] = getattr(cls, subscription.handler, None) + if not callable(handler): + return f"{subscription.handler} handles {subscription.hook.value} but is not callable" + redeclared: Final = _declared_subscription(handler) + if redeclared is not None and redeclared.hook is not subscription.hook: + return ( + f"{subscription.handler} is decorated for {redeclared.hook.value} but also inherits " + f"the {subscription.hook.value} subscription" + ) + blocking: Final = bool(subscription.writes) or subscription.may_abort or subscription.awaited + if blocking and not inspect.iscoroutinefunction(handler): + return f"{subscription.handler} must be async because it writes, may abort or is awaited" + return None + + +class CustomLoggerV2: + _own_subscriptions: ClassVar[Mapping[Hook, Subscription]] = MappingProxyType({}) + subscriptions: ClassVar[Mapping[Hook, Subscription]] = MappingProxyType({}) + request_state: ClassVar[type | None] = None + declared_hooks: ClassVar[frozenset[Hook]] = frozenset() + + selection: Selection | None = None + enabled_hooks: frozenset[Hook] | None = None + parallel: bool = False + activation_overrides: Mapping[Hook, Activation] = MappingProxyType({}) + + def __init_subclass__(cls) -> None: + super().__init_subclass__() + cls._own_subscriptions = _declared_in(cls) + layers: Final = tuple(base._own_subscriptions for base in cls.__mro__ if issubclass(base, CustomLoggerV2)) + resolved: Final = MappingProxyType( + {hook: subscription for hook in Hook if (subscription := _nearest(layers, hook)) is not None} + ) + problems: Final = tuple( + problem + for subscription in resolved.values() + if (problem := _handler_problem(cls, subscription)) is not None + ) + if problems: + raise InvalidDeclarationError(f"{cls.__name__}: {'; '.join(problems)}") + cls.subscriptions = resolved + CustomLoggerV2.declared_hooks = CustomLoggerV2.declared_hooks | resolved.keys() diff --git a/litellm/integrations/custom_logger_v2/manifest.py b/litellm/integrations/custom_logger_v2/manifest.py new file mode 100644 index 00000000000..5984628feeb --- /dev/null +++ b/litellm/integrations/custom_logger_v2/manifest.py @@ -0,0 +1,73 @@ +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Final, Protocol, TypeAlias, TypeVar + +from litellm.integrations.custom_logger_v2.enums import Activation, Field, Hook, OptOutBy +from litellm.integrations.custom_logger_v2.hooks import InvalidDeclarationError, Subscription +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.selection import Selection + +ResultT: Final = TypeVar("ResultT") + +LegacyBinding: TypeAlias = Callable[[object, object], Awaitable[object] | None] + + +class LegacyCall(Protocol): + def __call__( + self, binding: Callable[[object, object], ResultT], fields: Mapping[Field, object], / + ) -> ResultT | None: ... + + +@dataclass(frozen=True, slots=True) +class CallbackManifest: + logger: CustomLoggerV2 + selection: Selection + subscriptions: Mapping[Hook, Subscription] + request_state: type | None + legacy_call: LegacyCall | None = None + + +def _instance_subscription(subscription: Subscription, parallel: bool, activation: Activation | None) -> Subscription: + return replace( + subscription, + parallel=subscription.parallel or (parallel and subscription.may_abort), + activation=activation, + ) + + +def _caller_can_remove(selection: Selection) -> bool: + return selection.honors_no_log or OptOutBy.caller in selection.opt_out_by + + +def build_manifest(logger: CustomLoggerV2) -> CallbackManifest: + name: Final = type(logger).__name__ + declared: Final = type(logger).subscriptions + enabled: Final = frozenset(declared) if logger.enabled_hooks is None else logger.enabled_hooks + not_implemented: Final = enabled - declared.keys() + if not_implemented: + hooks: Final = ", ".join(sorted(hook.value for hook in not_implemented)) + raise InvalidDeclarationError(f"{name} enables {hooks}, which it does not implement") + stray_overrides: Final = logger.activation_overrides.keys() - enabled + if stray_overrides: + overridden: Final = ", ".join(sorted(hook.value for hook in stray_overrides)) + raise InvalidDeclarationError(f"{name} overrides activation for {overridden}, which it does not enable") + subscriptions: Final = MappingProxyType( + { + hook: _instance_subscription(declared[hook], logger.parallel, logger.activation_overrides.get(hook)) + for hook in Hook + if hook in enabled + } + ) + selection: Final = logger.selection or Selection(name=name) + blocking: Final = any(subscription.writes or subscription.may_abort for subscription in subscriptions.values()) + if blocking and _caller_can_remove(selection): + raise InvalidDeclarationError( + f"{selection.name} writes or may abort, so neither no-log nor a caller opt-out may remove it" + ) + return CallbackManifest( + logger=logger, + selection=selection, + subscriptions=subscriptions, + request_state=type(logger).request_state, + ) diff --git a/litellm/integrations/custom_logger_v2/offpath.py b/litellm/integrations/custom_logger_v2/offpath.py new file mode 100644 index 00000000000..e8f309d4878 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/offpath.py @@ -0,0 +1,27 @@ +import asyncio +from collections.abc import Coroutine +from concurrent.futures import ThreadPoolExecutor +from typing import Final, Protocol + + +class OffPathSink(Protocol): + def submit(self, delivery: Coroutine[object, object, None]) -> None: ... + + +class InProcessSink: + def __init__(self) -> None: + self._pending: set[asyncio.Task[None]] = set() # mutable-ok: strong references keep running tasks alive + self._loopless: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="custom-logger-v2-off-path") + + def submit(self, delivery: Coroutine[object, object, None]) -> None: + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + _ = self._loopless.submit(asyncio.run, delivery) + return + task: Final[asyncio.Task[None]] = loop.create_task(delivery) + self._pending.add(task) + task.add_done_callback(self._pending.discard) + + async def drain(self) -> None: + await asyncio.gather(*self._pending) diff --git a/litellm/integrations/custom_logger_v2/outcomes.py b/litellm/integrations/custom_logger_v2/outcomes.py new file mode 100644 index 00000000000..aed1355096d --- /dev/null +++ b/litellm/integrations/custom_logger_v2/outcomes.py @@ -0,0 +1,106 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import ClassVar, Final, TypeAlias + +from pydantic import BaseModel + +from litellm.integrations.custom_logger_v2.enums import Field + +JsonValue: TypeAlias = "str | int | float | bool | None | Sequence[JsonValue] | Mapping[str, JsonValue]" + +MERGEABLE_FIELDS: Final = frozenset( + { + Field.dynamic_params, + Field.guardrail_info, + Field.metadata, + Field.optional_params, + Field.request_headers, + Field.response_headers, + Field.provider_request_body, + Field.standard_logging_payload, + } +) +OUTCOME_PATCH_FIELDS: Final = frozenset({Field.guardrail_info, Field.metadata}) + + +class InvalidPatchError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class SetField: + field: Field + value: object + + +@dataclass(frozen=True, slots=True) +class MergeField: + field: Field + keys: Mapping[str, object] + + +PatchOp: TypeAlias = SetField | MergeField + + +@dataclass(frozen=True, slots=True) +class Patch: + ops: tuple[PatchOp, ...] = () + + def set(self, field: Field, value: object) -> "Patch": + return Patch((*self.ops, SetField(field, value))) + + def merge(self, field: Field, keys: Mapping[str, object]) -> "Patch": + if field not in MERGEABLE_FIELDS: + raise InvalidPatchError(f"{field.value} is not a mapping field, so it cannot be merged by key") + return Patch((*self.ops, MergeField(field, MappingProxyType(dict(keys))))) + + @property + def fields(self) -> frozenset[Field]: + return frozenset(op.field for op in self.ops) + + +def _require_outcome_patch(patch: Patch | None) -> None: + if patch is None: + return + outside: Final = patch.fields - OUTCOME_PATCH_FIELDS + if outside: + names: Final = ", ".join(sorted(field.value for field in outside)) + raise InvalidPatchError(f"an outcome may only patch guardrail_info and metadata, not {names}") + + +@dataclass(frozen=True, slots=True) +class Abort: + terminal: ClassVar[bool] = True + + status: int + message: str + code: str | None = None + headers: Mapping[str, str] | None = None + body: JsonValue = None + patch: Patch | None = None + + def __post_init__(self) -> None: + _require_outcome_patch(self.patch) + + +@dataclass(frozen=True, slots=True) +class Respond: + terminal: ClassVar[bool] = True + + response: "JsonValue | BaseModel" + status: int = 200 + patch: Patch | None = None + + def __post_init__(self) -> None: + _require_outcome_patch(self.patch) + + +@dataclass(frozen=True, slots=True) +class Reroute: + terminal: ClassVar[bool] = False + + model: str + + +Outcome: TypeAlias = Abort | Respond | Reroute diff --git a/litellm/integrations/custom_logger_v2/patches.py b/litellm/integrations/custom_logger_v2/patches.py new file mode 100644 index 00000000000..b5784295f87 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/patches.py @@ -0,0 +1,70 @@ +from abc import abstractmethod +from collections.abc import Iterable, Iterator, Mapping +from functools import reduce +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm.integrations.custom_logger_v2.enums import Field +from litellm.integrations.custom_logger_v2.outcomes import InvalidPatchError, MergeField, Patch, PatchOp, SetField + +_STRING_KEYED: Final = TypeAdapter(Mapping[str, object]) + + +class LiveFields(Mapping[Field, object]): + __slots__ = () + + @abstractmethod + def apply(self, patch: Patch) -> Mapping[Field, object]: ... + + +class PatchedFields(Mapping[Field, object]): + __slots__ = ("_base", "_changed") + + def __init__(self, base: Mapping[Field, object], changed: Mapping[Field, object]) -> None: + self._base = base + self._changed = changed + + def with_value(self, field: Field, value: object) -> "PatchedFields": + return PatchedFields(self._base, MappingProxyType({**self._changed, field: value})) + + def __getitem__(self, field: Field) -> object: + return self._changed[field] if field in self._changed else self._base[field] + + def __iter__(self) -> Iterator[Field]: + return iter(self._changed.keys() | self._base.keys()) + + def __len__(self) -> int: + return len(self._changed.keys() | self._base.keys()) + + +def _with_value(fields: Mapping[Field, object], field: Field, value: object) -> PatchedFields: + patched: Final = fields if isinstance(fields, PatchedFields) else PatchedFields(fields, MappingProxyType({})) + return patched.with_value(field, value) + + +def _merged(current: object, field: Field, keys: Mapping[str, object]) -> Mapping[str, object]: + if current is None: + return MappingProxyType(dict(keys)) + if not isinstance(current, Mapping): + raise InvalidPatchError(f"{field.value} holds no mapping, so keys cannot be merged into it") + return MappingProxyType({**_STRING_KEYED.validate_python(current), **keys}) + + +def _apply_op(fields: Mapping[Field, object], op: PatchOp) -> Mapping[Field, object]: + match op: + case SetField(field=field, value=value): + return _with_value(fields, field, value) + case MergeField(field=field, keys=keys): + return _with_value(fields, field, _merged(fields.get(field), field, keys)) + + +def apply_patch(fields: Mapping[Field, object], patch: Patch) -> Mapping[Field, object]: + if isinstance(fields, LiveFields): + return fields.apply(patch) + return reduce(_apply_op, patch.ops, fields) + + +def apply_patches(fields: Mapping[Field, object], patches: Iterable[Patch]) -> Mapping[Field, object]: + return reduce(apply_patch, patches, fields) diff --git a/litellm/integrations/custom_logger_v2/planner.py b/litellm/integrations/custom_logger_v2/planner.py new file mode 100644 index 00000000000..bbd2187eca2 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/planner.py @@ -0,0 +1,77 @@ +from collections.abc import Sequence +from dataclasses import dataclass +from functools import reduce +from typing import Final + +from litellm.integrations.custom_logger_v2.enums import Activation, Field, field_mask +from litellm.integrations.custom_logger_v2.hooks import Subscription +from litellm.integrations.custom_logger_v2.manifest import CallbackManifest + + +@dataclass(frozen=True, slots=True, eq=False) +class Entry: + position: int + manifest: CallbackManifest + subscription: Subscription + + @property + def activation(self) -> Activation: + return self.subscription.activation or self.manifest.selection.activation + + +@dataclass(frozen=True, slots=True) +class HookPlan: + stages: tuple[tuple[Entry, ...], ...] = () + inline: tuple[Entry, ...] = () + awaited: tuple[Entry, ...] = () + off_path: tuple[Entry, ...] = () + reads: frozenset[Field] = frozenset() + + +EMPTY_PLAN: Final = HookPlan() + + +def _blocks(subscription: Subscription) -> bool: + return bool(subscription.writes) or subscription.may_abort + + +def _level(placed: tuple[tuple[Subscription, int], ...], subscription: Subscription) -> int: + if subscription.may_abort: + return max((level for _, level in placed), default=-1) + 1 + touched: Final = field_mask(subscription.reads | subscription.writes) + writes: Final = field_mask(subscription.writes) + after_writers: Final = tuple(level + 1 for prior, level in placed if field_mask(prior.writes) & touched) + not_before_readers: Final = tuple(level for prior, level in placed if field_mask(prior.reads) & writes) + return max((*after_writers, *not_before_readers), default=0) + + +def _place( + placed: tuple[tuple[Subscription, int], ...], subscription: Subscription +) -> tuple[tuple[Subscription, int], ...]: + return (*placed, (subscription, _level(placed, subscription))) + + +def _levels(entries: Sequence[Entry]) -> tuple[tuple[Entry, ...], ...]: + placed: Final = reduce(_place, (entry.subscription for entry in entries), ()) + levels: Final = tuple(level for _, level in placed) + return tuple(_at_level(entries, levels, level) for level in range(max(levels, default=-1) + 1)) + + +def _at_level(entries: Sequence[Entry], levels: tuple[int, ...], level: int) -> tuple[Entry, ...]: + return tuple(entry for entry, placed_at in zip(entries, levels, strict=True) if placed_at == level) + + +def plan_hook(entries: Sequence[Entry]) -> HookPlan: + if not entries: + return EMPTY_PLAN + blocking: Final = tuple(entry for entry in entries if _blocks(entry.subscription)) + serial: Final = tuple(entry for entry in blocking if not entry.subscription.parallel) + parallel: Final = tuple(entry for entry in blocking if entry.subscription.parallel) + passive: Final = tuple(entry for entry in entries if not _blocks(entry.subscription)) + return HookPlan( + stages=(*_levels(serial), *((parallel,) if parallel else ())), + inline=tuple(entry for entry in passive if entry.subscription.legacy), + awaited=tuple(entry for entry in passive if entry.subscription.awaited and not entry.subscription.legacy), + off_path=tuple(entry for entry in passive if not (entry.subscription.awaited or entry.subscription.legacy)), + reads=frozenset[Field]().union(*(entry.subscription.reads for entry in entries)), + ) diff --git a/litellm/integrations/custom_logger_v2/registry.py b/litellm/integrations/custom_logger_v2/registry.py new file mode 100644 index 00000000000..ef094005545 --- /dev/null +++ b/litellm/integrations/custom_logger_v2/registry.py @@ -0,0 +1,51 @@ +from collections.abc import Callable, Mapping, Sequence +from functools import lru_cache +from types import MappingProxyType +from typing import Final + +from litellm.integrations.custom_logger_v2.enums import Hook +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.manifest import CallbackManifest +from litellm.integrations.custom_logger_v2.planner import Entry, HookPlan, plan_hook + +DEFAULT_PLAN_CACHE_SIZE: Final = 256 + + +def _hook_entries(manifests: tuple[CallbackManifest, ...], hook: Hook) -> tuple[Entry, ...]: + return tuple( + Entry(position, manifest, manifest.subscriptions[hook]) + for position, manifest in enumerate(manifests) + if hook in manifest.subscriptions + ) + + +class RegistrySnapshot: + __slots__ = ("entries", "manifests", "plan") + + def __init__(self, manifests: tuple[CallbackManifest, ...], plan_cache_size: int) -> None: + self.manifests: Final = manifests + self.entries: Final[Mapping[Hook, tuple[Entry, ...]]] = MappingProxyType( + {hook: _hook_entries(manifests, hook) for hook in Hook} + ) + self.plan: Final[Callable[[Hook, tuple[int, ...]], HookPlan]] = lru_cache(maxsize=plan_cache_size)( + self._build_plan + ) + + def _build_plan(self, hook: Hook, selected: tuple[int, ...]) -> HookPlan: + hook_entries: Final = self.entries[hook] + return plan_hook(tuple(hook_entries[index] for index in selected)) + + +class CallbackRegistry: + def __init__( + self, manifests: Sequence[CallbackManifest] = (), plan_cache_size: int = DEFAULT_PLAN_CACHE_SIZE + ) -> None: + self._plan_cache_size: Final = plan_cache_size + self.snapshot: RegistrySnapshot = RegistrySnapshot(tuple(manifests), plan_cache_size) + + def register(self, manifest: CallbackManifest) -> None: + self.snapshot = RegistrySnapshot((*self.snapshot.manifests, manifest), self._plan_cache_size) + + def unregister(self, logger: CustomLoggerV2) -> None: + kept: Final = tuple(manifest for manifest in self.snapshot.manifests if manifest.logger is not logger) + self.snapshot = RegistrySnapshot(kept, self._plan_cache_size) diff --git a/litellm/integrations/custom_logger_v2/selection.py b/litellm/integrations/custom_logger_v2/selection.py new file mode 100644 index 00000000000..ce80bcf12ee --- /dev/null +++ b/litellm/integrations/custom_logger_v2/selection.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass +from typing import Final + +from litellm.integrations.custom_logger_v2.enums import Activation, OptOutBy + + +@dataclass(frozen=True, slots=True) +class Selection: + name: str + aliases: frozenset[str] = frozenset() + activation: Activation = Activation.default_on + opt_out_by: frozenset[OptOutBy] = frozenset({OptOutBy.admin}) + honors_no_log: bool = False + + +@dataclass(frozen=True, slots=True) +class RequestSelection: + requested: frozenset[str] = frozenset() + admin_opt_outs: frozenset[str] = frozenset() + caller_opt_outs: frozenset[str] = frozenset() + no_log: bool = False + + +def _opted_out(selection: Selection, names: frozenset[str], request: RequestSelection) -> bool: + by_admin: Final = OptOutBy.admin in selection.opt_out_by and not names.isdisjoint(request.admin_opt_outs) + by_caller: Final = OptOutBy.caller in selection.opt_out_by and not names.isdisjoint(request.caller_opt_outs) + return by_admin or by_caller or (request.no_log and selection.honors_no_log) + + +def is_selected(selection: Selection, activation: Activation, request: RequestSelection) -> bool: + if activation is Activation.required: + return True + names: Final = selection.aliases | {selection.name} + if _opted_out(selection, names, request): + return False + return activation is Activation.default_on or not names.isdisjoint(request.requested) diff --git a/litellm/integrations/customer_budget_webhook.py b/litellm/integrations/customer_budget_webhook.py new file mode 100644 index 00000000000..7de75f13461 --- /dev/null +++ b/litellm/integrations/customer_budget_webhook.py @@ -0,0 +1,61 @@ +import os +from collections.abc import Mapping +from typing import Final + +from pydantic import BaseModel, ConfigDict, HttpUrl, TypeAdapter + +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # Legacy params are untyped. + httpxSpecialProvider, +) +from litellm.types.integrations.customer_budget import CustomerBudgetEvent + + +class _Settings(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + endpoint: HttpUrl | None = None + headers: Mapping[str, str] | None = None + + +_URL: Final = TypeAdapter(HttpUrl) +_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +class CustomerBudgetWebhook(CustomLogger): + def __init__( + self, + endpoint: str | None = None, + headers: Mapping[str, str] | None = None, + *, + async_httpx_client: AsyncHTTPHandler | None = None, + ) -> None: + super().__init__() # pyright: ignore[reportUnknownMemberType] # Base logger kwargs are untyped. + self.endpoint: Final = str(_URL.validate_python(endpoint or os.environ.get("CUSTOMER_BUDGET_WEBHOOK_URL"))) + self.headers: Final = { + "Content-Type": "application/json", + **_HEADERS.validate_json(os.environ.get("CUSTOMER_BUDGET_WEBHOOK_HEADERS", "{}")), + **(headers or {}), + } + self.async_httpx_client: Final = ( + async_httpx_client + if async_httpx_client is not None + else get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + ) + + @staticmethod + def from_config(value: object) -> "CustomerBudgetWebhook": + settings: Final = _Settings.model_validate(value or {}) + return CustomerBudgetWebhook( + endpoint=str(settings.endpoint) if settings.endpoint is not None else None, + headers=settings.headers, + ) + + async def async_on_customer_budget_event(self, event: CustomerBudgetEvent) -> None: + client: Final = self.async_httpx_client + response: Final = await client.post( # pyright: ignore[reportUnknownMemberType] # Legacy HTTP params are untyped. + url=self.endpoint, headers=dict(self.headers), json=event.model_dump(mode="json"), timeout=10 + ) + response.raise_for_status() diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 0a2921db6c7..ae21e65a860 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -77,6 +77,11 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom prompt_data=prompt_data, prompt_file=prompt_file, prompt_id=registration_prompt_id, + ignore_prompt_manager_optional_params=( + litellm_params.ignore_prompt_manager_optional_params + if hasattr(litellm_params, "ignore_prompt_manager_optional_params") + else False + ), ) return dot_prompt_manager diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index 77d70dd761c..d169f00851f 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -54,9 +54,11 @@ class DotpromptManager(CustomPromptManagement): prompt_file: str | None = None, prompt_data: dict | str | None = None, prompt_id: str | None = None, + ignore_prompt_manager_optional_params: bool | None = False, ): import litellm + self.ignore_prompt_manager_optional_params = ignore_prompt_manager_optional_params self.prompt_directory = prompt_directory or litellm.global_prompt_directory # Support for JSON-based prompts stored in memory/database if isinstance(prompt_data, str): diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 977fa816eed..dc60d18e0ca 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -11,7 +11,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.constants import REDACTED_BY_LITELLM +from litellm.constants import FAILED_PROMPT_TOKEN_ESTIMATE_KEY, REDACTED_BY_LITELLM from litellm.integrations._types.open_inference import ErrorAttributes from litellm.integrations.arize import _utils from litellm.integrations.langfuse.langfuse_otel_attributes import ( @@ -19,8 +19,10 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import ( serialize_langfuse_datetime, ) from litellm.integrations.opentelemetry import ( + ESTIMATED_INPUT_TOKENS_ATTRIBUTE, HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, LITELLM_PROXY_REQUEST_SPAN_NAME, + USAGE_PROVIDER_STATUS_ATTRIBUTE, OpenTelemetry, OpenTelemetryConfig, ) @@ -378,9 +380,15 @@ class LangfuseOtelLogger(OpenTelemetry): limit: Final = span_attribute_limit(span) carried: Final = carried_keys(span) new_keys: Final = tuple(key for key, _value in fields if key not in carried) - room: Final = ( - len(new_keys) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0) + pending_usage: Final[frozenset[str]] = ( + frozenset((USAGE_PROVIDER_STATUS_ATTRIBUTE, ESTIMATED_INPUT_TOKENS_ATTRIBUTE)) + if isinstance(kwargs.get(FAILED_PROMPT_TOKEN_ESTIMATE_KEY), int) + else frozenset((USAGE_PROVIDER_STATUS_ATTRIBUTE,)) + if FAILED_PROMPT_TOKEN_ESTIMATE_KEY in kwargs + else frozenset() ) + reserved: Final = _ERROR_ATTRIBUTES | pending_usage + room: Final = len(new_keys) if limit is None else max(limit - len(carried) - len(reserved - carried), 0) written: Final = carried | frozenset(new_keys[:room]) for key, value in fields: if key in written: diff --git a/litellm/integrations/model_deprecation_email.py b/litellm/integrations/model_deprecation_email.py index 4093a108dcc..11988d8ce57 100644 --- a/litellm/integrations/model_deprecation_email.py +++ b/litellm/integrations/model_deprecation_email.py @@ -214,9 +214,13 @@ class ModelDeprecationEmailSender: return True async def run(self, llm_router: Router, *, now: datetime | None = None) -> int: + if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"): + return 0 + if not await self.config.model_deprecations_enabled(): + return 0 current: Final = now or datetime.now(timezone.utc) settings: Final = await self.config.get_param("general_settings") - if settings is None or not _OptIn.model_validate(settings.param_value).enabled: + if not _OptIn.model_validate(settings.param_value if settings else {}).enabled: return 0 base_url: Final = os.getenv("PROXY_BASE_URL", "") if not base_url: @@ -253,7 +257,7 @@ class ModelDeprecationEmailSender: async def run_scheduled_model_deprecation_emails() -> None: - """Poll the email opt-in independently; a large owner mailing never delays Slack checks.""" + """Poll the email settings independently; a large owner mailing never delays Slack checks.""" from litellm._logging import verbose_proxy_logger while True: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 9bec220e073..829e70b2cb8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -6,14 +6,14 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import TYPE_CHECKING, Any, ClassVar, Final, Protocol, cast from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.constants import KUBERNETES_POD_ROUTING_KEY +from litellm.constants import FAILED_PROMPT_TOKEN_ESTIMATE_KEY, KUBERNETES_POD_ROUTING_KEY from litellm.integrations._types.open_inference import ( OpenInferenceSpanKindValues, ) @@ -33,9 +33,11 @@ from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import LiteLLM, Metric +from litellm.integrations.otel.model.utils import as_str from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.integrations.otel.routing import routed_model_name, routing_decision_attributes from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string @@ -140,6 +142,18 @@ def _response_field(response_obj: object, key: str) -> object: return require_attributes(response_obj, _SupportsGet, "get").get(key) +def _semconv_response_model( + response_obj: object, response_model: object, model_group: str | None, deployment_model: str | None +) -> object: + hidden_params: Final = get_hidden_params(response_obj) + provider_model: Final = hidden_params.get("provider_response_model") if hidden_params is not None else None + if isinstance(provider_model, str) and provider_model: + return provider_model + if response_model and response_model == model_group and deployment_model: + return deployment_model + return response_model + + def _billed_usage(response_obj: object, call_type: str | None, litellm_params: Mapping[str, object] | None) -> object: if not response_obj or is_unbilled_non_inference_call_from_params(call_type, litellm_params, response_obj): return None @@ -152,6 +166,7 @@ class _CacheHitView(TypedDict, total=False): _CACHE_HIT_VIEW: Final = TypeAdapter(_CacheHitView) +_OPAQUE_VALUE: Final = TypeAdapter(object) # Dedicated so a slow exporter shutdown cannot starve the shared logging executor. _PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown") @@ -166,6 +181,8 @@ HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE: Final = "http.response.status_code" HTTP_ROUTE_ATTRIBUTE: Final = "http.route" URL_PATH_ATTRIBUTE: Final = "url.path" PREPROCESSING_DURATION_MS_ATTRIBUTE: Final = "litellm.preprocessing.duration_ms" +USAGE_PROVIDER_STATUS_ATTRIBUTE: Final = "litellm.usage.provider_status" +ESTIMATED_INPUT_TOKENS_ATTRIBUTE: Final = "litellm.usage.estimated_input_tokens" TEAM_METADATA_ATTRIBUTE: Final = "litellm.team.metadata" MODEL_GROUP_ATTRIBUTE: Final = "litellm.model_group" PROVIDER_MODEL_ATTRIBUTE: Final = "litellm.provider.model" @@ -440,6 +457,8 @@ class OpenTelemetryConfig: class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): + wants_failed_prompt_token_estimate: ClassVar[bool] = True + def __init__( self, config: OpenTelemetryConfig | None = None, @@ -2225,6 +2244,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Always create if no parent span exists (backward compatibility) # OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled should_create_primary_span: Final = parent_otel_span is None or get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN") + failure_kwargs: Final = _JSON_OBJECT.validate_python(kwargs) span = None if should_create_primary_span: @@ -2239,6 +2259,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span_kwargs["kind"] = self.span_kind.CLIENT span = otel_tracer.start_span(**span_kwargs) self.set_attributes(span, kwargs, response_obj) + self._set_failure_usage_attributes(span, failure_kwargs) # Record exception information using OTEL standard method span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs) @@ -2252,6 +2273,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Note: parent_otel_span is guaranteed to be not None here if parent_otel_span is not None and parent_otel_span.is_recording(): self.set_attributes(parent_otel_span, kwargs, response_obj) + self._set_failure_usage_attributes(parent_otel_span, failure_kwargs) parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message)) @@ -2271,6 +2293,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ): parent_otel_span.end(end_time=self._to_ns(end_time)) + def _set_failure_usage_attributes(self, span: Span, kwargs: Mapping[str, object]) -> None: + if FAILED_PROMPT_TOKEN_ESTIMATE_KEY not in kwargs: + return + self.safe_set_attribute(span=span, key=USAGE_PROVIDER_STATUS_ATTRIBUTE, value="unknown") + estimate: Final = kwargs[FAILED_PROMPT_TOKEN_ESTIMATE_KEY] + if isinstance(estimate, int): + self.safe_set_attribute(span=span, key=ESTIMATED_INPUT_TOKENS_ATTRIBUTE, value=estimate) + def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None: """ Record exception information on the span using OTEL standard methods. @@ -2521,11 +2551,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ############################################# # The name of the LLM a request is being made to - if kwargs.get("model"): + request_model: Final = as_str(_OPAQUE_VALUE.validate_python(kwargs.get("model"))) + if request_model: self.safe_set_attribute( span=span, key=SpanAttributes.LLM_REQUEST_MODEL.value, - value=kwargs.get("model"), + value=request_model, ) # The LLM request type @@ -2617,10 +2648,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) # The model used to generate the response. - response_model: Final = routed_model_name( - _response_field(response_obj, "model"), - standard_logging_payload.get("model_group"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped - decision, + model_group: Final = as_str(standard_logging_payload.get("model_group")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped + routed_response_model: Final = routed_model_name( + _response_field(response_obj, "model"), model_group, decision + ) + response_model: Final = ( + _semconv_response_model(response_obj, routed_response_model, model_group, request_model) + if self._gen_ai_semconv_latest_experimental + else routed_response_model ) if response_model: self.safe_set_attribute( diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 70384e49f53..69a79419992 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -89,23 +89,13 @@ shows as `redis.incr router_cooldowns` followed by `redis.ttl router_cooldowns` way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue`, `managed_files`, `mcp_servers`, ...), so a job tick renders `redis.set pod_lock` rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every -module under `litellm/` and `enterprise/` that calls a shared cache or declares a -read or write on the request batch (`reserve_redis_batch_reads`, -`batch.mget`, `batch.set`, `batch.script`) and fails when +module under `litellm/` and `enterprise/` that calls a shared cache and fails when one has no declared family, with the process-local `InMemoryCache` callers listed as the only exemptions. A batch op carries the family that was active when it was declared, so the routing prefetch armed before deployment selection (`RoutingPrefetch.arm`) is `router_cooldowns` when only cooldown keys go out, `router_usage` when only usage counters do, and `router_cooldowns_usage` when both -ride the same MGET, whichever pipeline or standalone read later settles it. A -per-request pipeline (`RedisBatch`) that carries ops of -one family is `"redis.pipeline auth_objects"`; one that carries several owners' -ops is `"redis.pipeline mixed"` with the sorted family list on -`litellm.redis.families` and the op count on `litellm.metadata.op_count` (an int, -never stringified). A cluster client cannot pipeline across slots, so there every -batch op settles on its own and one write-back of three auth objects shows as three -parallel `redis.set auth_objects` spans with the same caller, not one pipeline -span. Every `call_type` the Redis cache layer emits maps to a verb, +ride the same MGET, whichever pipeline or standalone read later settles it. Every `call_type` the Redis cache layer emits maps to a verb, so the `{service} {call_type}` fallback is unreachable for Redis (a test asserts it). Postgres helpers are `postgres.{verb} {table}` (`"postgres.select LiteLLM_VerificationToken"`, `"postgres.update LiteLLM_TeamTable"`, @@ -130,12 +120,9 @@ Redis it is also `db.operation.name`), the target lands on `litellm.service.target`, and the litellm call chain that issued the call (`_retrieve_from_cache <- _async_get_cache`) travels as `ServiceLoggerPayload.caller` onto `litellm.service.caller`, with the forwarding -frames (cache facades, circuit-breaker guards, batch retry wrappers, the native -execution's `lifecycle`/`streams` drivers) skipped so it names the code that wanted -the call. A call whose own frames are all forwarders -(a batch op settled in a task of its own, on a cluster client or a NOSCRIPT retry) -reports the chain its declaring code captured and threaded through -`service_caller(...)`, never the forwarders, and `unknown` when there is none. +frames (cache facades, circuit-breaker guards, the native execution's +`lifecycle`/`streams` drivers) skipped so it names the code that wanted the call, +and `unknown` when its own frames are all forwarders. The cache key itself is never on the span: it is unbounded and carries key hashes and session ids, and the span is already named by key family. Like every other span they parent to the diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index 15b0f3f0cc2..8cb2e75ebca 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -20,14 +20,27 @@ from litellm.integrations.otel.plumbing.context import request_root_span if TYPE_CHECKING: from opentelemetry.trace import Span + from litellm.caching.dual_cache import DualCache from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth - from litellm.types.utils import ModelResponseStream + from litellm.types.utils import CallTypesLiteral, ModelResponseStream class LangfuseOpenTelemetryV2(OpenTelemetryV2): """Stamps the caller's trace controls (name, user, session, tags) on the request. Langfuse reads them off the root observation, and the proxy's root span is still recording when the LLM call starts.""" + async def async_pre_call_hook( + self, + user_api_key_dict: "UserAPIKeyAuth", + cache: "DualCache", + data: dict, + call_type: "CallTypesLiteral", + ) -> dict: + # Through v1.111.0 the proxy's leaf-only override check never ran OpenTelemetryV2's pre-call identity + # seeding for the Langfuse subclasses. The MRO-aware check that replaced it would, so this no-op keeps them + # as they were. + return data + def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: root: Final = request_root_span() if root is not None and root.is_recording(): diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 4e07d090edd..8c22381f031 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -10,7 +10,6 @@ table: one lambda per mapping operation, applied against the typed span data. from collections.abc import Callable from typing import Final -from litellm._internal_context import REDIS_FAMILIES_METADATA_KEY from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( MAX_TOOL_DEFINITION_ATTRS_PER_SPAN, @@ -200,12 +199,5 @@ class GenAIMapper: # semconv naming the server it reached. Internal services (router, budget # jobs, …) have no db.system, so they get only the litellm.service.* keys. attrs.update(db_span_attributes(data.service_name, data.call_type, postgres_operation(data))) - attrs.update( - { - LiteLLM.REDIS_FAMILIES - if key == REDIS_FAMILIES_METADATA_KEY - else f"{LiteLLM.METADATA_PREFIX}{key}": value - for key, value in data.event_metadata.items() - } - ) + attrs.update({f"{LiteLLM.METADATA_PREFIX}{key}": value for key, value in data.event_metadata.items()}) return attrs diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 8d186a2715a..02b713fdcdd 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -334,8 +334,6 @@ class LiteLLM: SERVICE_CALL_TYPE: Final = "litellm.service.call_type" SERVICE_CALLER: Final = "litellm.service.caller" SERVICE_TARGET: Final = "litellm.service.target" - # The sorted, comma-joined key families one Redis pipeline carried ops for; bounded, unlike the keys. - REDIS_FAMILIES: Final = "litellm.redis.families" PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms" # The logical name of the MCP server a tool call was routed to. There is no # semconv key for an MCP server's *name* (the convention uses ``server.address`` diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index a2d55aaaa06..85f4cbe107d 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -263,6 +263,10 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_Alert", "LiteLLM_PerformanceAlertState", "LiteLLM_BudgetTable", + "LiteLLM_CustomerOrganizationTable", + "LiteLLM_CustomerTeamTable", + "LiteLLM_CustomerMemberTable", + "LiteLLM_CustomerBudgetIncreaseRequest", "LiteLLM_CredentialsTable", "LiteLLM_ProxyModelTable", "LiteLLM_AgentsTable", @@ -305,9 +309,11 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_DailyUserSpend", "LiteLLM_DailyGlobalSpend", "LiteLLM_DailyOrganizationSpend", + "LiteLLM_DailyCustomerMemberSpend", "LiteLLM_DailyEndUserSpend", "LiteLLM_DailyAgentSpend", "LiteLLM_DailyTeamSpend", + "LiteLLM_DailyCustomerMemberSpend", "LiteLLM_DailyTagSpend", "LiteLLM_ProxyWorkerHeartbeat", "LiteLLM_CronJob", diff --git a/litellm/integrations/performance_alert_notifications.py b/litellm/integrations/performance_alert_notifications.py new file mode 100644 index 00000000000..1d1c8a8ab00 --- /dev/null +++ b/litellm/integrations/performance_alert_notifications.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import asyncio +import hashlib +import html +import os +from collections.abc import Awaitable, Callable, Sequence +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Final + +from pydantic import AliasPath, BaseModel, ConfigDict, Field + +from litellm._logging import verbose_proxy_logger +from litellm.models.alert import AlertSummary +from litellm.repositories.alert_repository import AlertRepository +from litellm.repositories.config_repository import ConfigRepository + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + + +class _EmailSettings(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + enabled: bool = Field(False, strict=True, validation_alias=AliasPath("email_settings", "Performance Alerts")) + + +class _Delivery(BaseModel): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + revision: int = 1 + cursor: int + first_window_start: datetime + next_send_at: datetime + lease_until: datetime + + +def _description(alert: AlertSummary) -> str: + labels: Final = { + "spend": "Spend spike", + "traffic": "Traffic spike", + "latency": "Slower responses", + "reliability": "More failed requests", + } + metric: Final = alert.evidence + + def amount(value: float) -> str: + match metric.unit: + case "usd": + return f"${value:,.2f}" + case "ratio": + return f"{value:.1%}" + case "milliseconds": + return f"{value:,.0f} ms" + case "requests": + return f"{value:,.0f} requests" + + return f"{labels[alert.type]} for {alert.resource_name}: {amount(metric.value)} vs {amount(metric.baseline)}" + + +def slack_text(alerts: Sequence[AlertSummary], base_url: str) -> str: + rows: Final = tuple( + f"<{base_url}/ui/alerts?alert={alert.id}|{html.escape(_description(alert), quote=False)}>" + for alert in alerts[:5] + ) + return "New performance alerts\n\n" + "\n".join(rows) + f"\n\n<{base_url}/ui/alerts|See all alerts>" + + +def email_body(alerts: Sequence[AlertSummary], base_url: str) -> tuple[str, str]: + rows: Final = tuple( + f'
  • {html.escape(_description(alert))}
  • ' + for alert in alerts[:5] + ) + text: Final = ( + "New performance alerts\n\n" + + "\n".join(f"{_description(alert)}\n{base_url}/ui/alerts?alert={alert.id}" for alert in alerts[:5]) + + f"\n\nSee all alerts: {base_url}/ui/alerts" + ) + return ( + f'

    New performance alerts

      {"".join(rows)}

    See all alerts

    ', + text, + ) + + +class PerformanceAlertNotifier: + def __init__(self, config: ConfigRepository, alerts: AlertRepository) -> None: + self.config: Final = config + self.alerts: Final = alerts + + async def deliver( + self, + destination: str, + send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], + now: datetime, + ) -> None: + key: Final = "performance_alert_digest:" + hashlib.sha256(destination.encode()).hexdigest() + row: Final = await self.config.get_param(key) + previous: Final = ( + _Delivery.model_validate(row.param_value) + if row + else _Delivery( + revision=0, + cursor=0, + first_window_start=now - timedelta(days=1), + next_send_at=now, + lease_until=now, + ) + ) + if previous.next_send_at > now or previous.lease_until > now: + return + through: Final = await self.alerts.latest_id() + found: Final = await self.alerts.summaries(previous.cursor, through) + alerts: Final = ( + found + if previous.cursor + else tuple(alert for alert in found if alert.created_at >= previous.first_window_start) + ) + claim: Final = previous.model_copy( + update={ + "revision": previous.revision + 1, + "lease_until": now + timedelta(minutes=2), + "next_send_at": now + timedelta(days=1) if alerts else now, + } + ) + if not await self.config.set_param_if_revision(key, claim.model_dump(mode="json"), previous.revision): + return + if alerts and not await send(alerts, f"{key}-{through}"): + return + sent: Final = claim.model_copy( + update={ + "revision": claim.revision + 1, + "cursor": through, + "next_send_at": now + timedelta(days=1) if alerts else now, + "lease_until": now, + } + ) + if not await self.config.set_param_if_revision(key, sent.model_dump(mode="json"), claim.revision): + verbose_proxy_logger.warning("Could not record performance alert summary delivery") + + async def run(self, slack: SlackAlerting | None, *, now: datetime | None = None) -> None: + from litellm.proxy.utils import send_email + + base_url: Final = os.getenv("PROXY_BASE_URL", "").rstrip("/") + if not base_url: + return + if slack is not None: + for webhook in slack.performance_alert_webhooks(): + + async def send_slack(alerts: Sequence[AlertSummary], receipt: str, webhook: str = webhook) -> bool: + return await slack.send_performance_alert_summary(webhook, slack_text(alerts, base_url)) + + await self._try_deliver("slack:" + webhook, send_slack, now or datetime.now(timezone.utc)) + if not os.getenv("SMTP_HOST") or not os.getenv("SMTP_SENDER_EMAIL"): + return + settings: Final = await self.config.get_param("general_settings") + if not _EmailSettings.model_validate(settings.param_value if settings else {}).enabled: + return + after: str = "" # rebind-ok: advances the bounded recipient page + while recipients := await self.alerts.recipients(after): + for recipient in recipients: + + async def send_mail( + alerts: Sequence[AlertSummary], receipt: str, address: str = recipient.email + ) -> bool: + body, text = email_body(alerts, base_url) + return await send_email( + receiver_email=address, + subject="New performance alerts", + html=body, + text=text, + message_id=f"<{receipt}@alerts.litellm.ai>", + ) + + await self._try_deliver("email:" + recipient.email, send_mail, now or datetime.now(timezone.utc)) + after = recipients[-1].user_id + await asyncio.sleep(0.1) + + async def _try_deliver( + self, destination: str, send: Callable[[Sequence[AlertSummary], str], Awaitable[bool]], now: datetime + ) -> None: + try: + await asyncio.wait_for(self.deliver(destination, send, now), timeout=60) + except Exception as exc: # noqa: BLE001 # one unavailable destination must not stop the other recipients + verbose_proxy_logger.warning("Performance alert summary delivery failed (%s)", type(exc).__name__) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 7984c7d04f3..6f56693b77e 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1061,18 +1061,39 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_deployment_latency_per_output_token"), ) - self.litellm_deployment_successful_fallbacks = self._counter_factory( + self.litellm_deployment_successful_fallbacks: _MetricLike = self._counter_factory( "litellm_deployment_successful_fallbacks", "LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model", labelnames=self.get_labels_for_metric("litellm_deployment_successful_fallbacks"), ) - self.litellm_deployment_failed_fallbacks = self._counter_factory( + self.litellm_deployment_failed_fallbacks: _MetricLike = self._counter_factory( "litellm_deployment_failed_fallbacks", "LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model", labelnames=self.get_labels_for_metric("litellm_deployment_failed_fallbacks"), ) + self.litellm_dynamic_limiter_share_tokens: _MetricLike = self._gauge_factory( + "litellm_dynamic_limiter_share_tokens", + "Dynamic rate limiter: the team's share of the model's TPM, as of the team's last settled request", + labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_share_tokens"), + multiprocess_mode="livemostrecent", + ) + + self.litellm_dynamic_limiter_borrowed_tokens: _MetricLike = self._gauge_factory( + "litellm_dynamic_limiter_borrowed_tokens", + "Dynamic rate limiter: tokens the team has used above its share this window, 0 while under its share", + labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_borrowed_tokens"), + multiprocess_mode="livemostrecent", + ) + + self.litellm_dynamic_limiter_throttled_total: _MetricLike = self._counter_factory( + "litellm_dynamic_limiter_throttled_total", + "Dynamic rate limiter: requests refused per team, model and reason, " + "the reason being the value of the response's rate_limited_by header", + labelnames=self.get_labels_for_metric("litellm_dynamic_limiter_throttled_total"), + ) + # Callback Logging Failure Metrics self.litellm_callback_logging_failures_metric = self._counter_factory( name="litellm_callback_logging_failures_metric", @@ -4172,6 +4193,66 @@ class PrometheusLogger(CustomLogger): sanitize_prometheus_label_value(exception_status), ).inc() + def set_dynamic_limiter_share( + self, + model_group: str, + team: str, + team_alias: str | None, + priority: str, + share_tokens: int, + borrowed_tokens: int, + ) -> None: + enum_values: Final = UserAPIKeyLabelValues( + model_group=model_group, team=team, team_alias=team_alias, priority=priority + ) + self._set_labeled_gauge( + self.litellm_dynamic_limiter_share_tokens, "litellm_dynamic_limiter_share_tokens", enum_values, share_tokens + ) + self._set_labeled_gauge( + self.litellm_dynamic_limiter_borrowed_tokens, + "litellm_dynamic_limiter_borrowed_tokens", + enum_values, + borrowed_tokens, + ) + + def _set_labeled_gauge( + self, + gauge: _MetricLike, + metric_name: DEFINED_PROMETHEUS_METRICS, + enum_values: UserAPIKeyLabelValues, + value: float, + ) -> None: + from prometheus_client import Gauge + + labels: Final = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric(metric_name=metric_name), enum_values=enum_values + ) + series: Final = gauge.labels(**labels) + if isinstance(series, Gauge): + series.set(value) + + def increment_dynamic_limiter_throttled( + self, + model_group: str, + team: str, + team_alias: str | None, + priority: str, + rate_limit_type: str, + reason: str, + ) -> None: + self._inc_labeled_counter( + self.litellm_dynamic_limiter_throttled_total, + "litellm_dynamic_limiter_throttled_total", + UserAPIKeyLabelValues( + model_group=model_group, + team=team, + team_alias=team_alias, + priority=priority, + rate_limit_type=rate_limit_type, + reason=reason, + ), + ) + def increment_callback_logging_failure( self, callback_name: str, @@ -5221,7 +5302,7 @@ def prometheus_label_factory( tag: str | None = None, *, label_context: PrometheusLabelFactoryContext | None = None, -) -> dict: +) -> dict[str, str | None]: """ Returns a dictionary of label + values for prometheus. diff --git a/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/grafana_dashboard.json b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/grafana_dashboard.json index cedbbd0f07b..073d31f2e8f 100644 --- a/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/grafana_dashboard.json +++ b/litellm/integrations/prometheus_helpers/grafana_dashboards/dashboard_all_metrics/grafana_dashboard.json @@ -5852,6 +5852,17 @@ "legendFormat": "self", "range": true, "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(litellm_redis_daily_customer_member_spend_update_queue_latency_bucket[$__rate_interval])) by (le))", + "legendFormat": "redis_daily_customer_member_spend_update_queue", + "range": true, + "refId": "N" } ], "title": "Service latency p95 (litellm__latency)", @@ -6052,6 +6063,17 @@ "legendFormat": "self", "range": true, "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_redis_daily_customer_member_spend_update_queue_total_requests_total[$__rate_interval]))", + "legendFormat": "redis_daily_customer_member_spend_update_queue", + "range": true, + "refId": "O" } ], "title": "Service request rate (litellm__total_requests)", @@ -6252,6 +6274,17 @@ "legendFormat": "self / {{error_class}}", "range": true, "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_redis_daily_customer_member_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)", + "legendFormat": "redis_daily_customer_member_spend_update_queue / {{error_class}}", + "range": true, + "refId": "O" } ], "title": "Service failure rate (litellm__failed_requests)", @@ -6644,6 +6677,190 @@ ], "title": "litellm_lens_export_rows_written and litellm_lens_export_rows_dropped rate", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 456 + }, + "id": 117, + "panels": [], + "title": "Dynamic rate limiter (needs dynamic_rate_limiter_v3 and prometheus callbacks)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The team's token share on the model this window, per team and model group", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 457 + }, + "id": 118, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "litellm_dynamic_limiter_share_tokens", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "litellm_dynamic_limiter_share_tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tokens the team has used above its share this window, 0 while under it", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 457 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "litellm_dynamic_limiter_borrowed_tokens", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "litellm_dynamic_limiter_borrowed_tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests the dynamic limiter refused per second, by team, model and the 429's rate_limited_by reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 465 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (team, team_alias, model_group, reason) (rate(litellm_dynamic_limiter_throttled_total[5m]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "litellm_dynamic_limiter_throttled_total", + "type": "timeseries" } ], "preload": false, diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 3c6b5284041..6c708414859 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -33,6 +33,8 @@ def resolve_prompt_manager_ignore_flags( class PromptManagementBase(ABC): + ignore_prompt_manager_optional_params: bool | None = False + @property @abstractmethod def integration_name(self) -> str: @@ -198,7 +200,8 @@ class PromptManagementBase(ABC): resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags( prompt_spec=prompt_spec, ignore_prompt_manager_model=ignore_prompt_manager_model, - ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params + or self.ignore_prompt_manager_optional_params, ) return self.post_compile_prompt_processing( prompt_template=prompt_template, @@ -245,7 +248,8 @@ class PromptManagementBase(ABC): resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags( prompt_spec=prompt_spec, ignore_prompt_manager_model=ignore_prompt_manager_model, - ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params, + ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params + or self.ignore_prompt_manager_optional_params, ) return self.post_compile_prompt_processing( prompt_template=prompt_template, diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index cd144646f79..76c65f7041d 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -300,6 +300,26 @@ def create( ) else: custom_llm_provider = custom_llm_provider or "gemini" + if agent and model and model != agent: + raise litellm.BadRequestError( + message=( + f"agent '{agent}' can only run on a deployment of that agent, but the model is '{model}'. " + "Configure a deployment whose model is the agent and send its model group." + ), + model=model, + llm_provider=custom_llm_provider, + ) + if extra_body and ( + ("agent" in extra_body and extra_body["agent"] != agent) or (agent and "model" in extra_body) + ): + raise litellm.BadRequestError( + message=( + "extra_body cannot set a different agent or add a model to an agent interaction. " + "Send the agent and model as top-level fields." + ), + model=model, + llm_provider=custom_llm_provider, + ) interactions_api_config: Final = get_provider_interactions_api_config( provider=custom_llm_provider, @@ -340,7 +360,7 @@ def create( litellm_logging_obj.update_from_kwargs( kwargs=kwargs, - model=model, + model=agent or model, optional_params=dict(optional_params), litellm_params={"litellm_call_id": litellm_call_id}, custom_llm_provider=custom_llm_provider, diff --git a/litellm/litellm_core_utils/AGENTS.md b/litellm/litellm_core_utils/AGENTS.md index 9cfeccba44c..09de3cd9f99 100644 --- a/litellm/litellm_core_utils/AGENTS.md +++ b/litellm/litellm_core_utils/AGENTS.md @@ -1,3 +1,11 @@ +## Compat names + +- A change here that removes a name outside code can import keeps the name only when one of these holds + - It has no leading underscore and shipped in a stable release + - It is private, code outside this repo calls it, and the compat keeps that code doing what it did +- A private name whose old behavior cannot be kept stays removed. An `ImportError` names what changed, while a no-op that answers wrongly hides it +- A vendored copy of litellm in another repo is not a caller + ## get_model_cost_map.py - This module decides when to fetch and install. Fetching, parsing, integrity checks and alias expansion run in Rust @@ -9,6 +17,8 @@ - `utils._register_on_current_map` reruns the whole pass on the new table after `NativeTableSwapped`, and its last attempt passes `rows=None`, so no entry is lost - Compat names exist only so code written against main keeps working. New code never calls them, and they never gain behavior of their own - `_finalize_model_cost_map`, use `model_cost_inputs.install` + - `git_blob_id`, which Rust computes for `source_revision` + - `FALLBACK_GENERALIZATIONS_KEY`, `RESERVED_TOP_LEVEL_KEYS` and `RETRYABLE_FETCH_STATUS_CODES`, public constants. Rust owns the retry rule - `GetModelCostMap.validate_model_cost_map` and `_check_*`, since Rust checks fetched documents - `ModelCostMapReloaded.model_cost_map`, use `GetModelCostMap.loaded_model_cost_map()` - the `sleep`, `rng` and `client` parameters, which are ignored with a `DeprecationWarning` @@ -25,3 +35,4 @@ - `utils._invalidate_model_cost_lowercase_map` forwards to `changed_in_place` - `utils.get_model_cost_mutation_generation` forwards to `model_cost_version().writes` - `utils._model_cost_lowercase_map` is always `None` + - `utils._runtime_registered_model_cost` is always empty, since kept registrations live in `model_cost_inputs` diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 70cb517168c..1a34b8f01ed 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -141,6 +141,13 @@ def without( return {key: value for key, value in mapping.items() if key not in keys} +def only( + mapping: Mapping[_Key, _Value], *keys: object +) -> dict[_Key, _Value]: # mutable-ok: a fresh copy the caller owns + """``mapping`` with only ``keys``, as a new dict in the same order; ``mapping`` is unchanged.""" + return {key: value for key, value in mapping.items() if key in keys} + + def decoded_json_object(value: object) -> Mapping[str, object]: mapping: Final = as_str_mapping(value) if mapping is not None: diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 95bc77e3eba..2abb1c7e4da 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -167,6 +167,15 @@ def _shift_months(moment: datetime, months: int) -> datetime: return moment.replace(year=year, month=month, day=min(moment.day, get_last_day_of_month(year, month))) +def is_annual_budget_duration(duration: str) -> bool: + return _parse_duration(normalize_duration(duration)) == (12, "mo") + + +def get_annual_budget_window_start(reset_at: datetime, timezone_str: str) -> datetime: + boundary: Final = _setup_timezone(reset_at, timezone_str)[0] + return _shift_months(boundary, -12).astimezone(timezone.utc) + + def get_budget_window_start(duration: str, reset_at: datetime) -> datetime: """Start of the budget period that ends at `reset_at`, under the same rules `get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar @@ -220,8 +229,12 @@ def _parse_duration(duration: str) -> tuple[int | None, str | None]: if not match: return None, None - value, unit = match.groups() - return int(value), unit + value: Final = int(match.group(1)) + spelled_unit: Final = match.group(2) + unit: Final = _extract_from_regex(duration)[1] or spelled_unit + if unit == "mo" and spelled_unit != "mo" and value not in (1, 12): + return value, spelled_unit + return value, unit def _apply_time_of_day(dt: datetime, reset_time_of_day: time) -> datetime: @@ -390,14 +403,22 @@ def _handle_month_reset( Args: current_time: Current datetime base_midnight: Midnight of current day - value: Number of months (currently only supports 1 month resets) + value: Number of months, 1 for monthly or 12 for annual resets reset_time_of_day: Wall-clock time the reset lands on Returns: datetime: First day of the next reset month at `reset_time_of_day` """ + if value == 12: + first_of_year: Final = base_midnight.replace(month=1, day=1) + annual_candidate: Final = _apply_time_of_day(first_of_year, reset_time_of_day) + return ( + annual_candidate.replace(year=annual_candidate.year + 1) + if annual_candidate <= current_time + else annual_candidate + ) if value != 1: - raise ValueError("Monthly resets currently only support 1 month intervals") + raise ValueError("Calendar resets support 1 month or 12 month intervals") first_of_this_month: Final = base_midnight.replace(day=1) candidate: Final = _apply_time_of_day(first_of_this_month, reset_time_of_day) diff --git a/litellm/litellm_core_utils/get_blog_posts.py b/litellm/litellm_core_utils/get_blog_posts.py index c3b4114d5b9..1201a73d6ce 100644 --- a/litellm/litellm_core_utils/get_blog_posts.py +++ b/litellm/litellm_core_utils/get_blog_posts.py @@ -8,9 +8,11 @@ Disable remote fetching entirely: export LITELLM_LOCAL_BLOG_POSTS=True """ +import asyncio import os import time import xml.etree.ElementTree as ET +from collections.abc import Awaitable, Callable from email.utils import parsedate_to_datetime from importlib.resources import files from typing import Final @@ -78,6 +80,7 @@ class GetBlogPosts: _cached_posts: tuple[BlogPost, ...] | None = None _last_fetch_time: float = 0.0 + _refresh_lock: tuple[asyncio.AbstractEventLoop, asyncio.Lock] | None = None @staticmethod def load_local_blog_posts() -> tuple[BlogPost, ...]: @@ -98,7 +101,7 @@ class GetBlogPosts: return response.text @staticmethod - def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]: + def parse_rss_to_posts(xml_text: str | bytes, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]: """ Parse RSS XML into blog posts. @@ -147,14 +150,52 @@ class GetBlogPosts: url, str(e), ) - return cls.load_local_blog_posts() + return cls._cache_posts(cls.load_local_blog_posts()) if not cls.validate_blog_posts(posts): + return cls._cache_posts(cls.load_local_blog_posts()) + + return cls._cache_posts(posts) + + @classmethod + def _cache_posts(cls, posts: tuple[BlogPost, ...]) -> tuple[BlogPost, ...]: + cls._cached_posts = posts + cls._last_fetch_time = time.time() + return posts + + @classmethod + async def aget_blog_posts(cls, url: str, fetch: Callable[[str], Awaitable[bytes]]) -> tuple[BlogPost, ...]: + if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true": return cls.load_local_blog_posts() - cls._cached_posts = posts - cls._last_fetch_time = now - return posts + cached: Final = cls._cached_posts + if cached is not None and (time.time() - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS: + return cached + + loop: Final = asyncio.get_running_loop() + state: Final = cls._refresh_lock + lock: Final = state[1] if state is not None and state[0] is loop else asyncio.Lock() + cls._refresh_lock = (loop, lock) + async with lock: + refreshed: Final = cls._cached_posts + if refreshed is not None and (time.time() - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS: + return refreshed + return await cls._refresh_blog_posts(url, fetch) + + @classmethod + async def _refresh_blog_posts(cls, url: str, fetch: Callable[[str], Awaitable[bytes]]) -> tuple[BlogPost, ...]: + try: + xml_text: Final = await fetch(url) + posts: Final = cls.parse_rss_to_posts(xml_text) + except (httpx.HTTPError, ET.ParseError, ValueError) as e: + verbose_logger.warning( + "LiteLLM: Failed to fetch blog posts from %s: %s. Falling back to local backup.", url, str(e) + ) + return cls._cache_posts(cls.load_local_blog_posts()) + + if not cls.validate_blog_posts(posts): + return cls._cache_posts(cls.load_local_blog_posts()) + return cls._cache_posts(posts) def get_blog_posts(url: str) -> tuple[BlogPost, ...]: diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 6f4405066ca..2a7a76ab3b1 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -9,6 +9,7 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ +import hashlib import json import os import sys @@ -51,6 +52,9 @@ from litellm.rust_bridge._native import ( _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) +FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY}) +RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504}) MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3 MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 MODEL_COST_MAP_FETCH_MIN_DELAY_SECONDS: Final = 2.0 @@ -456,18 +460,28 @@ def get_model_cost_map( return _rows_with_rules_set(installed) +def registration_rows(body: bytes) -> dict[str, dict[str, object]]: # mutable-ok: register_model merges these rows + """The rows of a fetched ``register_model`` document, without its fallback rules block. Raises ``ValueError`` on + a body that is not a JSON object of rows.""" + document: Final = _CATALOG_ADAPTER.validate_python(json.loads(body)) + return {key: entry for key, entry in document.items() if key != FALLBACK_GENERALIZATIONS_KEY} + + def fetch_registration_document(url: str) -> dict[str, dict[str, object]] | None: - """The document behind ``register_model(url)``, fetched in one attempt, or ``None`` when the fetch failed.""" + """The document behind ``register_model(url)``, fetched in one attempt. ``None`` when offline mode forbids the + fetch or it failed.""" + if is_offline_mode(): + verbose_logger.warning("LiteLLM: register_model did not fetch %s because LITELLM_OFFLINE_MODE is set", url) + return None fetched: Final = cost_map_source(url, use_http_settings=False).fetch_blocking(fetch_policy(5, 1)) if isinstance(fetched, NativeFetchFailed): verbose_logger.warning("LiteLLM: register_model could not fetch %s: %s", url, fetched.reason) return None try: - document: Final = _CATALOG_ADAPTER.validate_json(fetched.body) + return registration_rows(fetched.body) except ValueError as e: verbose_logger.warning("LiteLLM: register_model got an invalid cost map from %s: %s", url, e) return None - return {key: entry for key, entry in document.items() if key != "fallback_generalizations"} def _finalize_model_cost_map( # pyright: ignore[reportUnusedFunction] # main's private name, downstream code calls it @@ -478,15 +492,17 @@ def _finalize_model_cost_map( # pyright: ignore[reportUnusedFunction] # main's case NativeRejected(reason=reason): raise ValueError(reason) case installed: - model_cost.pop("fallback_generalizations", None) + model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None) return _rows_with_rules_set(installed) -_RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) +def git_blob_id(body: bytes) -> str: + """Compat for main's callers. ``git hash-object`` of the bytes, which ``source_revision`` reports.""" + return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() def _count_model_entries(model_cost: Mapping[str, object]) -> int: - return sum(1 for key in model_cost if key not in _RESERVED_TOP_LEVEL_KEYS) + return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS) @lru_cache(maxsize=1) diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index de17bcbdfb9..98c009552d1 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -44,16 +44,16 @@ BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost" ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved" OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved" -ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key" -OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key" +ITPM_COUNTER_KEY: Final = "_litellm_itpm_counter" +OTPM_COUNTER_KEY: Final = "_litellm_otpm_counter" IO_TOKEN_ATTEMPT_KEY: Final = "_litellm_io_token_attempt" IO_TOKEN_RESERVATION_ID_KEY: Final = "_litellm_io_token_reservation_id" IO_TOKEN_RESERVATION_KEYS: Final = frozenset( { ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, - ITPM_CACHE_KEY, - OTPM_CACHE_KEY, + ITPM_COUNTER_KEY, + OTPM_COUNTER_KEY, IO_TOKEN_ATTEMPT_KEY, IO_TOKEN_RESERVATION_ID_KEY, } diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5d21b2762cd..3951b49ab30 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -14,6 +14,7 @@ import sys import time import traceback from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager from dataclasses import dataclass from datetime import datetime as dt_object from enum import Enum @@ -28,7 +29,13 @@ from pydantic import BaseModel, JsonValue, TypeAdapter from typing_extensions import TypedDict import litellm -from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase +from litellm._internal_context import ( + in_mirrored_call, + outside_mirrored_call, + post_response_phase, + request_parent_otel_span, +) +from litellm._lazy_imports import get_token_counter_new from litellm._logging import ( is_debugging_on, redact_string, @@ -47,6 +54,7 @@ from litellm.constants import ( DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, EMPTY_MAPPING, + FAILED_PROMPT_TOKEN_ESTIMATE_KEY, INTERNAL_CALLBACK_NAMES, KUBERNETES_POD_ROUTING_KEY, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, @@ -74,6 +82,20 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.legacy import ( + Admit, + KwargsFields, + SyncLegacyBinding, + hook_subscribers, + kwargs_after, + kwargs_patch, + run_subscribers, + run_subscribers_sync, +) +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.manifest import LegacyBinding +from litellm.integrations.custom_logger_v2.outcomes import Patch from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.langtrace import langtrace_trace_endpoint from litellm.integrations.mlflow import MlflowLogger @@ -95,6 +117,7 @@ from litellm.litellm_core_utils.core_helpers import ( attribute_or_none, decoded_json_object, finite_cost, + get_parent_otel_span_from_kwargs, get_provider_response_headers_from_hidden_params, has_attributes, is_expected_client_error, @@ -178,6 +201,7 @@ from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.llms.openai import ( AllMessageValues, Batch, + ChatCompletionSystemMessage, FineTuningJob, HttpxBinaryResponseContent, OpenAIFileObject, @@ -297,6 +321,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation + from litellm.types.llms.base import HiddenParams try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, @@ -674,6 +699,9 @@ class UsageObservation(Enum): ESTIMATED = "estimated" +_CHAT_CALL_TYPES: Final = frozenset({CallTypes.completion.value, CallTypes.acompletion.value}) + + class _UsageFields(TypedDict, total=False, extra_items=object): pass @@ -728,6 +756,97 @@ def _callback_control_params(litellm_params: object) -> dict[str, object]: return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {} +LOGGING_KEYS: Final[Mapping[Field, str]] = MappingProxyType( + { + Field.model: "model", + Field.messages: "messages", + Field.input: "input", + Field.optional_params: "optional_params", + Field.standard_logging_payload: "standard_logging_object", + } +) + + +def _holds_v2(callbacks: Iterable[object]) -> bool: + return any(isinstance(callback, CustomLoggerV2) for callback in callbacks) + + +# include_v2=False runs only the legacy callbacks in the list. Sync handlers pass it when they also run for an async +# request (the async handler delivers the same event) or for one chunk of a stream (the complete response is +# delivered once at the end), so a V2 callback isn't sent the same event twice or once per chunk. Legacy callbacks +# keep their old behaviour there. +def _run_callbacks_sync( + hook: Hook, callbacks: Sequence[object], kwargs: object, binding: SyncLegacyBinding, include_v2: bool = True +) -> None: + if include_v2 and _holds_v2(callbacks): + _ = run_subscribers_sync( + hook_subscribers(hook, callbacks, admit=Admit.every_entry), KwargsFields(kwargs, LOGGING_KEYS), binding + ) + return + for callback in callbacks: + if not isinstance(callback, CustomLoggerV2): + _ = binding(callback, kwargs) + + +async def _run_callbacks( + hook: Hook, callbacks: Sequence[object], kwargs: object, binding: LegacyBinding, include_v2: bool = True +) -> None: + if include_v2 and _holds_v2(callbacks): + _ = kwargs_after( + await run_subscribers( + hook_subscribers(hook, callbacks, admit=Admit.every_entry), KwargsFields(kwargs, LOGGING_KEYS), binding + ) + ) + return + for callback in callbacks: + if isinstance(callback, CustomLoggerV2): + continue + started = binding(callback, kwargs) + if started is not None: + _ = await started + + +@dataclass(slots=True, eq=False) +class _LogHookRun: + details: dict[str, object] # mutable-ok: legacy logging hooks take and return the live call details + result: object + + +@dataclass(slots=True, eq=False) +class _SyncSuccessRun: + call_details: dict[str, object] # mutable-ok: legacy loggers take the live call details + result: object + kwargs: Mapping[str, object] + litellm_params: object + start_time: datetime.datetime + end_time: datetime.datetime + complete_streaming_response: object + is_sync_request: bool + for_async_request: bool + call_details_view: Mapping[str, object] + + +@dataclass(slots=True, eq=False) +class _McpToolCallRun: + kwargs: dict[str, object] # mutable-ok: forwarded unchanged to legacy hooks that take a dict + response_obj: "CallToolResult" + start_time: datetime.datetime + end_time: datetime.datetime + hidden_params: "HiddenParams" + + +@dataclass(frozen=True, slots=True) +class _FailureRun: + exception: Exception + traceback_exception: str + result: object + start_time: datetime.datetime + end_time: datetime.datetime | None + litellm_params: object + is_sync_request: bool + for_async_request: bool + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -748,6 +867,7 @@ class Logging(LiteLLMLoggingBaseClass): stream_options = None litellm_request_debug: bool = False streamed_anthropic_message_id: str | None = None + served_agentic_follow_up: bool = False bridged_model_response: ModelResponse | None = None classifier_input: Mapping[str, JsonValue] | None = None baseline_cache_context: "BaselineCacheContext | None" = None @@ -827,7 +947,7 @@ class Logging(LiteLLMLoggingBaseClass): # Shallow copy of the outer list only (inner message dicts are shared). # Safe because the logging layer does not mutate individual message dicts. _copy_start: Final = time.time() - self.messages = copy.copy(messages) if messages is not None else None + self.messages: Sequence[object] | None = copy.copy(messages) if messages is not None else None self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000 self.callback_duration_ms: float = 0.0 self.stream = stream @@ -1783,62 +1903,12 @@ class Logging(LiteLLMLoggingBaseClass): if self.model_call_details.get("first_api_call_start_time") is None: self.model_call_details["first_api_call_start_time"] = api_call_start_time # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made - callbacks: Final = ( - *self._without_replaced_global_callbacks(litellm.input_callback), - *(self.dynamic_input_callbacks or ()), + _run_callbacks_sync( + Hook.provider_request_sent, + self._input_callbacks(), + self.model_call_details, + self._log_provider_request_sent, ) - for callback in callbacks: - try: - if callback == "supabase" and supabaseClient is not None: - verbose_logger.debug("reaches supabase for logging!") - model = self._model_call_details_view["model"] - messages = self._model_call_details_view["input"] - verbose_logger.debug("supabaseClient: %s", supabaseClient) - supabaseClient.input_log_event( - model=model, - messages=messages, - end_user=self._model_call_details_view.get("user", "default"), - litellm_call_id=self.litellm_params["litellm_call_id"], - print_verbose=print_verbose, - ) - elif callback == "sentry" and add_breadcrumb: - try: - details_to_log = copy.deepcopy(self.model_call_details) - except Exception: - details_to_log = self.model_call_details - if litellm.turn_off_message_logging: - # make a copy of the _model_Call_details and log it - details_to_log.pop("messages", None) - details_to_log.pop("input", None) - details_to_log.pop("prompt", None) - - add_breadcrumb( - category="litellm.llm_call", - message=f"Model Call Details pre-call: {details_to_log}", - level="info", - ) - - elif isinstance(callback, CustomLogger): # custom logger class - callback.log_pre_api_call( - model=self.model, - messages=self.messages, - kwargs=self.model_call_details, - ) - elif callable(callback) and customLogger is not None: # custom logger functions - customLogger.log_input_event( - model=self.model, - messages=self.messages, - kwargs=self.model_call_details, - print_verbose=print_verbose, - callback_func=callback, - ) - except Exception as e: - verbose_logger.exception("litellm.Logging.pre_call(): Exception occured - %s", e) - verbose_logger.debug( - "LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception - ) - if capture_exception: # log this error to sentry for debugging - capture_exception(e) except Exception as e: verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) verbose_logger.error("LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception) @@ -1848,6 +1918,66 @@ class Logging(LiteLLMLoggingBaseClass): if self.raw_request_only: raise RawRequestCaptured() + def _input_callbacks(self) -> tuple[str | Callable[..., object] | CustomLogger, ...]: + return ( + *self._without_replaced_global_callbacks(litellm.input_callback), + *(self.dynamic_input_callbacks or ()), + ) + + def _breadcrumb_details(self) -> dict[str, object]: # mutable-ok: the breadcrumb pops message keys from it + try: + return copy.deepcopy(self.model_call_details) + except Exception: + return self.model_call_details + + def _log_provider_request_sent(self, callback: object, _kwargs: object) -> None: + try: + if callback == "supabase" and supabaseClient is not None: + verbose_logger.debug("reaches supabase for logging!") + model: Final = self._model_call_details_view["model"] + messages: Final = self._model_call_details_view["input"] + verbose_logger.debug("supabaseClient: %s", supabaseClient) + supabaseClient.input_log_event( + model=model, + messages=messages, + end_user=self._model_call_details_view.get("user", "default"), + litellm_call_id=self.litellm_params["litellm_call_id"], + print_verbose=print_verbose, + ) + elif callback == "sentry" and add_breadcrumb: + details_to_log: Final = self._breadcrumb_details() + if litellm.turn_off_message_logging: + # make a copy of the _model_Call_details and log it + details_to_log.pop("messages", None) + details_to_log.pop("input", None) + details_to_log.pop("prompt", None) + + add_breadcrumb( + category="litellm.llm_call", + message=f"Model Call Details pre-call: {details_to_log}", + level="info", + ) + + elif isinstance(callback, CustomLogger): # custom logger class + callback.log_pre_api_call( + model=self.model, + messages=self.messages, + kwargs=self.model_call_details, + ) + elif callable(callback) and customLogger is not None: # custom logger functions + customLogger.log_input_event( + model=self.model, + messages=self.messages, + kwargs=self.model_call_details, + print_verbose=print_verbose, + callback_func=callback, + ) + except Exception as e: + verbose_logger.exception("litellm.Logging.pre_call(): Exception occured - %s", e) + verbose_logger.debug("LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception) + if capture_exception: # log this error to sentry for debugging + capture_exception(e) + def _print_llm_call_debugging_log( self, api_base: str, @@ -1959,49 +2089,47 @@ class Logging(LiteLLMLoggingBaseClass): ) # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made - callbacks: Final = ( - *self._without_replaced_global_callbacks(litellm.input_callback), - *(self.dynamic_input_callbacks or ()), + _run_callbacks_sync( + Hook.provider_response_received, + self._input_callbacks(), + self.model_call_details, + self._log_provider_response_received, ) - for callback in callbacks: - try: - if callback == "sentry" and add_breadcrumb: - verbose_logger.debug("reaches sentry breadcrumbing") - try: - details_to_log = copy.deepcopy(self.model_call_details) - except Exception: - details_to_log = self.model_call_details - if litellm.turn_off_message_logging: - # make a copy of the _model_Call_details and log it - details_to_log.pop("messages", None) - details_to_log.pop("input", None) - details_to_log.pop("prompt", None) - - add_breadcrumb( - category="litellm.llm_call", - message=f"Model Call Details post-call: {details_to_log}", - level="info", - ) - elif isinstance(callback, CustomLogger): # custom logger class - callback.log_post_api_call( - kwargs=self.model_call_details, - response_obj=None, - start_time=self.start_time, - end_time=None, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations %s", - e, - ) - verbose_logger.debug( - "LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception - ) - if capture_exception: # log this error to sentry for debugging - capture_exception(e) except Exception as e: verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) + def _log_provider_response_received(self, callback: object, _kwargs: object) -> None: + try: + if callback == "sentry" and add_breadcrumb: + verbose_logger.debug("reaches sentry breadcrumbing") + details_to_log: Final = self._breadcrumb_details() + if litellm.turn_off_message_logging: + # make a copy of the _model_Call_details and log it + details_to_log.pop("messages", None) + details_to_log.pop("input", None) + details_to_log.pop("prompt", None) + + add_breadcrumb( + category="litellm.llm_call", + message=f"Model Call Details post-call: {details_to_log}", + level="info", + ) + elif isinstance(callback, CustomLogger): # custom logger class + callback.log_post_api_call( + kwargs=self.model_call_details, + response_obj=None, + start_time=self.start_time, + end_time=None, + ) + except Exception as e: + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations %s", + e, + ) + verbose_logger.debug("LiteLLM.Logging: is sentry capture exception initialized %s", capture_exception) + if capture_exception: # log this error to sentry for debugging + capture_exception(e) + async def async_post_mcp_tool_call_hook( self, kwargs: dict, @@ -2011,61 +2139,75 @@ class Logging(LiteLLMLoggingBaseClass): ) -> "CallToolResult": """Apply ordered MCP content callbacks to the result returned to the caller.""" from litellm.types.llms.base import HiddenParams - from litellm.types.mcp import MCPPostCallResponseObject + run: Final = _McpToolCallRun( + kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # kwargs arrives as a bare dict + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + hidden_params=HiddenParams(), + ) callbacks: Final = self.get_combined_callback_list( dynamic_success_callbacks=self.dynamic_success_callbacks, global_callbacks=litellm.success_callback, ) - hidden_params = HiddenParams() - for callback in callbacks: - try: - if isinstance(callback, CustomLogger): - original_content = copy.deepcopy(response_obj.content) - original_structured_content = copy.deepcopy(response_obj.structured_content) - callback_response = MCPPostCallResponseObject( - mcp_tool_call_response=copy.deepcopy(original_content), hidden_params=hidden_params - ) - try: - response = await callback.async_post_mcp_tool_call_hook( - kwargs=kwargs, - response_obj=callback_response, - start_time=start_time, - end_time=end_time, - ) - hook_content = ( - self._parse_post_mcp_call_hook_response(response=response) - if response is not None - else callback_response.mcp_tool_call_response - ) - if response is not None: - hidden_params = response.hidden_params - except Exception as e: - verbose_logger.exception( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e - ) - hook_content = None - structured_replacement_matches = ( - response_obj.structured_content != original_structured_content - and ( - hook_content is None - or hook_content == original_content - or response_obj.content == hook_content - ) - ) - if hook_content is not None and hook_content != original_content: - response_obj.content[:] = hook_content - if ( - response_obj.content != original_content - and response_obj.structured_content is not None - and not structured_replacement_matches - ): - response_obj.structured_content = None - response_obj.is_error = True - except Exception as e: - verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) + await _run_callbacks( + Hook.mcp_tool_result, callbacks, run.kwargs, functools.partial(self._post_mcp_tool_call_start, run) + ) return response_obj + def _post_mcp_tool_call_start( + self, run: "_McpToolCallRun", callback: object, _kwargs: object + ) -> Awaitable[None] | None: + return self._post_mcp_tool_call_one(run, callback) if isinstance(callback, CustomLogger) else None + + async def _post_mcp_tool_call_one(self, run: "_McpToolCallRun", callback: CustomLogger) -> None: + response_obj: Final = run.response_obj + try: + original_content: Final = copy.deepcopy(response_obj.content) + original_structured_content: Final = copy.deepcopy(response_obj.structured_content) + callback_response: Final = MCPPostCallResponseObject( + mcp_tool_call_response=copy.deepcopy(original_content), hidden_params=run.hidden_params + ) + hook_content: Final = await self._post_mcp_tool_call_content(run, callback, callback_response) + structured_replacement_matches: Final = response_obj.structured_content != original_structured_content and ( + hook_content is None or hook_content == original_content or response_obj.content == hook_content + ) + if hook_content is not None and hook_content != original_content: + response_obj.content[:] = hook_content + if ( + response_obj.content != original_content + and response_obj.structured_content is not None + and not structured_replacement_matches + ): + response_obj.structured_content = None + response_obj.is_error = True + except Exception as e: + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) + + async def _post_mcp_tool_call_content( + self, run: "_McpToolCallRun", callback: CustomLogger, callback_response: MCPPostCallResponseObject + ) -> "Sequence[TextContent | ImageContent | EmbeddedResource] | None": + try: + response: Final = await callback.async_post_mcp_tool_call_hook( + kwargs=run.kwargs, + response_obj=callback_response, + start_time=run.start_time, + end_time=run.end_time, + ) + hook_content: Final = ( + self._parse_post_mcp_call_hook_response(response=response) + if response is not None + else callback_response.mcp_tool_call_response + ) + if response is not None: + run.hidden_params = response.hidden_params + except Exception as e: + verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) + return None + else: + return hook_content + def _parse_post_mcp_call_hook_response( self, response: MCPPostCallResponseObject | None ) -> "Sequence[TextContent | ImageContent | EmbeddedResource] | None": @@ -3046,7 +3188,7 @@ class Logging(LiteLLMLoggingBaseClass): """Deprecated alias of mark_logging_complete, kept for callers of the old name""" self.mark_logging_complete(event_type=event_type) - def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: object, event_hook: str) -> bool: + def should_run_callback(self, callback: object, litellm_params: object, event_hook: str) -> bool: if litellm.global_disable_no_log_param: return True @@ -3732,6 +3874,11 @@ class Logging(LiteLLMLoggingBaseClass): if trace_id_var.get() == self._own_trace_id and session_id_var.get() == self._own_session_id: self._restore_correlation_context() + def _parent_otel_span_anchored(self) -> AbstractContextManager[None]: + """Every callback runs in one of the four handlers, streamed or not, so anchoring here gives each + callback's datastore spans the caller's span without the callback knowing.""" + return request_parent_otel_span(get_parent_otel_span_from_kwargs(self.model_call_details)) + def success_handler( self, result: object = None, # heterogeneous response object; varies by call type (ANN401 ignored, see ruff-strict.toml) @@ -3745,7 +3892,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(), outside_mirrored_call(): + with post_response_phase(), outside_mirrored_call(), self._parent_otel_span_anchored(): return self._success_handler_body( result=result, start_time=start_time, @@ -3849,8 +3996,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback_logging_payload is not None or shared_logging_payload is None else shared_logging_payload.copy() ) - call_details: dict[str, object] # mutable-ok: legacy loggers take a plain dict - call_details = ( # rebind-ok: logging hooks may return a new dict + call_details: Final[dict[str, object]] = ( # mutable-ok: legacy loggers take a plain dict self.model_call_details if not for_async_request else ( @@ -3878,378 +4024,423 @@ class Logging(LiteLLMLoggingBaseClass): ) ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) - for callback in hook_callbacks: - if isinstance(callback, CustomGuardrail): - from litellm.types.guardrails import GuardrailEventHooks + include_v2: Final = not for_async_request and not ( + bool(self.stream) and complete_streaming_response is None + ) + prepared: Final = _LogHookRun(details=call_details, result=result) + _run_callbacks_sync( + Hook.log_prepare, + hook_callbacks, + call_details, + functools.partial(self._sync_logging_hook, prepared, for_async_request), + include_v2=include_v2, + ) - if ( - callback.should_run_guardrail( - data=call_details, - event_type=GuardrailEventHooks.logging_only, - ) - is not True - ): - continue - - self.shadow_eval_request_snapshot = None - call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict - kwargs=call_details, - result=result, - call_type=self.call_type, - ) - elif isinstance(callback, CustomLogger): - call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict - kwargs=call_details, - result=result, - call_type=self.call_type, - ) - if not for_async_request: - self.model_call_details = call_details - - call_details_view: Final = MappingProxyType[str, object](call_details) self.mark_logging_complete(event_type="sync_success") - for callback in delivery_callbacks: - try: - should_run = self.should_run_callback( - callback=callback, - litellm_params=litellm_params, - event_hook="success_handler", - ) - if not should_run: - continue - if callback == "promptlayer" and promptLayerLogger is not None: - print_verbose("reaches promptlayer for logging!") - promptLayerLogger.log_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - if callback == "supabase" and supabaseClient is not None: - print_verbose("reaches supabase for logging!") - kwargs = call_details - - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - if self.stream: - if "complete_streaming_response" not in kwargs: - continue - else: - print_verbose("reaches supabase for streaming logging!") - result = call_details_view["complete_streaming_response"] - - model = call_details_view["model"] - messages = call_details_view["messages"] - optional_params = call_details_view.get("optional_params", {}) - litellm_params = call_details_view.get("litellm_params", {}) - supabaseClient.log_event( - model=model, - messages=messages, - end_user=require_attributes(optional_params, _ParamsLookup, "get").get("user", "default"), - response_obj=result, - start_time=start_time, - end_time=end_time, - litellm_call_id=( - current_call_id - if ( - current_call_id := require_attributes(litellm_params, _ParamsLookup, "get").get( - "litellm_call_id", None - ) - ) - is not None - else str(uuid.uuid4()) - ), - print_verbose=print_verbose, - ) - if callback == "wandb" and weightsBiasesLogger is not None: - print_verbose("reaches wandb for logging!") - weightsBiasesLogger.log_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - if callback == "logfire" and logfireLogger is not None: - verbose_logger.debug("reaches logfire for success logging!") - kwargs = {} - for k, v in call_details_view.items(): - if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine - kwargs[k] = v - - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - if self.stream: - if "complete_streaming_response" not in kwargs: - continue - else: - print_verbose("reaches logfire for streaming logging!") - result = call_details_view["complete_streaming_response"] - - logfireLogger.log_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - level=LogfireLevel.INFO, - ) - - if callback == "lunary" and lunaryLogger is not None: - print_verbose("reaches lunary for logging!") - model = self.model - kwargs = call_details - - input = kwargs.get("messages", kwargs.get("input", None)) - - type = "embed" if self.call_type == CallTypes.embedding.value else "llm" - - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - if self.stream: - if "complete_streaming_response" not in kwargs: - continue - else: - result = call_details_view["complete_streaming_response"] - - lunaryLogger.log_event( - type=type, - kwargs=kwargs, - event="end", - model=model, - input=input, - user_id=kwargs.get("user", None), - # user_props=self.model_call_details.get("user_props", None), - extra=call_details_view.get("optional_params", {}), - response_obj=result, - start_time=start_time, - end_time=end_time, - run_id=self.litellm_call_id, - print_verbose=print_verbose, - ) - if callback == "helicone" and heliconeLogger is not None: - print_verbose("reaches helicone for logging!") - model = self.model - messages = call_details_view["input"] - kwargs = call_details - - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - if self.stream: - if "complete_streaming_response" not in kwargs: - continue - else: - print_verbose("reaches helicone for streaming logging!") - result = call_details_view["complete_streaming_response"] - - heliconeLogger.log_success( - model=model, - messages=messages, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - kwargs=kwargs, - ) - if callback == "langfuse": - global langFuseLogger - print_verbose("reaches langfuse for success logging!") - kwargs = {} - for k, v in call_details_view.items(): - if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine - kwargs[k] = v - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - if self.stream: - verbose_logger.debug( - "is complete_streaming_response in kwargs: %s", - kwargs.get("complete_streaming_response", None), - ) - if complete_streaming_response is None: - continue - else: - print_verbose("reaches langfuse for streaming logging!") - result = call_details_view["complete_streaming_response"] - - langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request( - globalLangfuseLogger=langFuseLogger, - standard_callback_dynamic_params=self.standard_callback_dynamic_params, - in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache, - ) - if langfuse_logger_to_use is not None: - _response = langfuse_logger_to_use.log_event_on_langfuse( - kwargs=kwargs, - response_obj=result, - start_time=start_time, - end_time=end_time, - user_id=kwargs.get("user", None), - ) - if _response is not None and isinstance(_response, dict): - _trace_id = _response.get("trace_id", None) - if _trace_id is not None: - in_memory_trace_id_cache.set_cache( - litellm_call_id=self.litellm_call_id, - service_name="langfuse", - trace_id=_trace_id, - ) - if callback == "greenscale" and greenscaleLogger is not None: - kwargs = {} - for k, v in call_details_view.items(): - if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine - kwargs[k] = v - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - if self.stream: - verbose_logger.debug( - "is complete_streaming_response in kwargs: %s", - kwargs.get("complete_streaming_response", None), - ) - if complete_streaming_response is None: - continue - else: - print_verbose("reaches greenscale for streaming logging!") - result = call_details_view["complete_streaming_response"] - - greenscaleLogger.log_event( - kwargs=kwargs, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - if callback == "athina" and athinaLogger is not None: - deep_copy = {} - for k, v in call_details_view.items(): - deep_copy[k] = v - athinaLogger.log_event( - kwargs=deep_copy, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - if callback == "traceloop": - deep_copy = {} - for k, v in call_details_view.items(): - if k != "original_response": - deep_copy[k] = v - traceloopLogger.log_event( - kwargs=deep_copy, - response_obj=result, - start_time=start_time, - end_time=end_time, - user_id=kwargs.get("user", None), - print_verbose=print_verbose, - ) - if callback == "s3": - global s3Logger - if s3Logger is None: - s3Logger = S3Logger() - if self.stream: - if "complete_streaming_response" in call_details: - print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response") - s3Logger.log_event( - kwargs=call_details, - response_obj=call_details_view["complete_streaming_response"], - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - else: - print_verbose("S3Logger Logger: Got Stream Event - No complete stream response as yet") - else: - s3Logger.log_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - - if callback == "openmeter" and is_sync_request: - global openMeterLogger - if openMeterLogger is None: - print_verbose("Instantiates openmeter client") - openMeterLogger = OpenMeterLogger() - if self.stream and complete_streaming_response is None: - openMeterLogger.log_stream_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - else: - if self.stream and complete_streaming_response: - call_details["complete_response"] = call_details.get("complete_streaming_response", {}) - result = call_details_view["complete_response"] - openMeterLogger.log_success_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - if ( - isinstance(callback, CustomLogger) - and (is_sync_request or (for_async_request and not self.stream)) - and not ( - for_async_request - and self.stream - and callback in (self.dynamic_async_success_callbacks or ()) - ) - and self.call_type - != CallTypes.pass_through.value # pass-through endpoints call async_log_success_event - ): # custom logger class - if self.stream and complete_streaming_response is None: - callback.log_stream_event( - kwargs=redacted_model_call_details_for_custom_logger( - model_call_details=call_details, custom_logger=callback - ), - response_obj=redacted_response_obj_for_custom_logger(result, callback), - start_time=start_time, - end_time=end_time, - ) - else: - if self.stream and complete_streaming_response: - call_details["complete_response"] = call_details.get("complete_streaming_response", {}) - result = call_details_view["complete_response"] - - callback.log_success_event( - kwargs=redacted_model_call_details_for_custom_logger( - model_call_details=callback.redact_standard_logging_payload_from_model_call_details( - call_details - ), - custom_logger=callback, - ), - response_obj=redacted_response_obj_for_custom_logger(result, callback), - start_time=start_time, - end_time=end_time, - ) - if callable(callback) is True and is_sync_request: # custom logger functions - function_logger = customLogger if customLogger is not None else CustomLogger() - print_verbose(f"success callbacks: Running Custom Callback Function - {callback}") - - function_logger.log_event( - kwargs=call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - callback_func=callback, - ) - - except Exception as e: - print_verbose( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging with integrations {traceback.format_exc()}" - ) - print_verbose(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") - if capture_exception: # log this error to sentry for debugging - capture_exception(e) - # Track callback logging failures in Prometheus - try: - self._handle_callback_failure(callback=callback) - except Exception: - pass + run: Final = _SyncSuccessRun( + call_details=prepared.details, + result=prepared.result, + kwargs=kwargs, + litellm_params=litellm_params, + start_time=start_time, + end_time=end_time, + complete_streaming_response=complete_streaming_response, + is_sync_request=is_sync_request, + for_async_request=for_async_request, + call_details_view=MappingProxyType(prepared.details), + ) + _run_callbacks_sync( + Hook.attempt_ended, + delivery_callbacks, + prepared.details, + functools.partial(self._deliver_sync_success, run), + include_v2=include_v2, + ) except Exception as e: verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", e, ) + def _sync_logging_hook( + self, prepared: _LogHookRun, for_async_request: bool, callback: object, _kwargs: object + ) -> Patch | None: + before: Final = prepared.details + if isinstance(callback, CustomGuardrail): + from litellm.types.guardrails import GuardrailEventHooks + + if ( + callback.should_run_guardrail( + data=before, + event_type=GuardrailEventHooks.logging_only, + ) + is not True + ): + return None + + self.shadow_eval_request_snapshot = None + prepared.details, prepared.result = callback.logging_hook( # pyright: ignore[reportUnknownMemberType] # logging_hook is typed with a bare dict + kwargs=before, + result=prepared.result, + call_type=self.call_type, + ) + elif isinstance(callback, CustomLogger): + prepared.details, prepared.result = callback.logging_hook( # pyright: ignore[reportUnknownMemberType] # logging_hook is typed with a bare dict + kwargs=before, + result=prepared.result, + call_type=self.call_type, + ) + if not for_async_request: + self.model_call_details = prepared.details + return kwargs_patch(before, prepared.details) + + def _deliver_sync_success(self, run: _SyncSuccessRun, callback: object, _kwargs: object) -> None: + call_details: Final = run.call_details + start_time: Final = run.start_time + end_time: Final = run.end_time + complete_streaming_response: Final = run.complete_streaming_response + is_sync_request: Final = run.is_sync_request + for_async_request: Final = run.for_async_request + call_details_view: Final = run.call_details_view + result = run.result # rebind-ok: integrations swap in the assembled stream for later callbacks + kwargs: Mapping[str, object] = run.kwargs # rebind-ok: integrations leave their kwargs to later callbacks + litellm_params = run.litellm_params # rebind-ok: supabase leaves its params to later callbacks + try: + should_run: Final = self.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="success_handler", + ) + if not should_run: + return + if callback == "promptlayer" and promptLayerLogger is not None: + print_verbose("reaches promptlayer for logging!") + promptLayerLogger.log_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + if callback == "supabase" and supabaseClient is not None: + print_verbose("reaches supabase for logging!") + kwargs = call_details # rebind-ok: later callbacks see these kwargs + + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + if "complete_streaming_response" not in kwargs: + return + else: + print_verbose("reaches supabase for streaming logging!") + result = call_details_view["complete_streaming_response"] # rebind-ok: seen by later callbacks + + model = call_details_view["model"] # rebind-ok: each integration branch binds its own + messages = call_details_view["messages"] # rebind-ok: each integration branch binds its own + optional_params: Final = call_details_view.get("optional_params", {}) + litellm_params = call_details_view.get( # rebind-ok: later callbacks see the rebound params + "litellm_params", {} + ) + supabaseClient.log_event( + model=model, + messages=messages, + end_user=require_attributes(optional_params, _ParamsLookup, "get").get("user", "default"), + response_obj=result, + start_time=start_time, + end_time=end_time, + litellm_call_id=( + current_call_id + if ( + current_call_id := require_attributes(litellm_params, _ParamsLookup, "get").get( + "litellm_call_id", None + ) + ) + is not None + else str(uuid.uuid4()) + ), + print_verbose=print_verbose, + ) + if callback == "wandb" and weightsBiasesLogger is not None: + print_verbose("reaches wandb for logging!") + weightsBiasesLogger.log_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + if callback == "logfire" and logfireLogger is not None: + verbose_logger.debug("reaches logfire for success logging!") + kwargs = {} # rebind-ok: later callbacks see these kwargs + for k, v in call_details_view.items(): + if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + if "complete_streaming_response" not in kwargs: + return + else: + print_verbose("reaches logfire for streaming logging!") + result = call_details_view["complete_streaming_response"] # rebind-ok: seen by later callbacks + + logfireLogger.log_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + level=LogfireLevel.INFO, + ) + + if callback == "lunary" and lunaryLogger is not None: + print_verbose("reaches lunary for logging!") + model = self.model # rebind-ok: each integration branch binds its own + kwargs = call_details # rebind-ok: later callbacks see these kwargs + + input: Final = kwargs.get("messages", kwargs.get("input", None)) + + type: Final = "embed" if self.call_type == CallTypes.embedding.value else "llm" + + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + if "complete_streaming_response" not in kwargs: + return + else: + result = call_details_view["complete_streaming_response"] # rebind-ok: seen by later callbacks + + lunaryLogger.log_event( + type=type, + kwargs=kwargs, + event="end", + model=model, + input=input, + user_id=kwargs.get("user", None), + # user_props=self.model_call_details.get("user_props", None), + extra=call_details_view.get("optional_params", {}), + response_obj=result, + start_time=start_time, + end_time=end_time, + run_id=self.litellm_call_id, + print_verbose=print_verbose, + ) + if callback == "helicone" and heliconeLogger is not None: + print_verbose("reaches helicone for logging!") + model = self.model # rebind-ok: each integration branch binds its own + messages = call_details_view["input"] # rebind-ok: each integration branch binds its own + kwargs = call_details # rebind-ok: later callbacks see these kwargs + + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + if "complete_streaming_response" not in kwargs: + return + else: + print_verbose("reaches helicone for streaming logging!") + result = call_details_view["complete_streaming_response"] # rebind-ok: seen by later callbacks + + heliconeLogger.log_success( + model=model, + messages=messages, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + kwargs=kwargs, + ) + if callback == "langfuse": + global langFuseLogger + print_verbose("reaches langfuse for success logging!") + kwargs = {} # rebind-ok: later callbacks see these kwargs + for k, v in call_details_view.items(): + if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + verbose_logger.debug( + "is complete_streaming_response in kwargs: %s", + kwargs.get("complete_streaming_response", None), + ) + if complete_streaming_response is None: + return + else: + print_verbose("reaches langfuse for streaming logging!") + result = call_details_view["complete_streaming_response"] # rebind-ok: seen by later callbacks + + langfuse_logger_to_use: Final = LangFuseHandler.get_langfuse_logger_for_request( + globalLangfuseLogger=langFuseLogger, + standard_callback_dynamic_params=self.standard_callback_dynamic_params, + in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache, + ) + if langfuse_logger_to_use is not None: + _response: Final = langfuse_logger_to_use.log_event_on_langfuse( + kwargs=kwargs, + response_obj=result, + start_time=start_time, + end_time=end_time, + user_id=kwargs.get("user", None), # pyright: ignore[reportArgumentType] # logger takes the user id as given + ) + if _response is not None and isinstance(_response, dict): + _trace_id: Final = _response.get("trace_id", None) + if _trace_id is not None: + in_memory_trace_id_cache.set_cache( + litellm_call_id=self.litellm_call_id, + service_name="langfuse", + trace_id=_trace_id, + ) + if callback == "greenscale" and greenscaleLogger is not None: + kwargs = {} # rebind-ok: later callbacks see these kwargs + for k, v in call_details_view.items(): + if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + verbose_logger.debug( + "is complete_streaming_response in kwargs: %s", + kwargs.get("complete_streaming_response", None), + ) + if complete_streaming_response is None: + return + else: + print_verbose("reaches greenscale for streaming logging!") + result = call_details_view["complete_streaming_response"] # rebind-ok: seen by later callbacks + + greenscaleLogger.log_event( + kwargs=kwargs, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + if callback == "athina" and athinaLogger is not None: + deep_copy = {} # rebind-ok: each integration branch binds its own + for k, v in call_details_view.items(): + deep_copy[k] = v + athinaLogger.log_event( + kwargs=deep_copy, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + if callback == "traceloop": + deep_copy = {} # rebind-ok: each integration branch binds its own + for k, v in call_details_view.items(): + if k != "original_response": + deep_copy[k] = v + traceloopLogger.log_event( + kwargs=deep_copy, + response_obj=result, + start_time=start_time, + end_time=end_time, + user_id=kwargs.get("user", None), + print_verbose=print_verbose, + ) + if callback == "s3": + global s3Logger + if s3Logger is None: + s3Logger = S3Logger() # rebind-ok: lazily built module logger + if self.stream: + if "complete_streaming_response" in call_details: + print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response") + s3Logger.log_event( + kwargs=call_details, + response_obj=call_details_view["complete_streaming_response"], + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + else: + print_verbose("S3Logger Logger: Got Stream Event - No complete stream response as yet") + else: + s3Logger.log_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + + if callback == "openmeter" and is_sync_request: + global openMeterLogger + if openMeterLogger is None: + print_verbose("Instantiates openmeter client") + openMeterLogger = OpenMeterLogger() # rebind-ok: lazily built module logger + if self.stream and complete_streaming_response is None: + openMeterLogger.log_stream_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + else: + if self.stream and complete_streaming_response: + call_details["complete_response"] = call_details.get("complete_streaming_response", {}) + result = call_details_view["complete_response"] # rebind-ok: seen by later callbacks + openMeterLogger.log_success_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + if ( + isinstance(callback, CustomLogger) + and (is_sync_request or (for_async_request and not self.stream)) + and not (for_async_request and self.stream and callback in (self.dynamic_async_success_callbacks or ())) + and self.call_type + != CallTypes.pass_through.value # pass-through endpoints call async_log_success_event + ): # custom logger class + if self.stream and complete_streaming_response is None: + callback.log_stream_event( + kwargs=redacted_model_call_details_for_custom_logger( + model_call_details=call_details, custom_logger=callback + ), + response_obj=redacted_response_obj_for_custom_logger(result, callback), + start_time=start_time, + end_time=end_time, + ) + else: + if self.stream and complete_streaming_response: + call_details["complete_response"] = call_details.get("complete_streaming_response", {}) + result = call_details_view["complete_response"] # rebind-ok: seen by later callbacks + + callback.log_success_event( + kwargs=redacted_model_call_details_for_custom_logger( + model_call_details=callback.redact_standard_logging_payload_from_model_call_details( + call_details + ), + custom_logger=callback, + ), + response_obj=redacted_response_obj_for_custom_logger(result, callback), + start_time=start_time, + end_time=end_time, + ) + if callable(callback) is True and is_sync_request: # custom logger functions + function_logger: Final = customLogger if customLogger is not None else CustomLogger() + print_verbose(f"success callbacks: Running Custom Callback Function - {callback}") + + function_logger.log_event( + kwargs=call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + callback_func=callback, + ) + + except Exception as e: + print_verbose( + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging with integrations {traceback.format_exc()}" + ) + print_verbose(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") + if capture_exception: # log this error to sentry for debugging + capture_exception(e) + # Track callback logging failures in Prometheus + try: + self._handle_callback_failure(callback=callback) + except Exception: + pass + finally: + run.result, run.kwargs, run.litellm_params = result, kwargs, litellm_params + async def async_success_handler( self, result: object = None, # heterogeneous response object; varies by call type (ANN401 ignored, see ruff-strict.toml) @@ -4261,7 +4452,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own success logging (including any nested calls its callbacks trigger) is fully done.""" try: - with post_response_phase(), outside_mirrored_call(): + with post_response_phase(), outside_mirrored_call(), self._parent_otel_span_anchored(): return await self._async_success_handler_body( result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs ) @@ -4474,183 +4665,26 @@ class Logging(LiteLLMLoggingBaseClass): ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) - for callback in hook_callbacks: - try: - if isinstance(callback, CustomGuardrail): - from litellm.types.guardrails import GuardrailEventHooks - - if ( - callback.should_run_guardrail( - data=self.model_call_details, - event_type=GuardrailEventHooks.logging_only, - ) - is not True - ): - continue - - self.shadow_eval_request_snapshot = None - self.model_call_details, result = await callback.async_logging_hook( - kwargs=self.model_call_details, - result=result, - call_type=self.call_type, - ) - elif isinstance(callback, CustomLogger): - result = redact_message_input_output_from_custom_logger( - result=result, litellm_logging_obj=self, custom_logger=callback - ) - self.model_call_details, result = await callback.async_logging_hook( - kwargs=self.model_call_details, - result=result, - call_type=self.call_type, - ) - except Exception: # noqa: BLE001 # one failing hook must not skip later callbacks (slot release) - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred in async_logging_hook %s", - traceback.format_exc(), - ) - self._handle_callback_failure(callback=callback) + include_v2: Final = self.stream is not True or "async_complete_streaming_response" in self.model_call_details + prepared: Final = _LogHookRun(details=self.model_call_details, result=result) + await _run_callbacks( + Hook.log_prepare, + hook_callbacks, + self.model_call_details, + functools.partial(self._async_logging_hook_start, prepared), + include_v2=include_v2, + ) + result = prepared.result # rebind-ok: logging hooks may replace the result self.mark_logging_complete(event_type="async_success") - for callback in delivery_callbacks: - # check if callback can run for this request - litellm_params = self._model_call_details_view.get("litellm_params", {}) - should_run = self.should_run_callback( - callback=callback, - litellm_params=litellm_params, - event_hook="async_success_handler", - ) - if not should_run: - continue - try: - if callback == "openmeter" and openMeterLogger is not None: - if self.stream is True: - if "async_complete_streaming_response" in self.model_call_details: - await openMeterLogger.async_log_success_event( - kwargs=self.model_call_details, - response_obj=self.model_call_details["async_complete_streaming_response"], - start_time=start_time, - end_time=end_time, - ) - else: - await openMeterLogger.async_log_stream_event( # [TODO]: move this to being an async log stream event function - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - else: - await openMeterLogger.async_log_success_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - - if isinstance(callback, CustomLogger): # custom logger class - from litellm.integrations.shadow_eval_logger import ShadowEvalLogger - - model_call_details: dict = ( - self._model_call_details_with_request_messages() - if callback.exempt_from_message_redaction() - else self.model_call_details - ) - ################################## - # call redaction hook for custom logger - model_call_details = callback.redact_standard_logging_payload_from_model_call_details( - model_call_details=model_call_details - ) - model_call_details = redacted_model_call_details_for_custom_logger( - model_call_details=model_call_details, custom_logger=callback - ) - ################################## - if isinstance(callback, ShadowEvalLogger) and ( - not self.stream or "async_complete_streaming_response" in model_call_details - ): - await callback.async_log_success_event( - kwargs=model_call_details, - response_obj=model_call_details["async_complete_streaming_response"] - if self.stream - else redacted_response_obj_for_custom_logger(result, callback), - start_time=start_time, - end_time=end_time, - guardrail_snapshot=self.shadow_eval_request_snapshot, - ) - elif self.stream is True: - if "async_complete_streaming_response" in model_call_details: - await callback.async_log_success_event( - kwargs=model_call_details, - response_obj=model_call_details["async_complete_streaming_response"], - start_time=start_time, - end_time=end_time, - ) - else: - await callback.async_log_stream_event( # [TODO]: move this to being an async log stream event function - kwargs=model_call_details, - response_obj=redacted_response_obj_for_custom_logger(result, callback), - start_time=start_time, - end_time=end_time, - ) - else: - await callback.async_log_success_event( - kwargs=model_call_details, - response_obj=redacted_response_obj_for_custom_logger(result, callback), - start_time=start_time, - end_time=end_time, - ) - if callable(callback): # custom logger functions - global customLogger - if customLogger is None: - customLogger = CustomLogger() - if self.stream: - if "async_complete_streaming_response" in self.model_call_details: - await customLogger.async_log_event( - kwargs=self.model_call_details, - response_obj=self.model_call_details["async_complete_streaming_response"], - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - callback_func=callback, - ) - else: - await customLogger.async_log_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - callback_func=callback, - ) - if callback == "dynamodb": - global dynamoLogger - if dynamoLogger is None: - dynamoLogger = DyanmoDBLogger() - if self.stream: - if "async_complete_streaming_response" in self.model_call_details: - print_verbose("DynamoDB Logger: Got Stream Event - Completed Stream Response") - await dynamoLogger.async_log_event( - kwargs=self.model_call_details, - response_obj=self.model_call_details["async_complete_streaming_response"], - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - else: - print_verbose("DynamoDB Logger: Got Stream Event - No complete stream response as yet") - else: - await dynamoLogger.async_log_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - except Exception: - verbose_logger.error( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", - traceback.format_exc(), - ) - self._handle_callback_failure(callback=callback) + await _run_callbacks( + Hook.attempt_ended, + delivery_callbacks, + self.model_call_details, + functools.partial(self._deliver_async_success, result, start_time, end_time), + include_v2=include_v2, + ) if not self.stream: try: self.handle_sync_success_callbacks_for_async_calls( @@ -4660,6 +4694,199 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.exception("Could not schedule synchronous success callbacks") await flush_post_call_redis_batches() + def _async_logging_hook_start( + self, prepared: _LogHookRun, callback: object, _kwargs: object + ) -> Awaitable[Patch | None] | None: + return self._async_logging_hook(prepared, callback) if isinstance(callback, CustomLogger) else None + + async def _async_logging_hook(self, prepared: _LogHookRun, callback: CustomLogger) -> Patch | None: + before: Final = self.model_call_details + try: + if isinstance(callback, CustomGuardrail): + from litellm.types.guardrails import GuardrailEventHooks + + if ( + callback.should_run_guardrail( + data=self.model_call_details, + event_type=GuardrailEventHooks.logging_only, + ) + is not True + ): + return None + + self.shadow_eval_request_snapshot = None + self.model_call_details, prepared.result = await callback.async_logging_hook( + kwargs=self.model_call_details, + result=prepared.result, + call_type=self.call_type, + ) + else: + prepared.result = redact_message_input_output_from_custom_logger( + result=prepared.result, litellm_logging_obj=self, custom_logger=callback + ) + self.model_call_details, prepared.result = await callback.async_logging_hook( + kwargs=self.model_call_details, + result=prepared.result, + call_type=self.call_type, + ) + except Exception: # noqa: BLE001 # one failing hook must not skip later callbacks (slot release) + verbose_logger.error( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred in async_logging_hook %s", + traceback.format_exc(), + ) + self._handle_callback_failure(callback=callback) + prepared.details = self.model_call_details + return kwargs_patch(before, self.model_call_details) + + def _deliver_async_success( + self, + result: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + callback: object, + _kwargs: object, + ) -> Awaitable[None] | None: + litellm_params: Final = self._model_call_details_view.get("litellm_params", {}) + should_run: Final = self.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="async_success_handler", + ) + return self._deliver_async_success_to(callback, result, start_time, end_time) if should_run else None + + async def _deliver_async_success_to( + self, + callback: object, + result: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + try: + if callback == "openmeter" and openMeterLogger is not None: + if self.stream is True: + if "async_complete_streaming_response" in self.model_call_details: + await openMeterLogger.async_log_success_event( + kwargs=self.model_call_details, + response_obj=self.model_call_details["async_complete_streaming_response"], + start_time=start_time, + end_time=end_time, + ) + else: + await openMeterLogger.async_log_stream_event( # [TODO]: move this to being an async log stream event function + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + else: + await openMeterLogger.async_log_success_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + + if isinstance(callback, CustomLogger): # custom logger class + from litellm.integrations.shadow_eval_logger import ShadowEvalLogger + + model_call_details: Final[dict] = redacted_model_call_details_for_custom_logger( + model_call_details=callback.redact_standard_logging_payload_from_model_call_details( + model_call_details=( + self._model_call_details_with_request_messages() + if callback.exempt_from_message_redaction() + else self.model_call_details + ) + ), + custom_logger=callback, + ) + if isinstance(callback, ShadowEvalLogger) and ( + not self.stream or "async_complete_streaming_response" in model_call_details + ): + await callback.async_log_success_event( + kwargs=model_call_details, + response_obj=model_call_details["async_complete_streaming_response"] + if self.stream + else redacted_response_obj_for_custom_logger(result, callback), + start_time=start_time, + end_time=end_time, + guardrail_snapshot=self.shadow_eval_request_snapshot, + ) + elif self.stream is True: + if "async_complete_streaming_response" in model_call_details: + await callback.async_log_success_event( + kwargs=model_call_details, + response_obj=model_call_details["async_complete_streaming_response"], + start_time=start_time, + end_time=end_time, + ) + else: + await callback.async_log_stream_event( # [TODO]: move this to being an async log stream event function + kwargs=model_call_details, + response_obj=redacted_response_obj_for_custom_logger(result, callback), + start_time=start_time, + end_time=end_time, + ) + else: + await callback.async_log_success_event( + kwargs=model_call_details, + response_obj=redacted_response_obj_for_custom_logger(result, callback), + start_time=start_time, + end_time=end_time, + ) + if callable(callback): # custom logger functions + global customLogger + if customLogger is None: + customLogger = CustomLogger() # rebind-ok: lazily built module logger + if self.stream: + if "async_complete_streaming_response" in self.model_call_details: + await customLogger.async_log_event( + kwargs=self.model_call_details, + response_obj=self.model_call_details["async_complete_streaming_response"], + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + callback_func=callback, + ) + else: + await customLogger.async_log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + callback_func=callback, + ) + if callback == "dynamodb": + global dynamoLogger + if dynamoLogger is None: + dynamoLogger = DyanmoDBLogger() # rebind-ok: lazily built module logger + if self.stream: + if "async_complete_streaming_response" in self.model_call_details: + print_verbose("DynamoDB Logger: Got Stream Event - Completed Stream Response") + await dynamoLogger.async_log_event( + kwargs=self.model_call_details, + response_obj=self.model_call_details["async_complete_streaming_response"], + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + else: + print_verbose("DynamoDB Logger: Got Stream Event - No complete stream response as yet") + else: + await dynamoLogger.async_log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + except Exception: + verbose_logger.error( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging %s", + traceback.format_exc(), + ) + self._handle_callback_failure(callback=callback) + def _handle_callback_failure(self, callback: object): """ Handle callback logging failures by incrementing Prometheus metrics. @@ -4682,7 +4909,9 @@ class Logging(LiteLLMLoggingBaseClass): except Exception as e: verbose_logger.debug("Error in _handle_callback_failure: %s", e) - def _failure_handler_helper_fn(self, exception, traceback_exception, start_time=None, end_time=None): + def _failure_handler_helper_fn( + self, exception, traceback_exception, start_time=None, end_time=None, *, estimate_prompt_tokens: bool = True + ): if start_time is None: start_time = self.start_time if end_time is None: @@ -4713,6 +4942,8 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = self.finalized_cost.amount elif self.model_call_details.get("combined_usage_object") is None: self.model_call_details["response_cost"] = 0 + if estimate_prompt_tokens and self._should_estimate_failed_prompt(): + self.model_call_details[FAILED_PROMPT_TOKEN_ESTIMATE_KEY] = self._prompt_token_estimate() raised_exception: Final[object] = exception exception_headers: Final = ( @@ -4747,6 +4978,43 @@ class Logging(LiteLLMLoggingBaseClass): ) return start_time, end_time + def _should_estimate_failed_prompt(self) -> bool: + return ( + self._model_call_details_view.get("call_type") in _CHAT_CALL_TYPES + and not isinstance(self.model_call_details.get("combined_usage_object"), Usage) + and self._failure_callbacks_want_prompt_token_estimate() + ) + + def _failure_callbacks_want_prompt_token_estimate(self) -> bool: + callbacks: Final = ( + *(self.dynamic_failure_callbacks or ()), + *(self.dynamic_async_failure_callbacks or ()), + *litellm.callbacks, + *litellm.logging_callback_manager.get_failure_callbacks(), + ) + return any(isinstance(cb, CustomLogger) and cb.wants_failed_prompt_token_estimate for cb in callbacks) + + def _prompt_token_estimate(self) -> int | None: + from litellm.litellm_core_utils.token_counter import has_only_text_content + + messages: Final = self.messages + if litellm.disable_token_counter or not messages: + return None + try: + if not has_only_text_content(messages): + return None + return get_token_counter_new()(model=self.model, messages=messages, use_default_image_token_count=True) + except (ValueError, TypeError, LookupError, AttributeError, RuntimeError, OSError): + return None + + async def _offloaded_prompt_token_estimate(self) -> int | None: + from litellm.litellm_core_utils.token_counter import offload_token_count + + try: + return await offload_token_count(self._prompt_token_estimate)() + except (RuntimeError, OSError): + return None + async def special_failure_handlers(self, exception: Exception): """ Custom events, emitted for specific failures. @@ -4794,7 +5062,7 @@ class Logging(LiteLLMLoggingBaseClass): ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: - with outside_mirrored_call(): + with outside_mirrored_call(), self._parent_otel_span_anchored(): return self._failure_handler_body( exception=exception, traceback_exception=traceback_exception, @@ -4825,6 +5093,7 @@ class Logging(LiteLLMLoggingBaseClass): traceback_exception=traceback_exception, start_time=start_time, end_time=end_time, + estimate_prompt_tokens=not for_async_request, ) callbacks: Final = self.get_combined_callback_list( dynamic_success_callbacks=self.dynamic_failure_callbacks, @@ -4840,150 +5109,176 @@ class Logging(LiteLLMLoggingBaseClass): ) ) self.mark_logging_complete(event_type="sync_failure") - for callback in callbacks: - try: - should_run = self.should_run_callback( - callback=callback, - litellm_params=litellm_params, - event_hook="failure_handler", - ) - if not should_run: - continue - if callback == "lunary" and lunaryLogger is not None: - print_verbose("reaches lunary for logging error!") - - model = self.model - - input = self._model_call_details_view["input"] - - _type = "embed" if self.call_type == CallTypes.embedding.value else "llm" - - lunaryLogger.log_event( - kwargs=self.model_call_details, - type=_type, - event="error", - user_id=self._model_call_details_view.get("user", "default"), - model=model, - input=input, - error=traceback_exception, - run_id=self.litellm_call_id, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) - if callback == "sentry": - print_verbose("sending exception to sentry") - if capture_exception: - capture_exception(exception) - else: - print_verbose(f"capture exception not initialized: {capture_exception}") - elif callback == "supabase" and supabaseClient is not None: - print_verbose("reaches supabase for logging!") - print_verbose(f"supabaseClient: {supabaseClient}") - supabaseClient.log_event( - model=self.model if hasattr(self, "model") else "", - messages=self.messages, - end_user=self._model_call_details_view.get("user", "default"), - response_obj=result, - start_time=start_time, - end_time=end_time, - litellm_call_id=self._model_call_details_view["litellm_call_id"], - print_verbose=print_verbose, - ) - if callable(callback): # custom logger functions - global customLogger - if customLogger is None: - customLogger = CustomLogger() - customLogger.log_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - callback_func=callback, - ) - if ( - isinstance(callback, CustomLogger) - and is_sync_request - and self.call_type != CallTypes.pass_through.value - and not (for_async_request and self._async_handler_logs_to(callback)) - ): # custom logger class - callback.log_failure_event( - start_time=start_time, - end_time=end_time, - response_obj=result, - kwargs=self.model_call_details, - ) - if callback == "langfuse": - global langFuseLogger - verbose_logger.debug("reaches langfuse for logging failure") - kwargs = {} - for k, v in self._model_call_details_view.items(): - if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine - kwargs[k] = v - # this only logs streaming once, complete_streaming_response exists i.e when stream ends - langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request( - globalLangfuseLogger=langFuseLogger, - standard_callback_dynamic_params=self.standard_callback_dynamic_params, - in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache, - ) - _response = langfuse_logger_to_use.log_event_on_langfuse( - start_time=start_time, - end_time=end_time, - response_obj=None, - user_id=kwargs.get("user", None), - status_message=str(exception), - level="ERROR", - kwargs=kwargs, - ) - if _response is not None and isinstance(_response, dict): - _trace_id = _response.get("trace_id", None) - if _trace_id is not None: - in_memory_trace_id_cache.set_cache( - litellm_call_id=self.litellm_call_id, - service_name="langfuse", - trace_id=_trace_id, - ) - if callback == "traceloop": - traceloopLogger.log_event( - start_time=start_time, - end_time=end_time, - response_obj=None, - user_id=self.model_call_details.get("user", None), - print_verbose=print_verbose, - status_message=str(exception), - level="ERROR", - kwargs=self.model_call_details, - ) - if callback == "logfire" and logfireLogger is not None: - verbose_logger.debug("reaches logfire for failure logging!") - kwargs = {} - for k, v in self._model_call_details_view.items(): - if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine - kwargs[k] = v - kwargs["exception"] = exception - - logfireLogger.log_event( - kwargs=kwargs, - response_obj=result, - start_time=start_time, - end_time=end_time, - level=LogfireLevel.ERROR, - print_verbose=print_verbose, - ) - - except Exception as e: - print_verbose( - f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {e}" - ) - print_verbose(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") - if capture_exception: # log this error to sentry for debugging - capture_exception(e) + run: Final = _FailureRun( + exception=exception, + traceback_exception=traceback_exception, + result=result, + start_time=start_time, # pyright: ignore[reportUnknownArgumentType] # untyped failure helper + end_time=end_time, # pyright: ignore[reportUnknownArgumentType] # untyped failure helper + litellm_params=litellm_params, + is_sync_request=is_sync_request, + for_async_request=for_async_request, + ) + _run_callbacks_sync( + Hook.attempt_ended, + callbacks, + self.model_call_details, + functools.partial(self._deliver_sync_failure, run), + include_v2=not for_async_request, + ) except Exception as e: verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s", e ) + def _deliver_sync_failure(self, run: _FailureRun, callback: object, _kwargs: object) -> None: + exception: Final = run.exception + traceback_exception: Final = run.traceback_exception + result: Final = run.result + start_time: Final = run.start_time + end_time: Final = run.end_time + litellm_params: Final = run.litellm_params + is_sync_request: Final = run.is_sync_request + for_async_request: Final = run.for_async_request + try: + should_run: Final = self.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="failure_handler", + ) + if not should_run: + return + if callback == "lunary" and lunaryLogger is not None: + print_verbose("reaches lunary for logging error!") + + model: Final = self.model + + input: Final = self._model_call_details_view["input"] + + _type: Final = "embed" if self.call_type == CallTypes.embedding.value else "llm" + + lunaryLogger.log_event( + kwargs=self.model_call_details, + type=_type, + event="error", + user_id=self._model_call_details_view.get("user", "default"), + model=model, + input=input, + error=traceback_exception, + run_id=self.litellm_call_id, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) + if callback == "sentry": + print_verbose("sending exception to sentry") + if capture_exception: + capture_exception(exception) + else: + print_verbose(f"capture exception not initialized: {capture_exception}") + elif callback == "supabase" and supabaseClient is not None: + print_verbose("reaches supabase for logging!") + print_verbose(f"supabaseClient: {supabaseClient}") + supabaseClient.log_event( + model=self.model if hasattr(self, "model") else "", + messages=self.messages, + end_user=self._model_call_details_view.get("user", "default"), + response_obj=result, + start_time=start_time, + end_time=end_time, + litellm_call_id=self._model_call_details_view["litellm_call_id"], + print_verbose=print_verbose, + ) + if callable(callback): # custom logger functions + global customLogger + if customLogger is None: + customLogger = CustomLogger() # rebind-ok: lazily built module logger + customLogger.log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + callback_func=callback, + ) + if ( + isinstance(callback, CustomLogger) + and is_sync_request + and self.call_type != CallTypes.pass_through.value + and not (for_async_request and self._async_handler_logs_to(callback)) + ): # custom logger class + callback.log_failure_event( + start_time=start_time, + end_time=end_time, + response_obj=result, + kwargs=self.model_call_details, + ) + if callback == "langfuse": + global langFuseLogger + verbose_logger.debug("reaches langfuse for logging failure") + kwargs = {} # rebind-ok: each integration branch binds its own + for k, v in self._model_call_details_view.items(): + if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + langfuse_logger_to_use: Final = LangFuseHandler.get_langfuse_logger_for_request( + globalLangfuseLogger=langFuseLogger, + standard_callback_dynamic_params=self.standard_callback_dynamic_params, + in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache, + ) + _response: Final = langfuse_logger_to_use.log_event_on_langfuse( + start_time=start_time, + end_time=end_time, + response_obj=None, + user_id=kwargs.get("user", None), + status_message=str(exception), + level="ERROR", + kwargs=kwargs, + ) + if _response is not None and isinstance(_response, dict): + _trace_id: Final = _response.get("trace_id", None) + if _trace_id is not None: + in_memory_trace_id_cache.set_cache( + litellm_call_id=self.litellm_call_id, + service_name="langfuse", + trace_id=_trace_id, + ) + if callback == "traceloop": + traceloopLogger.log_event( + start_time=start_time, + end_time=end_time, + response_obj=None, + user_id=self.model_call_details.get("user", None), + print_verbose=print_verbose, + status_message=str(exception), + level="ERROR", + kwargs=self.model_call_details, + ) + if callback == "logfire" and logfireLogger is not None: + verbose_logger.debug("reaches logfire for failure logging!") + kwargs = {} # rebind-ok: each integration branch binds its own + for k, v in self._model_call_details_view.items(): + if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + kwargs["exception"] = exception + + logfireLogger.log_event( + kwargs=kwargs, + response_obj=result, + start_time=start_time, + end_time=end_time, + level=LogfireLevel.ERROR, + print_verbose=print_verbose, + ) + + except Exception as e: + print_verbose( + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {e}" + ) + print_verbose(f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}") + if capture_exception: # log this error to sentry for debugging + capture_exception(e) + async def async_failure_handler( self, exception: Exception, @@ -4994,7 +5289,7 @@ class Logging(LiteLLMLoggingBaseClass): """Restores trace_id/session_id contextvars once this attempt's own failure logging (including any nested calls its callbacks trigger) is fully done.""" try: - with outside_mirrored_call(): + with outside_mirrored_call(), self._parent_otel_span_anchored(): return await self._async_failure_handler_body( exception=exception, traceback_exception=traceback_exception, @@ -5024,7 +5319,10 @@ class Logging(LiteLLMLoggingBaseClass): traceback_exception=traceback_exception, start_time=start_time, end_time=end_time, + estimate_prompt_tokens=False, ) + if self._should_estimate_failed_prompt(): + self.model_call_details[FAILED_PROMPT_TOKEN_ESTIMATE_KEY] = await self._offloaded_prompt_token_estimate() if self._on_failure_payload_built is not None: self._on_failure_payload_built() callbacks: Final = self.get_combined_callback_list( @@ -5032,48 +5330,64 @@ class Logging(LiteLLMLoggingBaseClass): global_callbacks=litellm._async_failure_callback, ) - result: Final = None # result sent to all loggers, init this to None incase it's not created - self.mark_logging_complete(event_type="async_failure") - for callback in callbacks: - try: - litellm_params = self._model_call_details_view.get("litellm_params", {}) - should_run = self.should_run_callback( - callback=callback, - litellm_params=litellm_params, - event_hook="async_failure_handler", - ) - if not should_run: - continue - if isinstance(callback, CustomLogger): # custom logger class - await callback.async_log_failure_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - if callable(callback): # custom logger functions - global customLogger - if customLogger is None: - customLogger = CustomLogger() - await customLogger.async_log_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - callback_func=callback, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s\nCallback=%s", - e, - callback, - ) - # Track callback logging failures in Prometheus - self._handle_callback_failure(callback=callback) + await _run_callbacks( + Hook.attempt_ended, + callbacks, + self.model_call_details, + functools.partial( + self._deliver_async_failure, + start_time, # pyright: ignore[reportUnknownArgumentType] # untyped failure helper + end_time, # pyright: ignore[reportUnknownArgumentType] # untyped failure helper + ), + ) await flush_post_call_redis_batches() + async def _deliver_async_failure( + self, + start_time: datetime.datetime, + end_time: datetime.datetime | None, + callback: object, + _kwargs: object, + ) -> None: + result: Final = None # result sent to all loggers, init this to None incase it's not created + try: + litellm_params: Final = self._model_call_details_view.get("litellm_params", {}) + should_run: Final = self.should_run_callback( + callback=callback, + litellm_params=litellm_params, + event_hook="async_failure_handler", + ) + if not should_run: + return + if isinstance(callback, CustomLogger): # custom logger class + await callback.async_log_failure_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + if callable(callback): # custom logger functions + global customLogger + if customLogger is None: + customLogger = CustomLogger() # rebind-ok: lazily built module logger + await customLogger.async_log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + callback_func=callback, + ) + except Exception as e: + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging %s\nCallback=%s", + e, + callback, + ) + # Track callback logging failures in Prometheus + self._handle_callback_failure(callback=callback) + def get_trace_id(self, service_name: Literal["langfuse"]) -> str | None: """ For the given service (e.g. langfuse), return the trace_id actually logged. @@ -5400,6 +5714,10 @@ class Logging(LiteLLMLoggingBaseClass): def record_streamed_anthropic_message_id(self, message_id: str) -> None: self.streamed_anthropic_message_id = message_id + def record_served_agentic_follow_up(self) -> None: + """This call served an agentic follow-up's reply, so its own spend row is keyed by its upstream reply.""" + self.served_agentic_follow_up = True + def record_bridged_model_response(self, response: ModelResponse) -> None: """The chat completion a /v1/messages response was translated from. @@ -5515,6 +5833,8 @@ class Logging(LiteLLMLoggingBaseClass): json_mode=False, litellm_params={}, ) + if self.served_agentic_follow_up: + return self._first_round_row(result, httpx_response) elif result is None: verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one") return litellm.ModelResponse(model=self.model) @@ -5539,6 +5859,15 @@ class Logging(LiteLLMLoggingBaseClass): set_response_cost_in_hidden_params(result, reported_cost) return result + def _first_round_row(self, reparsed: ModelResponse, upstream: Response) -> ModelResponse: + body: Final = _LITELLM_PARAMS.validate_python(upstream.json()) + upstream_id: Final = _provider_response_id(body) + row: Final = reparsed if upstream_id is None else reparsed.model_copy(update={"id": upstream_id}) + set_response_cost_in_hidden_params( + row, native_messages_reported_cost(body.get("usage"), self.model_call_details.get("custom_llm_provider")) + ) + return row + def _translate_responses_api_response_to_model_response( self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None ) -> ModelResponse: @@ -7076,27 +7405,21 @@ class StandardLoggingPayloadSetup: @staticmethod def append_system_prompt_messages(kwargs: dict | None = None, messages: Any | None = None): """ - Append system prompt messages to the messages + Prepend the request's top-level `system` prompt, text or content blocks, to the logged messages """ - if kwargs is not None: - if kwargs.get("system") is not None and isinstance(kwargs.get("system"), str): - if messages is None: - return [{"role": "system", "content": kwargs.get("system")}] - elif isinstance(messages, list): - if len(messages) == 0: - return [{"role": "system", "content": kwargs.get("system")}] - # check for duplicates - if messages[0].get("role") == "system" and messages[0].get("content") == kwargs.get("system"): - return messages - messages = [{"role": "system", "content": kwargs.get("system")}] + messages - elif isinstance(messages, str): - messages = [ - {"role": "system", "content": kwargs.get("system")}, - {"role": "user", "content": messages}, - ] - return messages - - return messages + system: Final = kwargs.get("system") if kwargs is not None else None + if not isinstance(system, str) and not (isinstance(system, list) and system): + return messages + system_message: Final = ChatCompletionSystemMessage(role="system", content=system) + if messages is None: + return [system_message] + if isinstance(messages, str): + return [system_message, {"role": "user", "content": messages}] + if not isinstance(messages, list): + return messages + if messages and messages[0].get("role") == "system" and messages[0].get("content") == system: + return messages + return [system_message, *messages] @staticmethod def merge_litellm_metadata(litellm_params: Mapping[str, object]) -> dict[str, object]: @@ -7196,6 +7519,9 @@ class StandardLoggingPayloadSetup: user_api_key_team_alias=None, user_api_key_user_email=None, user_api_key_end_user_id=None, + user_api_key_customer_member_id=None, + user_api_key_customer_team_id=None, + user_api_key_customer_org_id=None, user_api_key_request_route=None, spend_logs_metadata=None, requester_ip_address=None, @@ -8340,6 +8666,9 @@ def get_standard_logging_metadata( user_agent=None, requester_metadata=None, user_api_key_end_user_id=None, + user_api_key_customer_member_id=None, + user_api_key_customer_team_id=None, + user_api_key_customer_org_id=None, prompt_management_metadata=None, applied_guardrails=None, mcp_tool_call_metadata=None, diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 56c6436eae6..70aca83e70f 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -428,6 +428,9 @@ class LoggingCallbackManager: _get_all_callbacks = get_all_callbacks + def get_failure_callbacks(self) -> tuple[CustomLogger | Callable[..., object] | str, ...]: + return (*litellm.failure_callback, *litellm._async_failure_callback) + def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None: """ Remove a callback object from every callback list it may have been @@ -489,7 +492,7 @@ class LoggingCallbackManager: """ # Get callback lists success_callbacks: Final = set(litellm.success_callback + litellm._async_success_callback) - failure_callbacks: Final = set(litellm.failure_callback + litellm._async_failure_callback) + failure_callbacks: Final = set(self.get_failure_callbacks()) general_callbacks: Final = set(litellm.callbacks) # Get all unique callbacks diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index f4ba3f43712..cbee7c818c5 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -318,6 +318,7 @@ class _BodyPrefixReader: headers: Mapping[str, str] | None = None, follow_redirects: bool = False, timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, ) -> Response: with self.client.stream( "GET", @@ -342,6 +343,7 @@ class _ImageBodyReader: headers: Mapping[str, str] | None = None, follow_redirects: bool = False, timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, ) -> Response: with self.client.stream( "GET", diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 2b782f0a4b9..f4648fafb47 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -14,11 +14,13 @@ from litellm.rust_bridge import diagnostics REDACTED: Final = "REDACTED" +_PEM_PRIVATE_KEY_BLOCK: Final = r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----" + def _build_secret_patterns() -> "re.Pattern[str]": patterns: Final[list[str]] = [ # PEM private key / certificate blocks - r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", + _PEM_PRIVATE_KEY_BLOCK, # GCP OAuth2 access tokens (ya29.*) r"\bya29\.[A-Za-z0-9_.~+/-]+", # Credential %s formatting (space separator, no key= prefix) @@ -100,6 +102,57 @@ def redact_string(value: str) -> str: return diagnostics.run(lambda native: native.redact_text(value), lambda: python_redact_string(value)) +_TOKEN_SHAPES: Final = ( + r"ya29\.[A-Za-z0-9_.~+/-]{20,}", + r"(?:AKIA|ASIA)[0-9A-Z]{16}\b", + rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}", + r"(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{24,}", + r"AIza[0-9A-Za-z\-_]{35}", + r"dapi[0-9a-f]{32}\b", + r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}", + r"github_pat_[A-Za-z0-9_]{82}", + r"glpat-[A-Za-z0-9\-_]{20,}", + r"lin_api_[A-Za-z0-9]{40}\b", + r"xox[abposr]-[A-Za-z0-9-]{10,}", + r"hf_[A-Za-z0-9]{34,}", + r"npm_[A-Za-z0-9]{36}\b", + r"pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,}", + r"SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}(?![A-Za-z0-9_-])", + r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", +) +# Tokens start at a word boundary or right after a JSON escape like "\n", since tool output is often serialized JSON +_TOKEN_START: Final = r"(?:(?eyJ(?:(?!-----BEGIN|{_DOTTED_TOKEN})[A-Za-z0-9_-])*" + rf"(?:{_DOTTED_TOKEN})?[^A-Za-z0-9_-]?)" +) +_TOKEN_RE: Final = re.compile(f"{_NON_JWT_TOKEN_RE.pattern}|{_TOKEN_START}(?:{_TOKEN_SHAPES[-1]}|{_UNMATCHED_JWT})") + + +def redact_tokens(value: str, replacement: str = REDACTED) -> str: + """Replace substrings shaped like a credential (provider key prefixes, JWTs, PEM private keys). + + Unlike redact_string() this never matches on surrounding words such as ``password:`` or ``Basic``, + so it is safe to run over free-form prompt and tool content. + """ + + def replace(match: re.Match[str]) -> str: + if match.lastgroup == "unmatched_jwt": + return _NON_JWT_TOKEN_RE.sub(replacement, match.group()) + return match.expand(replacement) + + return _TOKEN_RE.sub(replace, value) + + _UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+" _WINDOWS_DRIVE_PATH: Final = r"[A-Za-z]:\\[^\s'\"\)\]}>,]+" _PRIVATE_OR_LOOPBACK_IPV4: Final = ( diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e56dd83a277..11b02b9272b 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -17,7 +17,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar @@ -31,6 +31,7 @@ from litellm.constants import ( DEFAULT_IMAGE_HEIGHT, DEFAULT_IMAGE_TOKEN_COUNT, DEFAULT_IMAGE_WIDTH, + DEFAULT_MAX_RECURSE_DEPTH, MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, @@ -47,7 +48,7 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping from litellm.litellm_core_utils.prompt_templates.chat_template_tools import ( ChatTemplateToolSection, chat_template_tool_section, @@ -928,7 +929,6 @@ def token_counter( if litellm.disable_token_counter is True: return 0 - verbose_logger.debug("messages in token_counter: %s, text in token_counter: %s", messages, text) if text is not None and messages is not None: raise ValueError("text and messages cannot both be set") if use_default_image_token_count is None: @@ -1758,31 +1758,72 @@ def _count_file_tokens( ) +_ANTHROPIC_SERVER_TOOL_FIELDS: Final = MappingProxyType( + { + "server_tool_use": ("name", "input"), + "mcp_tool_use": ("name", "input"), + "web_search_tool_result": ("content",), + "web_fetch_tool_result": ("content",), + "code_execution_tool_result": ("content",), + "bash_code_execution_tool_result": ("content",), + "text_editor_code_execution_tool_result": ("content",), + "mcp_tool_result": ("content",), + "web_search_result": ("url", "title", "snippet"), + "web_fetch_result": ("url", "content"), + "code_execution_result": ("stdout", "stderr", "content"), + "bash_code_execution_result": ("stdout", "stderr", "content"), + "text_editor_code_execution_view_result": ("content",), + "text_editor_code_execution_str_replace_result": ("lines",), + "text_editor_code_execution_create_result": (), + "encrypted_code_execution_result": ("stderr", "content"), + "code_execution_output": (), + "bash_code_execution_output": (), + "web_search_tool_result_error": ("error_code",), + "web_fetch_tool_result_error": ("error_code",), + "code_execution_tool_result_error": ("error_code",), + "bash_code_execution_tool_result_error": ("error_code",), + "text_editor_code_execution_tool_result_error": ("error_code", "error_message"), + } +) +_ANTHROPIC_DOCUMENT: Final = TypeAdapter(AnthropicMessagesDocumentParam) + + def _count_anthropic_content( content: Mapping[str, object], count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, pdf_pricing: PdfPagePricing = "claude", + *, + _content_depth: int = 0, ) -> int: """ Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). - Uses TypedDict definitions from litellm.types.llms.anthropic to determine - what fields to count and how to handle nested structures. - - Dynamically infers which fields to count based on the TypedDict definition, - avoiding hardcoded field names. + Uses readable server tool fields or the client tool TypedDict definitions. """ - typeddict_cls: Final = _validate_anthropic_content(content) - type_hints: Final[Mapping[str, object]] = getattr(typeddict_cls, "__annotations__", {}) + if _content_depth >= DEFAULT_MAX_RECURSE_DEPTH: + raise UnsupportedContentTypeError("Anthropic tool content nesting exceeds the token counter limit") + content_type: Final = content.get("type") + if content_type == "document": + return _count_document_tokens( + _ANTHROPIC_DOCUMENT.validate_python(content), + count_function, + use_default_image_token_count, + default_token_count, + pdf_pricing, + ) + server_fields: Final = _ANTHROPIC_SERVER_TOOL_FIELDS.get(content_type) if isinstance(content_type, str) else None + fields: Final[Mapping[str, object]] = ( + dict.fromkeys(server_fields) + if server_fields is not None + else getattr(_validate_anthropic_content(content), "__annotations__", {}) + ) tokens = 0 - # Fields to skip (metadata/identifiers that don't contribute to prompt tokens) skip_fields: Final = {"type", "id", "tool_use_id", "cache_control", "is_error"} - # Iterate over all fields defined in the TypedDict - for field_name, field_type in type_hints.items(): + for field_name in fields: if field_name in skip_fields: continue @@ -1801,7 +1842,21 @@ def _count_anthropic_content( pdf_pricing, ) elif isinstance(field_value, dict): - tokens += count_function(str(field_value)) + if ( + server_fields is not None + and field_name == "content" + and (field_mapping := as_str_mapping(content[field_name])) is not None + ): + tokens += _count_anthropic_content( + field_mapping, + count_function, + use_default_image_token_count, + default_token_count, + pdf_pricing, + _content_depth=_content_depth + 1, + ) + else: + tokens += count_function(str(field_value)) except Exception as e: if default_token_count is not None: return default_token_count @@ -1816,6 +1871,29 @@ _TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType( ) +def _is_text_block(block: object) -> bool: + mapping: Final = as_str_mapping(block) + block_type: Final = mapping.get("type") if mapping is not None else None + return isinstance(block, str) or (isinstance(block_type, str) and block_type in _TEXT_FIELD_BY_BLOCK_TYPE) + + +def _has_only_text_content(message: object) -> bool: + normalized: Final[object] = message.model_dump(exclude_none=True) if isinstance(message, BaseModel) else message + mapping: Final = as_str_mapping(normalized) + if mapping is None: + return False + content: Final = mapping.get("content") + if content is None or isinstance(content, str): + return True + blocks: Final = as_object_list(content) + return blocks is not None and all(_is_text_block(block) for block in blocks) + + +def has_only_text_content(messages: Sequence[object]) -> bool: + """Whether counting ``messages`` stays on the tokenizer, never decoding images, files or documents.""" + return all(_has_only_text_content(message) for message in messages) + + def _count_content_list( count_function: TokenCounterFunction, content_list: str @@ -1867,7 +1945,9 @@ def _count_content_list( use_default_image_token_count, pdf_pricing, ) - elif c["type"] in ("tool_use", "tool_result", "search_result"): + elif ( + c["type"] in ("tool_use", "tool_result", "search_result") or c["type"] in _ANTHROPIC_SERVER_TOOL_FIELDS + ): num_tokens += _count_anthropic_content( c, count_function, @@ -1886,13 +1966,16 @@ def _count_content_list( tool_name = str(c.get("tool_name") or "") if tool_name: num_tokens += count_function(tool_name) + elif (change := as_str_mapping(c)) is not None and change.get("type") in ("tool_addition", "tool_removal"): + num_tokens += _tool_change_tokens(count_function, as_str_mapping(change.get("tool"))) else: content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__ raise UnsupportedContentTypeError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"compaction, tool_reference, search_result)." + f"compaction, tool_reference, tool_addition, tool_removal, search_result, " + f"or Anthropic server tool blocks)." ) return num_tokens except Exception as e: @@ -1904,6 +1987,24 @@ def _count_content_list( ) +def _tool_change_tokens(count_function: TokenCounterFunction, tool: Mapping[str, object] | None) -> int: + match tool: + case {"type": "tool_definition", "definition": definition} if ( + function := as_str_mapping(definition) + ) is not None: + return count_function(_format_function_definitions((function,))) + case {"type": "tool_reference", "name": str() as name}: + return count_function(name) + case {"type": "mcp_tool_reference", "server_name": str() as server_name, "name": str() as name}: + return count_function(server_name) + count_function(name) + case {"type": "mcp_toolset_reference", "server_name": str() as server_name}: + return count_function(server_name) + case _: + raise UnsupportedContentTypeError( + f"Invalid tool change target type: {None if tool is None else tool.get('type')}" + ) + + def _format_function_definitions(tools: Sequence[object]) -> str: """Formats tool definitions in the format that OpenAI appears to use. Based on https://github.com/forestwanglin/openai-java/blob/main/jtokkit/src/main/java/xyz/felh/openai/jtokkit/utils/TikTokenUtils.java diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index eae8ac04e8e..490d96746bd 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -51,6 +51,7 @@ class _UrlFetcher(Protocol): headers: dict[str, str] | None = None, follow_redirects: bool = False, timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, ) -> httpx.Response: ... @@ -80,10 +81,11 @@ class _FetchOptions(TypedDict, total=False): headers: ReadOnly[dict[str, str]] follow_redirects: ReadOnly[bool] timeout: ReadOnly[float | httpx.Timeout | None] + max_response_bytes: ReadOnly[int | None] class _AsyncFetchOptions(_FetchOptions, total=False): - max_response_bytes: ReadOnly[int | None] + pass # Cloud-internal IPs that ``ip.is_global`` can report as public. Everything diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index e68360e69b9..660c7c1dbc9 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -1,4 +1,3 @@ -import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence @@ -20,6 +19,8 @@ from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_ from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, + object_mapping, + openai_strict_json_schema, prompt_cache_key_from_user_id, ) from litellm.responses.utils import auto_reasoning_summary @@ -1017,66 +1018,23 @@ class LiteLLMAnthropicMessagesAdapter: Returns: OpenAI-compatible response_format dict, or None if invalid """ - if not isinstance(output_format, dict): + output_format_object: Final = object_mapping(output_format) + if output_format_object is None or output_format_object.get("type") != "json_schema": return None - output_type: Final = output_format.get("type") - if output_type != "json_schema": - return None - - schema = output_format.get("schema") + schema: Final = output_format_object.get("schema") if not schema: return None - # Deep copy to avoid mutating the original schema - schema = copy.deepcopy(schema) - # OpenAI strict mode requires additionalProperties: false on every object - self._add_additional_properties_false(schema) - - # Convert to OpenAI response_format structure return { "type": "json_schema", "json_schema": { "name": "structured_output", - "schema": schema, + "schema": openai_strict_json_schema(schema), "strict": True, }, } - @staticmethod - def _add_additional_properties_false(schema: dict) -> None: - """ - Recursively ensure object schemas comply with OpenAI strict mode. - - OpenAI's strict mode requires: - 1. 'additionalProperties': false at every object nesting level - 2. All property keys listed in 'required' - """ - if not isinstance(schema, dict): - return - - if schema.get("type") == "object" and "properties" in schema: - schema["additionalProperties"] = False - schema["required"] = list(schema["properties"].keys()) - for prop in schema["properties"].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop) - - # Handle array items - if "items" in schema: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) - - # Handle anyOf/oneOf/allOf - for key in ("anyOf", "oneOf", "allOf"): - if key in schema: - for sub_schema in schema[key]: - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) - - # Handle $defs / definitions - for key in ("$defs", "definitions"): - if key in schema: - for def_schema in schema[key].values(): - LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) - def _translate_midturn_system_message_to_openai( self, message: AnthropicMessagesSystemMessageParam, diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 16b03a711ac..a81ba087f30 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -19,6 +19,19 @@ from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import aload_beta_headers_config from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.enums import Hook +from litellm.integrations.custom_logger_v2.legacy import ( + PROXY_REQUEST_KEYS, + Admit, + KwargsFields, + hook_subscribers, + kwargs_after, + kwargs_patch, + run_subscribers, +) +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.outcomes import Patch from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -374,7 +387,7 @@ async def _execute_pre_request_hooks( pass # Build complete request kwargs dict - request_kwargs = { + request_kwargs: dict[str, object] = { # mutable-ok: the hooks take and may replace the request dict "tools": tools, "stream": stream, "litellm_params": { @@ -386,10 +399,21 @@ async def _execute_pre_request_hooks( if not litellm.callbacks: return request_kwargs - from litellm.integrations.custom_logger import CustomLogger as _CustomLogger + callbacks: Final = litellm.callbacks + if ( + Hook.pre_deployment_call in CustomLoggerV2.declared_hooks + and (subscribers := hook_subscribers(Hook.pre_deployment_call, callbacks, admit=Admit.every_logger)).has_v2 + ): + return kwargs_after( # pyright: ignore[reportReturnType] # whatever the last hook returned, as before + await run_subscribers( + subscribers, + KwargsFields(request_kwargs, PROXY_REQUEST_KEYS), + partial(_start_pre_request_hook, model, messages), + ) + ) - for callback in litellm.callbacks: - if not isinstance(callback, _CustomLogger): + for callback in callbacks: + if not isinstance(callback, CustomLogger): continue # Call the pre-request hook @@ -402,6 +426,27 @@ async def _execute_pre_request_hooks( return request_kwargs +def _start_pre_request_hook( + model: str, + messages: list[dict[str, object]], # mutable-ok: the hook takes the messages list + callback: object, + request_kwargs: object, +) -> Awaitable[Patch | None] | None: + if not isinstance(callback, CustomLogger): + return None + return _pre_request_hook(callback, model, messages, request_kwargs) + + +async def _pre_request_hook( + callback: CustomLogger, + model: str, + messages: list[dict[str, object]], # mutable-ok: see above + request_kwargs: object, +) -> Patch | None: + modified_kwargs: Final = await callback.async_pre_request_hook(model, messages, request_kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownVariableType] # the request dict as the previous hook left it + return None if modified_kwargs is None else kwargs_patch(request_kwargs, modified_kwargs) # pyright: ignore[reportUnknownArgumentType] # see above + + async def _try_websearch_short_circuit( model: str, messages: list[dict], diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index d7b8592f45c..1d186c8aee4 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -35,6 +35,8 @@ from litellm.llms.anthropic.messages.native.utils import ( from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, + object_mapping, + openai_strict_json_schema, prompt_cache_key_from_user_id, ) from litellm.types.llms.anthropic import ( @@ -588,10 +590,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) @staticmethod - def _resolve_output_format_strict(strict: bool | None) -> bool: - if strict is None: - return True - return strict + def _resolve_output_format_strict(strict: object) -> bool: + return True if strict is None else bool(strict) def translate_request( self, @@ -690,19 +690,23 @@ class LiteLLMAnthropicToResponsesAPIAdapter: # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} # output_config: {"format": {"type": "json_schema", "schema": {...}}} - output_format: object = anthropic_request.get("output_format") - output_config = anthropic_request.get("output_config") - if not isinstance(output_format, dict) and isinstance(output_config, dict): - output_format = output_config.get("format") - if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + output_config_object: Final = object_mapping(anthropic_request.get("output_config")) + top_level_output_format: Final = object_mapping(anthropic_request.get("output_format")) + output_format: Final = ( + top_level_output_format + if top_level_output_format is not None or output_config_object is None + else object_mapping(output_config_object.get("format")) + ) + if output_format is not None and output_format.get("type") == "json_schema": schema: Final = output_format.get("schema") if schema: + output_strict: Final = self._resolve_output_format_strict(output_format.get("strict")) responses_kwargs["text"] = { "format": { "type": "json_schema", "name": "structured_output", - "schema": schema, - "strict": self._resolve_output_format_strict(output_format.get("strict")), + "schema": openai_strict_json_schema(schema) if output_strict else schema, + "strict": output_strict, } } diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 95c7914befa..94f81eadb3b 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -1,6 +1,7 @@ +import copy import json import os -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -37,6 +38,61 @@ def json_copy_or_original(value: _JsonValue) -> _JsonValue: return value +_SUBSCHEMA_LIST_KEYWORDS: Final = ("anyOf", "oneOf", "allOf") +_SUBSCHEMA_MAP_KEYWORDS: Final = ("$defs", "definitions") + + +_JSON_ARRAY: Final = TypeAdapter(list[object]) + + +def _json_array(value: object) -> Sequence[object] | None: + try: + return _JSON_ARRAY.validate_python(value, strict=True) + except ValidationError: + return None + + +def openai_strict_json_schema(schema: object) -> object: + """New copy of a JSON schema where every object with properties forbids extra keys and requires all of them. + + OpenAI strict structured outputs rejects any schema that leaves an object open or a property optional. + """ + source: Final = object_mapping(schema) + if source is None: + return copy.deepcopy(schema) + properties: Final = object_mapping(source.get("properties")) + closing: Final[Mapping[str, object]] = ( + {"required": list(properties), "additionalProperties": False} + if properties is not None and _is_object_type(source.get("type")) + else {} + ) + return {**{key: _strict_subschema(key, value) for key, value in source.items()}, **closing} + + +def _is_object_type(schema_type: object) -> bool: + """Whether a ``type`` keyword names an object, alone or in a union such as ``["object", "null"]``.""" + types: Final = _json_array(schema_type) + return schema_type == "object" if types is None else "object" in types + + +def _strict_subschema(key: str, value: object) -> object: + if key == "items": + return openai_strict_json_schema(value) + if key == "properties" or key in _SUBSCHEMA_MAP_KEYWORDS: + named: Final = object_mapping(value) + return ( + copy.deepcopy(value) + if named is None + else {name: openai_strict_json_schema(subschema) for name, subschema in named.items()} + ) + if key in _SUBSCHEMA_LIST_KEYWORDS: + variants: Final = _json_array(value) + return ( + copy.deepcopy(value) if variants is None else [openai_strict_json_schema(variant) for variant in variants] + ) + return copy.deepcopy(value) + + class _ClaudeCodeUserId(LiteLLMBaseModel): """The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation.""" @@ -136,7 +192,7 @@ def proxy_sub_call_logging_kwargs(parent_kwargs: Mapping[str, object]) -> ProxyS if disable_header is None else {**parent_params, "proxy_server_request": {"headers": {X_LITELLM_DISABLE_CALLBACKS: disable_header}}} ) - trusted_vars: Final = _object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD)) + trusted_vars: Final = object_mapping(parent_kwargs.get(TRUSTED_CALLBACK_VARS_FIELD)) if trusted_vars is None: return dynamic_params return {**dynamic_params, "litellm_trusted_callback_vars": trusted_vars} @@ -154,13 +210,13 @@ def proxy_sub_call_trace_id(parent_kwargs: Mapping[str, object]) -> str | None: def _disable_callbacks_header(parent_kwargs: Mapping[str, object]) -> str | None: - parent_request: Final = _object_mapping(parent_kwargs.get("proxy_server_request")) - headers: Final = _object_mapping(parent_request.get("headers")) if parent_request is not None else None + parent_request: Final = object_mapping(parent_kwargs.get("proxy_server_request")) + headers: Final = object_mapping(parent_request.get("headers")) if parent_request is not None else None header: Final = headers.get(X_LITELLM_DISABLE_CALLBACKS) if headers is not None else None return header if isinstance(header, str) else None -def _object_mapping(value: object) -> Mapping[str, object] | None: +def object_mapping(value: object) -> Mapping[str, object] | None: try: return _OBJECT_MAPPING.validate_python(value) except ValidationError: diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 637d3cfc219..670614078d6 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -145,7 +145,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return BaseAzureLLM.base_validate_azure_environment(headers=headers, litellm_params=litellm_params) def get_stripped_model_name(self, model: str) -> str: - return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "") + return ( + model.replace("responses/", "") + .replace("o_series/", "") + .replace("azure_ai/", "") + .replace(AzureOpenAIGPT5Config.GPT5_SERIES_ROUTE, "") + ) def _validate_input_param( self, input: str | ResponseInputParam, *, drop_params: bool = False diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index 46ac3ccf433..d25a0c3d524 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -1,7 +1,9 @@ from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues @@ -16,6 +18,24 @@ else: LiteLLMLoggingObj = Any +_BODY_FIELDS: Final = TypeAdapter(dict[str, object]) + + +def extra_body_fields(optional_params: Mapping[str, object] | None) -> Mapping[str, object]: + """The caller's ``extra_body`` fields, merged under the provider body after the transform so they add fields but never + replace the input or a mapped param.""" + extra_body: Final = (optional_params or {}).get("extra_body") + return _BODY_FIELDS.validate_python(extra_body) if isinstance(extra_body, Mapping) else {} + + +def split_extra_body( + optional_params: Mapping[str, object] | None, +) -> tuple[Mapping[str, object], Mapping[str, object]]: + """The params without ``extra_body``, and the ``extra_body`` fields to merge into the provider body.""" + params: Final = _BODY_FIELDS.validate_python(optional_params or {}) + return {k: v for k, v in params.items() if k != "extra_body"}, extra_body_fields(params) + + class BaseEmbeddingConfig(BaseConfig, ABC): @abstractmethod def transform_embedding_request( diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 7ac440c6f48..28bf3d407f3 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -78,7 +78,7 @@ class BaseImageGenerationConfig(ABC): optional_params: dict, litellm_params: dict, headers: dict, - ) -> dict: + ) -> dict[str, object]: # mutable-ok: inherited contract raise NotImplementedError( "ImageVariationConfig implementa 'transform_request_image_variation' for image variation models" ) @@ -100,6 +100,32 @@ class BaseImageGenerationConfig(ABC): "ImageVariationConfig implements 'transform_response_image_variation' for image variation models" ) + async def async_transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict[str, object], # mutable-ok: inherited contract + optional_params: dict[str, object], # mutable-ok: inherited contract + litellm_params: dict[str, object], # mutable-ok: inherited contract + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ImageResponse: + return self.transform_image_generation_response( # pyright: ignore[reportUnknownMemberType] # legacy sync hook has untyped dict params + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + def use_multipart_form_data(self) -> bool: """ Returns True if this provider requires multipart/form-data instead of JSON. diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index e28e39ef2d4..0743bd6e7e7 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -6,6 +6,7 @@ import copy import json import urllib.parse from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, get_args, overload import httpx @@ -15,6 +16,7 @@ import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.embedding.transformation import extra_body_fields from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -103,7 +105,7 @@ class BedrockEmbedding(BaseAWSLLM): region_from_model: str | None = None, ) -> tuple[Credentials | None, str]: auth_params: Final = pop_aws_auth_params(optional_params) - explicit_region_name: Final = optional_params.pop("aws_region_name", None) + explicit_region_name: Final = _OPTIONAL_STR.validate_python(optional_params.pop("aws_region_name", None)) aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name ### SET REGION NAME ### @@ -291,9 +293,10 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, api_key: str | None = None, is_async_invoke: bool | None = False, + extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in batch_data: + for data in ({**extra_body, **item} for item in batch_data): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} @@ -360,9 +363,10 @@ class BedrockEmbedding(BaseAWSLLM): provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL, api_key: str | None = None, is_async_invoke: bool | None = False, + extra_body: Mapping[str, object] = MappingProxyType({}), ): responses: Final[list[dict]] = [] - for data in batch_data: + for data in ({**extra_body, **item} for item in batch_data): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} @@ -462,9 +466,12 @@ class BedrockEmbedding(BaseAWSLLM): ) inference_params = copy.deepcopy(optional_params) inference_params = { - k: v for k, v in inference_params.items() if k.lower() not in self.aws_authentication_params + k: v + for k, v in inference_params.items() + if k.lower() not in self.aws_authentication_params and k != "extra_body" } inference_params.pop("user", None) # make sure user is not passed in for bedrock call + extra_body: Final = extra_body_fields(optional_params) data: CohereEmbeddingRequest | None = None batch_data: list | None = None @@ -535,6 +542,8 @@ class BedrockEmbedding(BaseAWSLLM): ) batch_data.append(nova_request) + request_data: Final = None if data is None else {**extra_body, **data} + ### SET RUNTIME ENDPOINT ### endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( api_base=api_base, @@ -552,6 +561,7 @@ class BedrockEmbedding(BaseAWSLLM): client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None), timeout=timeout, batch_data=batch_data, + extra_body=extra_body, credentials=credentials, extra_headers=extra_headers, endpoint_url=endpoint_url, @@ -566,6 +576,7 @@ class BedrockEmbedding(BaseAWSLLM): client=(client if client is not None and isinstance(client, HTTPHandler) else None), timeout=timeout, batch_data=batch_data, + extra_body=extra_body, credentials=credentials, extra_headers=extra_headers, endpoint_url=endpoint_url, @@ -579,7 +590,7 @@ class BedrockEmbedding(BaseAWSLLM): if returned_response is None: raise Exception("Unable to map Bedrock request to provider") return returned_response - elif data is None: + elif request_data is None: raise Exception("Unable to map Bedrock request to provider") headers = {"Content-Type": "application/json"} @@ -591,7 +602,7 @@ class BedrockEmbedding(BaseAWSLLM): aws_region_name=aws_region_name, extra_headers=extra_headers, endpoint_url=endpoint_url, - data=json.dumps(data), + data=json.dumps(request_data), headers=headers, api_key=api_key, ) @@ -607,7 +618,7 @@ class BedrockEmbedding(BaseAWSLLM): logging_obj=logging_obj, optional_params=optional_params, encoding=encoding, - data=data, + data=request_data, complete_api_base=prepped.url, api_key=None, aembedding=aembedding, diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 901c16afaa7..ad6586f6043 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Optional, TypeVar, cast import httpx from httpx import Response -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.constants import AWS_ERROR_TYPE_HEADER @@ -23,7 +23,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation bedrock_invoke_guardrail_headers, ) from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage -from litellm.types.utils import ModelResponseStream +from litellm.types.utils import Embedding, EmbeddingResponse, ModelResponseStream, PromptTokensDetailsWrapper, Usage from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( @@ -376,6 +376,53 @@ def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]: return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)} +class _TitanEmbeddingsByType(BaseModel): + model_config = ConfigDict(frozen=True) + + binary: tuple[int, ...] | None = None + floats: tuple[float, ...] | None = Field(default=None, alias="float") + + +class _TitanEmbeddingReply(BaseModel): + model_config = ConfigDict(frozen=True) + + embedding: tuple[float, ...] | None = None + embeddings_by_type: _TitanEmbeddingsByType = Field(default=_TitanEmbeddingsByType(), alias="embeddingsByType") + input_text_token_count: int = Field(alias="inputTextTokenCount") + + @property + def vector(self) -> tuple[float, ...] | None: + if self.embedding is not None: + return self.embedding + if self.embeddings_by_type.floats is not None: + return self.embeddings_by_type.floats + return self.embeddings_by_type.binary + + +def _invoke_embedding_response(model: str, httpx_response: Response, has_input_image: bool) -> EmbeddingResponse | None: + reply: Final = relayed_json_object(httpx_response) + if reply is None or reply.keys().isdisjoint(("embedding", "embeddingsByType")): + return None + try: + parsed: Final = _TitanEmbeddingReply.model_validate(reply) + except ValidationError: + return None + if parsed.vector is None: + return None + return EmbeddingResponse( + model=model, + usage=Usage( + prompt_tokens=parsed.input_text_token_count, + completion_tokens=0, + total_tokens=parsed.input_text_token_count, + prompt_tokens_details=PromptTokensDetailsWrapper(image_count=1, text_tokens=parsed.input_text_token_count) + if has_input_image + else None, + ), + data=[Embedding(embedding=list(parsed.vector), index=0, object="embedding")], + ) + + def _served_service_tier(response_headers: Mapping[str, str]) -> str | None: headers: Final = httpx.Headers(response_headers) if _BEDROCK_SERVICE_TIER_HEADER not in headers: @@ -574,6 +621,14 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD from litellm.types.utils import LlmProviders, ModelResponse from litellm.utils import ProviderConfigManager + embedding_response: Final = ( + _invoke_embedding_response(model, httpx_response, has_input_image="inputImage" in request_data) + if "invoke" in endpoint + else None + ) + if embedding_response is not None: + return embedding_response + if "invoke" in endpoint: chat_config_model = "invoke/" + model elif "converse" in endpoint: diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index b290a7abf3d..24ee7958e45 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1671,6 +1671,7 @@ class HTTPHandler: follow_redirects: bool | None = None, timeout: float | httpx.Timeout | None = None, stream: bool = False, + max_response_bytes: int | None = None, ): # Set follow_redirects to UseClientDefault if None _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT @@ -1681,6 +1682,16 @@ class HTTPHandler: url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout ) + if max_response_bytes is not None: + return self._get_with_response_limit( + url, + params=query, + headers=headers, + max_bytes=max_response_bytes, + follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects, + timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout), + ) + response: Final = self.client.get( url, params=query, @@ -1691,6 +1702,67 @@ class HTTPHandler: return response + def _get_with_response_limit( + self, + url: str, + *, + params: httpx.QueryParams, + headers: Mapping[str, str] | None, + timeout: httpx.Timeout, + max_bytes: int, + follow_redirects: bool, + ) -> httpx.Response: + request: Final = self.client.build_request( + "GET", + url, + headers=MappingProxyType({**(headers or {}), "accept-encoding": "identity"}), + params=params, + timeout=timeout, + ) + response: Final = self.client.send(request, stream=True, follow_redirects=False) + return self._read_with_response_limit(response, max_bytes=max_bytes, follow_redirects=follow_redirects) + + def _next_redirect_hop(self, hop: httpx.Response) -> httpx.Response: + if hop.next_request is None: + return hop + hop.close() + return self.client.send(hop.next_request, auth=None, stream=True, follow_redirects=False) + + def _last_redirect_hop(self, response: httpx.Response, *, follow_redirects: bool) -> httpx.Response: + hop = response # rebind-ok: walks the redirect chain without recursion + for _ in range(10): + if hop.next_request is None or not follow_redirects: + return hop + hop = self._next_redirect_hop(hop) + if hop.next_request is None or not follow_redirects: + return hop + hop.close() + raise ValueError("Too many redirects") + + def _read_with_response_limit( + self, response: httpx.Response, *, max_bytes: int, follow_redirects: bool + ) -> httpx.Response: + final: Final = self._last_redirect_hop(response, follow_redirects=follow_redirects) + try: + if final.is_redirect or final.is_error: + return httpx.Response(final.status_code, headers=final.headers, request=final.request) + content_encoding: Final = header_value(final.headers, "content-encoding") + if ("identity" if content_encoding is None else content_encoding).lower() != "identity": + raise HTTPResponseLimitError("Response size limits require an uncompressed response") + content_length: Final = header_value(final.headers, "content-length") + if int("0" if content_length is None else content_length) > max_bytes: + raise HTTPResponseLimitError("Response exceeds the configured size limit") + with BytesIO() as body: + for chunk in final.iter_bytes(chunk_size=65536): + if body.tell() + len(chunk) > max_bytes: + raise HTTPResponseLimitError("Response exceeds the configured size limit") + body.write(chunk) + return httpx.Response( + final.status_code, headers=final.headers, content=body.getvalue(), request=final.request + ) + finally: + final.close() + def _send_streaming_get( self, url: str, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 68b13c6172f..9c3d428d2f6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -45,6 +45,9 @@ from litellm.anthropic_beta_headers_manager import ( ) from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult +from litellm.integrations.custom_logger_v2.enums import Hook +from litellm.integrations.custom_logger_v2.legacy import Admit, hook_subscribers +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_refusal import ( called_tool_names, @@ -562,6 +565,14 @@ def _run_responses_deployment_hook( "custom_llm_provider": custom_llm_provider, }, CallTypes.responses.value, + include_v2=True, + ) + + +def _has_v2_deployment_hook() -> bool: + return ( + Hook.pre_deployment_call in CustomLoggerV2.declared_hooks + and hook_subscribers(Hook.pre_deployment_call, litellm.callbacks, admit=Admit.every_logger).has_v2 ) @@ -2940,7 +2951,7 @@ class BaseLLMHTTPHandler: dict[str, object], GenericLiteLLMParams, ]: - if not _has_pre_call_deployment_hook(logging_obj): + if not _has_pre_call_deployment_hook(logging_obj) and not _has_v2_deployment_hook(): return ( model, input, @@ -6188,6 +6199,7 @@ class BaseLLMHTTPHandler: **kwargs_for_followup, } ) + logging_obj.record_served_agentic_follow_up() if callback is not None: try: @@ -7881,14 +7893,16 @@ class BaseLLMHTTPHandler: provider_config=image_generation_provider_config, ) - model_response: Final[ImageResponse] = image_generation_provider_config.transform_image_generation_response( + model_response: Final[ + ImageResponse + ] = await image_generation_provider_config.async_transform_image_generation_response( model=model, raw_response=response, model_response=litellm.ImageResponse(), logging_obj=logging_obj, request_data=data, - optional_params=image_generation_optional_request_params, - litellm_params=dict(litellm_params), + optional_params=image_generation_optional_request_params, # pyright: ignore[reportUnknownArgumentType] # legacy handler params are untyped dicts + litellm_params=dict(litellm_params), # pyright: ignore[reportUnknownArgumentType] # legacy handler params are untyped dicts encoding=None, ) set_provider_response_headers_in_hidden_params(model_response, response.headers) diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index 2acf1c69594..b68e9234c09 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -1,13 +1,15 @@ import os from collections.abc import Mapping -from math import ceil +from math import ceil, isfinite from types import MappingProxyType from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import get_provider_response_headers_from_hidden_params from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, resolve_image_model_info +from litellm.llms.openai.cost_calculation import video_clip_cost from litellm.types.utils import ImageObject, ImageResponse, ModelInfo FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high" @@ -28,6 +30,7 @@ FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType( _OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) FAL_AI_QUEUE_DEFAULT_BASE: Final[str] = "https://queue.fal.run" +FAL_BILLABLE_UNITS_HEADER: Final[str] = "x-fal-billable-units" def fal_ai_queue_base() -> str: @@ -150,6 +153,26 @@ def _resolution_cost_per_image(entry: Mapping[str, object] | None, resolution: o return float(cost) if isinstance(cost, (int, float)) else None +def _billable_units(response: object) -> float | None: + headers: Final = get_provider_response_headers_from_hidden_params(response) + raw_units: Final = None if headers is None else headers.get(FAL_BILLABLE_UNITS_HEADER) + if raw_units is None: + return None + try: + units: Final = float(raw_units) + except ValueError: + return None + return units if isfinite(units) and units >= 0 else None + + +def _billable_units_cost(prices: Mapping[str, object] | None, response: object) -> float | None: + unit_cost: Final = None if prices is None else prices.get("output_cost_per_billable_unit") + units: Final = _billable_units(response) + if isinstance(unit_cost, bool) or not isinstance(unit_cost, (int, float)) or units is None: + return None + return unit_cost * units + + def _requested_image_count(request_body: Mapping[str, object]) -> int: num_images: Final = request_body.get("num_images") return num_images if type(num_images) is int and num_images > 0 else 1 @@ -165,8 +188,14 @@ def _passthrough_cost_per_image(entry: Mapping[str, object], request_body: Mappi def fal_ai_passthrough_cost(model: str, request_body: Mapping[str, object]) -> float | None: entry: Final = _entry(f"{litellm.LlmProviders.FAL_AI.value}/{model}") - if entry is None: + if entry is None or entry.get("output_cost_per_billable_unit") is not None: return None + clip_cost: Final = video_clip_cost(entry, request_body.get("duration"), request_body.get("resolution")) + if clip_cost is not None: + return clip_cost + cost_per_video: Final = entry.get("output_cost_per_video") + if isinstance(cost_per_video, (int, float)): + return float(cost_per_video) cost_per_image: Final = _passthrough_cost_per_image(entry, request_body) if cost_per_image is None: return None @@ -187,15 +216,20 @@ def cost_calculator( normalized_model: Final = model.removeprefix(f"{litellm.LlmProviders.FAL_AI.value}/") images: Final = tuple(image_response.data or ()) deployment_prices: Final = deployment_pricing(model_info) + deployment_billable_units_cost: Final = _billable_units_cost(deployment_prices, image_response) + if deployment_billable_units_cost is not None: + return deployment_billable_units_cost deployment_cost_per_image: Final = ( None if deployment_prices is None else deployment_prices.get("output_cost_per_image") ) if deployment_cost_per_image is not None: return deployment_cost_per_image * len(images) + entry: Final = _entry(f"{litellm.LlmProviders.FAL_AI.value}/{normalized_model}") + billable_units_cost: Final = _billable_units_cost(entry, image_response) + if billable_units_cost is not None: + return billable_units_cost params: Final[Mapping[str, object]] = optional_params or MappingProxyType({}) - resolution_cost_per_image: Final = _resolution_cost_per_image( - _entry(f"{litellm.LlmProviders.FAL_AI.value}/{normalized_model}"), params.get("resolution") - ) + resolution_cost_per_image: Final = _resolution_cost_per_image(entry, params.get("resolution")) if resolution_cost_per_image is not None: return resolution_cost_per_image * len(images) keyed_costs: Final = tuple( diff --git a/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py b/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py index 35df7a72fe0..f848b86246f 100644 --- a/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py +++ b/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py @@ -5,6 +5,8 @@ from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from .transformation import FalAIBaseConfig +_BARE_APP_MARKERS: Final = ("nano-banana", "gemini-25-flash-image") + class FalAINanoBananaConfig(FalAIBaseConfig): """ @@ -16,6 +18,7 @@ class FalAINanoBananaConfig(FalAIBaseConfig): - fal-ai/gemini-25-flash-image - fal-ai/nano-banana-2 - fal-ai/nano-banana-pro + - google/nano-banana-2.1 Provider-specific params such as ``resolution`` ("0.5K", "1K", "2K", "4K") are forwarded as-is and drive the per-resolution price in the cost map. @@ -46,7 +49,9 @@ class FalAINanoBananaConfig(FalAIBaseConfig): stream: bool | None = None, ) -> str: base_url: Final[str] = (api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL).rstrip("/") - endpoint: Final = model if model.startswith("fal-ai/") else f"fal-ai/{model}" + app: Final = model.partition("/")[0].lower() + is_bare_app: Final = any(marker in app for marker in _BARE_APP_MARKERS) + endpoint: Final = f"fal-ai/{model}" if is_bare_app else model return f"{base_url}/{endpoint}" def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: diff --git a/litellm/llms/fal_ai/videos/transformation.py b/litellm/llms/fal_ai/videos/transformation.py index 9974f72c025..bea3de75f71 100644 --- a/litellm/llms/fal_ai/videos/transformation.py +++ b/litellm/llms/fal_ai/videos/transformation.py @@ -2,7 +2,7 @@ import math import sys import time from collections.abc import Callable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final, TypeAlias @@ -49,6 +49,7 @@ class _ModelProfile: integer_duration: bool reference_key: str reference_as_list: bool + default_duration: str | None = None _SEEDANCE_PROFILE: Final[_ModelProfile] = _ModelProfile( @@ -59,6 +60,15 @@ _SEEDANCE_PROFILE: Final[_ModelProfile] = _ModelProfile( reference_key="image_url", reference_as_list=False, ) +_PIXVERSE_PROFILE: Final[_ModelProfile] = _ModelProfile( + resolutions=frozenset({"360p", "540p", "720p", "1080p"}), + resolution_tiers=((360, "360p"), (540, "540p"), (720, "720p"), (sys.maxsize, "1080p")), + default_resolution="720p", + integer_duration=False, + reference_key="image_url", + reference_as_list=False, + default_duration="5", # https://fal.ai/models/pixverse/v6/text-to-video/api (2026-11-13) +) _H3_PROFILE: Final[_ModelProfile] = _ModelProfile( resolutions=frozenset({"480P", "768P", "2K", "4K"}), resolution_tiers=((480, "480P"), (768, "768P"), (1440, "2K"), (sys.maxsize, "4K")), @@ -75,6 +85,19 @@ _H3_MAX_PROFILE: Final[_ModelProfile] = _ModelProfile( reference_key="image_url", reference_as_list=False, ) +_LUMA_RAY_PROFILE: Final[_ModelProfile] = _ModelProfile( + resolutions=frozenset({"540p", "720p", "1080p"}), + resolution_tiers=((540, "540p"), (720, "720p"), (sys.maxsize, "1080p")), + default_resolution="540p", + integer_duration=False, + reference_key="image_url", + reference_as_list=False, +) +_SEEDANCE_REFERENCE_PROFILE: Final[_ModelProfile] = replace( + _SEEDANCE_PROFILE, + reference_key="image_urls", + reference_as_list=True, +) _QUEUE_NAMESPACES: Final[frozenset[str]] = frozenset(("workflows", "comfy")) _STATUS_MAP: Final[Mapping[str, str]] = MappingProxyType( { @@ -109,11 +132,16 @@ def _duration_value(value: object) -> str | None: def _profile_for_model(model: str) -> _ModelProfile: + if model.removeprefix("fal_ai/").removeprefix("fal-ai/").startswith("pixverse/"): + return _PIXVERSE_PROFILE if model.startswith("minimax/h3/"): return _H3_PROFILE if model.startswith("minimax/h3-max"): return _H3_MAX_PROFILE - return _SEEDANCE_PROFILE + if model.startswith("luma/agent/ray/"): + return _LUMA_RAY_PROFILE + is_seedance_reference: Final = model.startswith("bytedance/seedance-") and model.endswith("/reference-to-video") + return _SEEDANCE_REFERENCE_PROFILE if is_seedance_reference else _SEEDANCE_PROFILE def _resolution_for_short_side(short_side: int, profile: _ModelProfile) -> str: @@ -340,8 +368,8 @@ class FalAIVideoConfig(BaseVideoConfig): ) duration_params: Final[Mapping[str, object]] = ( MappingProxyType({}) - if "seconds" not in video_create_optional_params - else self._duration_params(video_create_optional_params["seconds"], profile) + if "seconds" not in video_create_optional_params and profile.default_duration is None + else self._duration_params(video_create_optional_params.get("seconds", profile.default_duration), profile) ) size_params: Final[Mapping[str, str]] = ( _size_params(video_create_optional_params["size"], profile) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index d42b4ddbbf7..c4276166b30 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -38,11 +38,10 @@ def get_base_model_for_pricing(model_name: str) -> str: elif total_billion <= FIREWORKS_AI_176_B_MOE: return "fireworks-ai-56b-to-176b" - # Check for standard models in the form b - re_params_match: Final = re.search(r"(\d+)b", model_name) + # Check for standard models in the form b, where Fireworks writes 1.5b as 1p5b + re_params_match: Final = re.search(r"(\d+(?:p\d+)?)b", model_name) if re_params_match is not None: - params_match: Final = str(re_params_match.group(1)) - params_billion: Final = float(params_match) + params_billion: Final = float(re_params_match.group(1).replace("p", ".")) # Determine the category based on the number of parameters if params_billion <= FIREWORKS_AI_4_B: diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index a02afa3a13f..b9a2cc6493c 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -198,11 +198,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): """ request_body: Final[dict[str, object]] = {} - # Model or Agent (one required) - if model: - request_body["model"] = GeminiModelInfo.get_base_model(model) or model - elif agent: + if agent: request_body["agent"] = agent + elif model: + request_body["model"] = GeminiModelInfo.get_base_model(model) or model else: raise ValueError("Either 'model' or 'agent' must be provided") @@ -225,7 +224,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): dropped_keys: Final = ( frozenset({"agent_config"}) if drops_unsupported_params(litellm_params.get("drop_params")) - and (model or not _has_agent_config_type(optional_params.get("agent_config"))) + and (not agent or not _has_agent_config_type(optional_params.get("agent_config"))) else frozenset() ) for key in optional_keys: diff --git a/litellm/llms/huggingface/chat/transformation.py b/litellm/llms/huggingface/chat/transformation.py index 3fcb738eacb..f5269603cbc 100644 --- a/litellm/llms/huggingface/chat/transformation.py +++ b/litellm/llms/huggingface/chat/transformation.py @@ -24,6 +24,10 @@ logger: Final = logging.getLogger(__name__) BASE_URL: Final = "https://router.huggingface.co" +def _api_base_from_env() -> str | None: + return os.getenv("HF_API_BASE") or os.getenv("HUGGINGFACE_API_BASE") + + def _build_chat_completion_url(model_url: str) -> str: # Strip trailing / model_url = model_url.rstrip("/") @@ -96,8 +100,8 @@ class HuggingFaceChatConfig(OpenAIGPTConfig): if api_base is not None: complete_url = api_base complete_url = _build_chat_completion_url(complete_url) - elif os.getenv("HF_API_BASE") or os.getenv("HUGGINGFACE_API_BASE"): - complete_url = str(os.getenv("HF_API_BASE")) or str(os.getenv("HUGGINGFACE_API_BASE")) + elif env_api_base := _api_base_from_env(): + complete_url = env_api_base elif model.startswith(("http://", "https://")): complete_url = model complete_url = _build_chat_completion_url(complete_url) @@ -129,8 +133,12 @@ class HuggingFaceChatConfig(OpenAIGPTConfig): litellm_params: dict, headers: dict, ) -> dict: - if litellm_params.get("api_base"): - return dict(ChatCompletionRequest(model=model, messages=messages, **optional_params)) + lacks_explicit_api_base: Final = not litellm_params.get("api_base") + if not lacks_explicit_api_base or _api_base_from_env(): + endpoint_messages: Final = ( + self._transform_messages(messages=messages, model=model) if lacks_explicit_api_base else messages + ) + return dict(ChatCompletionRequest(model=model, messages=endpoint_messages, **optional_params)) if "max_retries" in optional_params: logger.warning("`max_retries` is not supported. It will be ignored.") optional_params.pop("max_retries", None) diff --git a/litellm/llms/nace/decisions/transformation.py b/litellm/llms/nace/decisions/transformation.py index c96177bb7d5..774dd230fe0 100644 --- a/litellm/llms/nace/decisions/transformation.py +++ b/litellm/llms/nace/decisions/transformation.py @@ -1,9 +1,71 @@ -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +import re +from collections.abc import Mapping +from dataclasses import replace +from typing import Final + +from litellm._version import version as _litellm_version +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, remote_image_urls +from litellm.types.decisions import ( + DecisionsBase64Image, + DecisionsImage, + DecisionsIRRequest, + DecisionsIRState, + UnsupportedDecisionsRequest, +) + +NACE_CLIENT_HEADER: Final = "X-Nace-Client" +_NACE_CLIENT_DISALLOWED: Final = re.compile(r"[^A-Za-z0-9 ._@+-]") +_NACE_CLIENT_MAX_LENGTH: Final = 64 + + +def nace_client_header() -> Mapping[str, str]: + version: Final = _NACE_CLIENT_DISALLOWED.sub("-", _litellm_version) + return {NACE_CLIENT_HEADER: f"litellm/{version}"[:_NACE_CLIENT_MAX_LENGTH]} + + +def _data_url(image: DecisionsImage) -> DecisionsImage: + if isinstance(image, DecisionsBase64Image): + return f"data:{image.content_type};base64,{image.base64}" + return image + + +def _with_images_as_data_urls(request: DecisionsIRRequest) -> DecisionsIRRequest: + if not isinstance(request.input, DecisionsIRState): + return request + return replace(request, input=replace(request.input, images=tuple(map(_data_url, request.input.images)))) class NaceDecisionsConfig(BaseDecisionsConfig): api_key_env = ("NACE_API_KEY",) api_base_env = ("NACE_API_BASE", "NACE_BASE_URL") + max_images = 8 + image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"}) + + def __init__(self, model: str = "") -> None: + self.supports_images = model != "drex-v2.0" def get_default_api_base(self) -> str | None: return "https://console.nace.ai" + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + base: Final = super().validate_environment(headers, model, api_key, litellm_params) + return { + **{name: value for name, value in base.items() if name.lower() != NACE_CLIENT_HEADER.lower()}, + **nace_client_header(), + } + + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if remote_image_urls(request.input): + return UnsupportedDecisionsRequest("Nace requires inline base64 images or data URLs, not remote image URLs") + return super().transform_decisions_request(model, _with_images_as_data_urls(request), custom_llm_provider) diff --git a/litellm/llms/nvidia_nim/embed.py b/litellm/llms/nvidia_nim/embed.py index d2ef446299e..7368831f7a8 100644 --- a/litellm/llms/nvidia_nim/embed.py +++ b/litellm/llms/nvidia_nim/embed.py @@ -66,7 +66,7 @@ class NvidiaNimEmbeddingConfig: self, non_default_params: dict, optional_params: dict, - kwargs: dict | None = None, + kwargs: Mapping[str, object] | None = None, ): if "extra_body" not in optional_params: optional_params["extra_body"] = {} @@ -80,5 +80,11 @@ class NvidiaNimEmbeddingConfig: if kwargs is not None: # pass kwargs in extra_body - optional_params["extra_body"].update(kwargs) + nested_extra_body: Final = kwargs.get("extra_body") + optional_params["extra_body"].update( + { + **{k: v for k, v in kwargs.items() if k != "extra_body"}, + **(nested_extra_body if isinstance(nested_extra_body, Mapping) else {}), + } + ) return optional_params diff --git a/litellm/llms/ollama/completion/handler.py b/litellm/llms/ollama/completion/handler.py index e1fc9a8d262..fd874246980 100644 --- a/litellm/llms/ollama/completion/handler.py +++ b/litellm/llms/ollama/completion/handler.py @@ -14,6 +14,7 @@ from typing_extensions import NotRequired, ReadOnly import litellm from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.llms.base_llm.embedding.transformation import split_extra_body from litellm.types.mcp import has_header from litellm.types.utils import EmbeddingResponse @@ -45,14 +46,15 @@ def _prepare_ollama_embedding_payload( data: Final[dict[str, object]] = {"model": model, "input": prompts} special_optional_params: Final = ["truncate", "options", "keep_alive", "dimensions"] - for k, v in optional_params.items(): + body_params, extra_body = split_extra_body(optional_params) + for k, v in body_params.items(): if k in special_optional_params: data[k] = v else: data.setdefault("options", {}) if isinstance(data["options"], dict): data["options"].update({k: v}) - return data + return {**extra_body, **data} def _process_ollama_embedding_response( diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index a6dbc79254e..10f23b02afb 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -6,6 +6,7 @@ Helper util for handling openai-specific cost calculation from collections.abc import Mapping from typing import Final, Literal, SupportsFloat, SupportsIndex +from pydantic import TypeAdapter, ValidationError from typing_extensions import Buffer from litellm._logging import verbose_logger @@ -136,6 +137,28 @@ def video_output_cost_per_second( _video_output_cost_per_second = video_output_cost_per_second +_VIDEO_CLIP_PRICES: Final = TypeAdapter(Mapping[str, Mapping[str, float]]) + + +def video_clip_cost(model_info: Mapping[str, object], duration: object, resolution: object) -> float | None: + if not isinstance(resolution, str): + return None + try: + prices: Final = _VIDEO_CLIP_PRICES.validate_python(model_info.get("output_cost_per_video_clip")) + except ValidationError: + return None + if isinstance(duration, bool) or not isinstance(duration, (str, int, float)): + return None + try: + duration_seconds: Final = float(duration) + if duration_seconds <= 0: + return None + requested_key: Final = f"{round(duration_seconds)}s" + except (ValueError, OverflowError): + return None + return prices.get(requested_key, {}).get(resolution.strip().lower()) + + def video_generation_cost( model: str, duration_seconds: float, @@ -162,6 +185,14 @@ def video_generation_cost( if model_info is None: model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider or "openai") + clip_cost: Final = video_clip_cost(model_info, duration_seconds, video_resolution) + if clip_cost is not None: + return clip_cost + + video_cost: Final = model_info.get("output_cost_per_video") + if video_cost is not None: + return video_cost + # Check for video-specific cost per second video_cost_per_second: Final = model_info.get("output_cost_per_video_per_second") if video_cost_per_second is not None: diff --git a/litellm/llms/replicate/common_utils.py b/litellm/llms/replicate/common_utils.py index e1fa3828379..4626babe14c 100644 --- a/litellm/llms/replicate/common_utils.py +++ b/litellm/llms/replicate/common_utils.py @@ -1,6 +1,38 @@ -import httpx +import asyncio +import time +from collections import deque +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final +import httpx +from pydantic import ConfigDict, TypeAdapter + +import litellm +from litellm.constants import ( + DEFAULT_REPLICATE_POLLING_DELAY_SECONDS, + DEFAULT_REPLICATE_SYNC_POLLING_RETRIES, + REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES, +) +from litellm.litellm_core_utils.url_utils import ( + assert_same_origin, + async_safe_get, + encode_url_path_segment, + ensure_api_route, + join_api_base, + safe_get, +) from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, HTTPResponseLimitError +from litellm.secret_managers.main import get_secret_str + +REPLICATE_DEFAULT_API_BASE: Final[str] = "https://api.replicate.com/v1" +REPLICATE_TERMINAL_STATUSES: Final[frozenset[str]] = frozenset({"succeeded", "failed", "canceled"}) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_REQUEST_HEADERS_NOT_FORWARDED: Final[frozenset[str]] = frozenset( + {"host", "content-length", "content-type", "transfer-encoding", "prefer"} +) class ReplicateError(BaseLLMException): @@ -8,6 +40,231 @@ class ReplicateError(BaseLLMException): self, status_code: int, message: str, - headers: dict | httpx.Headers | None, + headers: dict[str, str] | httpx.Headers | None, ): super().__init__(status_code=status_code, message=message, headers=headers) + + +@dataclass(frozen=True, slots=True) +class ReplicatePredictionRequest: + url: str + body: Mapping[str, object] + + +def replicate_api_base(api_base: str | None) -> str: + return ensure_api_route(api_base or get_secret_str("REPLICATE_API_BASE") or REPLICATE_DEFAULT_API_BASE, "v1") + + +def replicate_auth_headers( + headers: Mapping[str, str], api_key: str | None +) -> dict[str, str]: # mutable-ok: base class contracts return a plain dict + resolved_key: Final = ( + api_key or litellm.replicate_key or get_secret_str("REPLICATE_API_KEY") or get_secret_str("REPLICATE_API_TOKEN") + ) + if not resolved_key: + raise ReplicateError( + status_code=401, + message="Replicate API key is required. Set REPLICATE_API_KEY or pass api_key.", + headers=None, + ) + return {**headers, "Authorization": f"Bearer {resolved_key}", "Content-Type": "application/json"} + + +def replicate_model_path(model: str) -> str: + segments: Final = tuple(model.split("/")) + if any(segment in ("", ".", "..") for segment in segments): + raise ReplicateError(status_code=400, message=f"Invalid Replicate model path: {model!r}", headers={}) + return "/".join(encode_url_path_segment(segment, field_name="model") for segment in segments) + + +def replicate_prediction_request( + api_base: str, model: str, prediction_input: Mapping[str, object] +) -> ReplicatePredictionRequest: + if model.startswith("deployments/"): + return ReplicatePredictionRequest( + url=join_api_base(api_base, f"{replicate_model_path(model)}/predictions"), + body=MappingProxyType({"input": prediction_input}), + ) + if ":" in model: + model_path, _, version = model.partition(":") + if not version: + raise ReplicateError(status_code=400, message=f"Invalid Replicate model version: {model!r}", headers={}) + return ReplicatePredictionRequest( + url=join_api_base( + api_base, + f"models/{replicate_model_path(model_path)}/versions/" + f"{encode_url_path_segment(version, field_name='version')}/predictions", + ), + body=MappingProxyType({"input": prediction_input}), + ) + return ReplicatePredictionRequest( + url=join_api_base(api_base, f"models/{replicate_model_path(model)}/predictions"), + body=MappingProxyType({"input": prediction_input}), + ) + + +def replicate_prediction_url(api_base: str, prediction_id: str) -> str: + return join_api_base(api_base, f"predictions/{encode_url_path_segment(prediction_id, field_name='prediction id')}") + + +def replicate_model_owner(model: str) -> str: + return model.removeprefix("deployments/").split(":", 1)[0].split("/", 1)[0] + + +def replicate_prediction(raw_response: httpx.Response) -> Mapping[str, object]: + try: + return _JSON_OBJECT.validate_python(raw_response.json()) + except ValueError as error: + raise ReplicateError( + status_code=raw_response.status_code, + message=f"Replicate returned a non-JSON response: {raw_response.text[:500]}", + headers=raw_response.headers, + ) from error + + +def replicate_prediction_string(prediction: Mapping[str, object], key: str) -> str | None: + value: Final = prediction.get(key) + return value if isinstance(value, str) and value else None + + +def replicate_prediction_input(prediction: Mapping[str, object]) -> Mapping[str, object]: + raw_input: Final = prediction.get("input") + return _JSON_OBJECT.validate_python(raw_input) if isinstance(raw_input, Mapping) else MappingProxyType({}) + + +def replicate_prediction_metrics(prediction: Mapping[str, object]) -> Mapping[str, object]: + raw_metrics: Final = prediction.get("metrics") + return _JSON_OBJECT.validate_python(raw_metrics) if isinstance(raw_metrics, Mapping) else MappingProxyType({}) + + +def replicate_positive_int(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + + +def _output_leaves(output: object) -> Iterator[object]: + pending: Final = deque([output]) + while pending: + if isinstance((item := pending.popleft()), Mapping): + pending.extendleft(reversed(tuple(_JSON_OBJECT.validate_python(item).values()))) + elif isinstance(item, (list, tuple)): + pending.extendleft(reversed(TypeAdapter(tuple[object, ...]).validate_python(item))) + else: + yield item + + +def replicate_output_urls(output: object) -> tuple[str, ...]: + return tuple( + leaf for leaf in _output_leaves(output) if isinstance(leaf, str) and leaf.startswith(("http://", "https://")) + ) + + +def replicate_prediction_failure(prediction: Mapping[str, object]) -> str | None: + status: Final = replicate_prediction_string(prediction, "status") + error: Final = replicate_prediction_string(prediction, "error") + if status in ("failed", "canceled"): + return error or f"Replicate prediction {status}" + return None + + +def replicate_prediction_is_done(prediction: Mapping[str, object]) -> bool: + return replicate_prediction_string(prediction, "status") in REPLICATE_TERMINAL_STATUSES + + +def replicate_poll_headers( + request: httpx.Request, +) -> dict[str, str]: # mutable-ok: HTTP handler clients take a plain dict + return {key: value for key, value in request.headers.items() if key.lower() not in _REQUEST_HEADERS_NOT_FORWARDED} + + +def _poll_url(prediction: Mapping[str, object], raw_response: httpx.Response) -> str: + urls: Final = prediction.get("urls") + poll_url: Final = ( + replicate_prediction_string(_JSON_OBJECT.validate_python(urls), "get") if isinstance(urls, Mapping) else None + ) + if poll_url is not None: + assert_same_origin(poll_url, str(raw_response.request.url)) + return poll_url + prediction_id: Final = replicate_prediction_string(prediction, "id") + if prediction_id is None: + raise ReplicateError( + status_code=raw_response.status_code, + message="Replicate prediction has no id to poll", + headers=raw_response.headers, + ) + api_base: Final = str(raw_response.request.url).split("/predictions", 1)[0].rsplit("/models/", 1)[0] + return replicate_prediction_url(api_base.rsplit("/deployments/", 1)[0], prediction_id) + + +def _finished_prediction(raw_response: httpx.Response) -> Mapping[str, object] | None: + if raw_response.status_code >= 400: + raise ReplicateError( + status_code=raw_response.status_code, message=raw_response.text, headers=raw_response.headers + ) + prediction: Final = replicate_prediction(raw_response) + return prediction if replicate_prediction_is_done(prediction) else None + + +def wait_for_replicate_prediction( + raw_response: httpx.Response, + client: HTTPHandler, +) -> Mapping[str, object]: + prediction: Final = replicate_prediction(raw_response) + if replicate_prediction_is_done(prediction): + return prediction + poll_url: Final = _poll_url(prediction, raw_response) + headers: Final = replicate_poll_headers(raw_response.request) + for _ in range(DEFAULT_REPLICATE_SYNC_POLLING_RETRIES): + time.sleep(DEFAULT_REPLICATE_POLLING_DELAY_SECONDS) + if (polled := _finished_prediction(client.get(url=poll_url, headers=headers))) is not None: # pyright: ignore[reportUnknownMemberType] # HTTP handler stubs are untyped + return polled + raise ReplicateError( + status_code=504, + message=f"Replicate prediction did not finish after {DEFAULT_REPLICATE_SYNC_POLLING_RETRIES} polls", + headers=raw_response.headers, + ) + + +async def async_wait_for_replicate_prediction( + raw_response: httpx.Response, + client: AsyncHTTPHandler, +) -> Mapping[str, object]: + prediction: Final = replicate_prediction(raw_response) + if replicate_prediction_is_done(prediction): + return prediction + poll_url: Final = _poll_url(prediction, raw_response) + headers: Final = replicate_poll_headers(raw_response.request) + for _ in range(DEFAULT_REPLICATE_SYNC_POLLING_RETRIES): + await asyncio.sleep(DEFAULT_REPLICATE_POLLING_DELAY_SECONDS) + if (polled := _finished_prediction(await client.get(url=poll_url, headers=headers))) is not None: # pyright: ignore[reportUnknownMemberType] # HTTP handler stubs are untyped + return polled + raise ReplicateError( + status_code=504, + message=f"Replicate prediction did not finish after {DEFAULT_REPLICATE_SYNC_POLLING_RETRIES} polls", + headers=raw_response.headers, + ) + + +def _download_too_large(url: str, error: HTTPResponseLimitError) -> ReplicateError: + return ReplicateError( + status_code=502, + message=f"Replicate output at {url} exceeds REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES={REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES}: {error}", + headers={}, + ) + + +def replicate_download(client: HTTPHandler, url: str) -> httpx.Response: + try: + response: Final = safe_get(client, url, max_response_bytes=REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES) + except HTTPResponseLimitError as error: + raise _download_too_large(url, error) from error + response.raise_for_status() + return response + + +async def async_replicate_download(client: AsyncHTTPHandler, url: str) -> httpx.Response: + try: + response: Final = await async_safe_get(client, url, max_response_bytes=REPLICATE_MAX_OUTPUT_DOWNLOAD_BYTES) + except HTTPResponseLimitError as error: + raise _download_too_large(url, error) from error + response.raise_for_status() + return response diff --git a/litellm/llms/replicate/image_generation/transformation.py b/litellm/llms/replicate/image_generation/transformation.py new file mode 100644 index 00000000000..b80a5f6811d --- /dev/null +++ b/litellm/llms/replicate/image_generation/transformation.py @@ -0,0 +1,228 @@ +import base64 +import math +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +import httpx +from pydantic import ConfigDict, TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.image_generation.transformation import BaseImageGenerationConfig +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params + get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private +) +from litellm.llms.replicate.common_utils import ( + ReplicateError, + async_replicate_download, + async_wait_for_replicate_prediction, + replicate_api_base, + replicate_auth_headers, + replicate_download, + replicate_output_urls, + replicate_positive_int, + replicate_prediction_failure, + replicate_prediction_input, + replicate_prediction_request, + wait_for_replicate_prediction, +) +from litellm.types.llms.openai import AllMessageValues, OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse, LlmProviders + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + +_ImageParams: TypeAlias = dict[str, object] +_ImageHeaders: TypeAlias = dict[str, str] + + +def _aspect_ratio(size: object) -> Mapping[str, str]: + if not isinstance(size, str) or size.lower().count("x") != 1: + return MappingProxyType({}) + width_text, height_text = size.lower().split("x") + if not (width_text.isdigit() and height_text.isdigit()) or int(width_text) <= 0 or int(height_text) <= 0: + return MappingProxyType({}) + divisor: Final = math.gcd(int(width_text), int(height_text)) + return MappingProxyType({"aspect_ratio": f"{int(width_text) // divisor}:{int(height_text) // divisor}"}) + + +_NATIVE_INPUT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) + + +def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler: + return get_httpx_client(params={"ssl_verify": ssl_verify}) + + +def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify}) + + +class ReplicateImageGenerationConfig(BaseImageGenerationConfig): + def __init__( + self, + sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client, + async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client, + ) -> None: + self._sync_client_factory: Final = sync_client_factory + self._async_client_factory: Final = async_client_factory + + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: base class contract returns a plain list + return ["n", "response_format", "size"] + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: _ImageParams, + model: str, + drop_params: bool, + ) -> _ImageParams: + supported: Final = frozenset(self.get_supported_openai_params(model)) + unsupported: Final = tuple(key for key in non_default_params if key not in supported) + if unsupported and not drop_params: + raise ValueError( + f"Parameters {', '.join(unsupported)} are not supported for model {model}. " + f"Supported parameters are {sorted(supported)}. Set drop_params=True to drop unsupported parameters." + ) + count: Final = non_default_params.get("n") + response_format: Final = non_default_params.get("response_format") + return { + **optional_params, + **({"num_outputs": count} if isinstance(count, int) and not isinstance(count, bool) else {}), + **_aspect_ratio(non_default_params.get("size")), + **({"response_format": response_format} if isinstance(response_format, str) else {}), + } + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: _ImageParams, + litellm_params: _ImageParams, + stream: bool | None = None, + ) -> str: + return replicate_prediction_request(replicate_api_base(api_base), model, MappingProxyType({})).url + + def validate_environment( + self, + headers: _ImageHeaders, + model: str, + messages: list[AllMessageValues], + optional_params: _ImageParams, + litellm_params: _ImageParams, + api_key: str | None = None, + api_base: str | None = None, + ) -> _ImageHeaders: + return {**replicate_auth_headers(headers, api_key), "Prefer": "wait"} + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: _ImageParams, + litellm_params: _ImageParams, + headers: _ImageHeaders, + ) -> _ImageParams: + extra_body: Final = optional_params.get("extra_body") + prediction_input: Final = { + **{key: value for key, value in optional_params.items() if key not in ("response_format", "extra_body")}, + **(_NATIVE_INPUT.validate_python(extra_body) if isinstance(extra_body, Mapping) else {}), + "prompt": prompt, + } + return dict(replicate_prediction_request("", model, prediction_input).body) + + @staticmethod + def _image_urls(prediction: Mapping[str, object], raw_response: httpx.Response) -> tuple[str, ...]: + failure: Final = replicate_prediction_failure(prediction) + if failure is not None: + raise ReplicateError(status_code=raw_response.status_code, message=failure, headers=raw_response.headers) + urls: Final = replicate_output_urls(prediction.get("output")) + if not urls: + raise ReplicateError( + status_code=raw_response.status_code, + message="Replicate prediction finished without image output", + headers=raw_response.headers, + ) + return urls + + @staticmethod + def _native_size(prediction: Mapping[str, object]) -> Mapping[str, str]: + prediction_input: Final = replicate_prediction_input(prediction) + width: Final = replicate_positive_int(prediction_input.get("width")) + height: Final = replicate_positive_int(prediction_input.get("height")) + return {"size": f"{width}x{height}"} if width is not None and height is not None else {} + + @staticmethod + def _image_objects(urls: tuple[str, ...], contents: tuple[bytes, ...] | None) -> tuple[ImageObject, ...]: + if contents is None: + return tuple(ImageObject(url=url, b64_json=None) for url in urls) + return tuple(ImageObject(url=None, b64_json=base64.b64encode(content).decode()) for content in contents) + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: _ImageParams, + optional_params: _ImageParams, + litellm_params: _ImageParams, + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ImageResponse: + client: Final = self._sync_client_factory(litellm_params.get("ssl_verify")) + prediction: Final = wait_for_replicate_prediction(raw_response, client) + urls: Final = self._image_urls(prediction, raw_response) + contents: Final = ( + tuple(self._download(client, url) for url in urls) + if optional_params.get("response_format") == "b64_json" + else None + ) + return model_response.model_copy( + update={"data": list(self._image_objects(urls, contents)), **self._native_size(prediction)} + ) + + async def async_transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: _ImageParams, + optional_params: _ImageParams, + litellm_params: _ImageParams, + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ImageResponse: + client: Final = self._async_client_factory(litellm_params.get("ssl_verify")) + prediction: Final = await async_wait_for_replicate_prediction(raw_response, client) + urls: Final = self._image_urls(prediction, raw_response) + contents: Final = ( + tuple([await self._async_download(client, url) for url in urls]) + if optional_params.get("response_format") == "b64_json" + else None + ) + return model_response.model_copy( + update={"data": list(self._image_objects(urls, contents)), **self._native_size(prediction)} + ) + + @staticmethod + def _download(client: HTTPHandler, url: str) -> bytes: + return replicate_download(client, url).content + + @staticmethod + async def _async_download(client: AsyncHTTPHandler, url: str) -> bytes: + return (await async_replicate_download(client, url)).content + + def get_error_class( + self, error_message: str, status_code: int, headers: _ImageHeaders | httpx.Headers + ) -> BaseLLMException: + return ReplicateError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/replicate/text_to_speech/transformation.py b/litellm/llms/replicate/text_to_speech/transformation.py new file mode 100644 index 00000000000..d75fc29ba54 --- /dev/null +++ b/litellm/llms/replicate/text_to_speech/transformation.py @@ -0,0 +1,204 @@ +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +import httpx +from pydantic import ConfigDict, TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig, TextToSpeechRequestData +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params + get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private +) +from litellm.llms.replicate.common_utils import ( + ReplicateError, + async_replicate_download, + async_wait_for_replicate_prediction, + replicate_api_base, + replicate_auth_headers, + replicate_download, + replicate_model_owner, + replicate_output_urls, + replicate_positive_int, + replicate_prediction_failure, + replicate_prediction_metrics, + replicate_prediction_request, + wait_for_replicate_prediction, +) +from litellm.types.utils import LlmProviders, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent + +_SpeechParams: TypeAlias = dict[str, object] +_SpeechHeaders: TypeAlias = dict[str, str] +_NATIVE_INPUT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_VOICE_KEY_BY_OWNER: Final[Mapping[str, str]] = MappingProxyType( + {"minimax": "voice_id", "inworld": "voice_id", "qwen": "speaker"} +) + + +def _voice_key(model: str) -> str: + return _VOICE_KEY_BY_OWNER.get(replicate_model_owner(model), "voice") + + +def _get_replicate_httpx_client(ssl_verify: object) -> HTTPHandler: + return get_httpx_client(params={"ssl_verify": ssl_verify}) + + +def _get_replicate_async_httpx_client(ssl_verify: object) -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE, params={"ssl_verify": ssl_verify}) + + +def _ssl_verify(logging_obj: "LiteLLMLoggingObj") -> object: + litellm_params: Final = getattr(logging_obj, "litellm_params", None) + return litellm_params.get("ssl_verify") if isinstance(litellm_params, Mapping) else None + + +class ReplicateTextToSpeechConfig(BaseTextToSpeechConfig): + def __init__( + self, + sync_client_factory: Callable[[object], HTTPHandler] = _get_replicate_httpx_client, + async_client_factory: Callable[[object], AsyncHTTPHandler] = _get_replicate_async_httpx_client, + ) -> None: + super().__init__() + self._sync_client_factory: Final = sync_client_factory + self._async_client_factory: Final = async_client_factory + + def get_supported_openai_params( + self, model: str + ) -> list[str]: # mutable-ok: base class contract returns a plain list + return ["voice", "speed"] + + def map_openai_params( + self, + model: str, + optional_params: Mapping[str, object], + voice: object = None, + drop_params: bool = False, + kwargs: Mapping[str, object] | None = None, + ) -> tuple[str | None, _SpeechParams]: + speed: Final = optional_params.get("speed") + extra_body: Final = (kwargs or {}).get("extra_body") + return ( + voice if isinstance(voice, str) and voice else None, + { + **(_NATIVE_INPUT.validate_python(extra_body) if isinstance(extra_body, Mapping) else {}), + **({"speed": speed} if isinstance(speed, (int, float)) and not isinstance(speed, bool) else {}), + }, + ) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None = None, + api_base: str | None = None, + ) -> _SpeechHeaders: + return {**replicate_auth_headers(headers, api_key), "Prefer": "wait"} + + def get_complete_url(self, model: str, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + return replicate_prediction_request(replicate_api_base(api_base), model, MappingProxyType({})).url + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + headers: Mapping[str, str], + ) -> TextToSpeechRequestData: + prediction_input: Final = { + **optional_params, + **({_voice_key(model): voice} if voice else {}), + "text": input, + } + return {"dict_body": dict(replicate_prediction_request("", model, prediction_input).body)} + + @staticmethod + def _audio_url(prediction: Mapping[str, object], raw_response: httpx.Response) -> str: + failure: Final = replicate_prediction_failure(prediction) + if failure is not None: + raise ReplicateError(status_code=raw_response.status_code, message=failure, headers=raw_response.headers) + urls: Final = replicate_output_urls(prediction.get("output")) + if not urls: + raise ReplicateError( + status_code=raw_response.status_code, + message="Replicate prediction finished without audio output", + headers=raw_response.headers, + ) + return urls[0] + + @staticmethod + def _usage(prediction: Mapping[str, object]) -> Usage | None: + metrics: Final = replicate_prediction_metrics(prediction) + prompt_tokens: Final = replicate_positive_int( + metrics.get("token_input_count", metrics.get("input_token_count")) + ) + completion_tokens: Final = replicate_positive_int( + metrics.get("token_output_count", metrics.get("output_token_count")) + ) + if prompt_tokens is None and completion_tokens is None: + return None + return Usage( + prompt_tokens=prompt_tokens or 0, + completion_tokens=completion_tokens or 0, + total_tokens=(prompt_tokens or 0) + (completion_tokens or 0), + ) + + @staticmethod + def _binary_response( + audio_response: httpx.Response, raw_response: httpx.Response, usage: Usage | None + ) -> "HttpxBinaryResponseContent": + from litellm.types.llms.openai import HttpxBinaryResponseContent + + audio_response.raise_for_status() + return HttpxBinaryResponseContent( + httpx.Response( + status_code=200, + headers={ + "content-type": audio_response.headers.get("content-type", "application/octet-stream"), + "content-length": str(len(audio_response.content)), + }, + content=audio_response.content, + request=raw_response.request, + ), + usage=usage, + ) + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + client: Final = self._sync_client_factory(_ssl_verify(logging_obj)) + prediction: Final = wait_for_replicate_prediction(raw_response, client) + audio_url: Final = self._audio_url(prediction, raw_response) + return self._binary_response(replicate_download(client, audio_url), raw_response, self._usage(prediction)) + + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + client: Final = self._async_client_factory(_ssl_verify(logging_obj)) + prediction: Final = await async_wait_for_replicate_prediction(raw_response, client) + audio_url: Final = self._audio_url(prediction, raw_response) + return self._binary_response( + await async_replicate_download(client, audio_url), raw_response, self._usage(prediction) + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: _SpeechHeaders | httpx.Headers, + ) -> BaseLLMException: + return ReplicateError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/replicate/videos/transformation.py b/litellm/llms/replicate/videos/transformation.py new file mode 100644 index 00000000000..cd01c832686 --- /dev/null +++ b/litellm/llms/replicate/videos/transformation.py @@ -0,0 +1,486 @@ +import math +import sys +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, TypeAlias + +import httpx +from httpx._types import FileContent, RequestFiles + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.videos.async_task_utils import unsupported_video_fields +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params + get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private +) +from litellm.llms.replicate.common_utils import ( + ReplicateError, + async_replicate_download, + replicate_api_base, + replicate_auth_headers, + replicate_download, + replicate_model_owner, + replicate_output_urls, + replicate_prediction, + replicate_prediction_failure, + replicate_prediction_input, + replicate_prediction_request, + replicate_prediction_string, + replicate_prediction_url, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.types.videos.main import CharacterObject, VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import decode_video_id_with_provider, encode_video_id_with_provider + +_REPLICATE_PROVIDER: Final[str] = LlmProviders.REPLICATE.value +_STATUS_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "starting": "queued", + "processing": "in_progress", + "succeeded": "completed", + "failed": "failed", + "canceled": "failed", + } +) +_VideoParams: TypeAlias = dict[str, object] +_VideoHeaders: TypeAlias = dict[str, str] +_VideoStringParams: TypeAlias = dict[str, str] +_VideoFiles: TypeAlias = list[object] + + +@dataclass(frozen=True, slots=True) +class _ModelProfile: + resolution_tiers: tuple[tuple[int, str], ...] + aspect_ratio_key: str + reference_key: str + + +_GENERIC_PROFILE: Final[_ModelProfile] = _ModelProfile( + resolution_tiers=((480, "480p"), (720, "720p"), (1080, "1080p"), (sys.maxsize, "4k")), + aspect_ratio_key="aspect_ratio", + reference_key="image", +) +_MINIMAX_PROFILE: Final[_ModelProfile] = _ModelProfile( + resolution_tiers=((480, "480P"), (768, "768P"), (1440, "2K"), (sys.maxsize, "4K")), + aspect_ratio_key="ratio", + reference_key="first_frame_image", +) + + +def _profile_for_model(model: str) -> _ModelProfile: + return _MINIMAX_PROFILE if replicate_model_owner(model) == "minimax" else _GENERIC_PROFILE + + +def _duration_value(seconds: object) -> int | float: + if isinstance(seconds, bool) or not isinstance(seconds, (int, float, str)): + raise TypeError("Replicate seconds must be a numeric value") + try: + duration: Final = float(seconds) + except ValueError as error: + raise ValueError("Replicate seconds must be a numeric value") from error + return int(duration) if duration.is_integer() else duration + + +def _size_params(size: object, profile: _ModelProfile) -> Mapping[str, object]: + if not isinstance(size, str): + return MappingProxyType({}) + normalized: Final = size.strip().lower() + named_resolution: Final = next( + (resolution for _, resolution in profile.resolution_tiers if resolution.lower() == normalized), None + ) + if named_resolution is not None: + return MappingProxyType({"resolution": named_resolution}) + if normalized.count("x") != 1: + return MappingProxyType({}) + width_text, height_text = normalized.split("x") + if not (width_text.isdigit() and height_text.isdigit()) or int(width_text) <= 0 or int(height_text) <= 0: + return MappingProxyType({}) + width: Final = int(width_text) + height: Final = int(height_text) + divisor: Final = math.gcd(width, height) + short_side: Final = min(width, height) + resolution: Final = next( + (resolution for threshold, resolution in profile.resolution_tiers if short_side <= threshold), + profile.resolution_tiers[-1][1], + ) + return MappingProxyType( + {"resolution": resolution, profile.aspect_ratio_key: f"{width // divisor}:{height // divisor}"} + ) + + +def _usage_from_input(prediction_input: Mapping[str, object]) -> _VideoParams: + duration: Final = prediction_input.get("duration") + resolution: Final = prediction_input.get("resolution") + return { + key: value + for key, value in ( + ( + "duration_seconds", + float(duration) if isinstance(duration, (int, float)) and not isinstance(duration, bool) else None, + ), + ("video_resolution", resolution if isinstance(resolution, str) else None), + ) + if value is not None + } + + +def _video_object( + prediction: Mapping[str, object], + model: str | None, + custom_llm_provider: str | None, + created_at: int, +) -> VideoObject: + prediction_id: Final = replicate_prediction_string(prediction, "id") or "" + failure: Final = replicate_prediction_failure(prediction) + raw_status: Final = replicate_prediction_string(prediction, "status") or "starting" + prediction_input: Final = replicate_prediction_input(prediction) + duration: Final = prediction_input.get("duration") + resolution: Final = prediction_input.get("resolution") + return VideoObject( + id=encode_video_id_with_provider(prediction_id, custom_llm_provider or _REPLICATE_PROVIDER, model), + object="video", + status="failed" if failure else _STATUS_MAP.get(raw_status, "queued"), + created_at=created_at, + model=model, + seconds=str(duration) if isinstance(duration, (int, float)) and not isinstance(duration, bool) else None, + size=resolution if isinstance(resolution, str) else None, + error={"code": "replicate_error", "message": failure} if failure else None, + usage=dict(_usage_from_input(prediction_input)), + ) + + +def _owned_prediction(raw_response: httpx.Response) -> Mapping[str, object]: + prediction: Final = replicate_prediction(raw_response) + requested: Final = raw_response.request.url.fragment + if not requested or requested.startswith("deployments/"): + return prediction + requested_path, _, requested_version = requested.partition(":") + actual_path: Final = replicate_prediction_string(prediction, "model") + actual_version: Final = replicate_prediction_string(prediction, "version") + if (actual_path is not None and actual_path != requested_path) or ( + requested_version and actual_version is not None and actual_version != requested_version + ): + raise ReplicateError( + status_code=404, + message=f"Replicate prediction {replicate_prediction_string(prediction, 'id')} does not belong to {requested}", + headers=raw_response.headers, + ) + return prediction + + +def _get_replicate_async_httpx_client() -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=LlmProviders.REPLICATE) + + +class ReplicateVideoConfig(BaseVideoConfig): + def __init__( + self, + sync_client_factory: Callable[[], HTTPHandler] = get_httpx_client, + async_client_factory: Callable[[], AsyncHTTPHandler] = _get_replicate_async_httpx_client, + ) -> None: + super().__init__() + self._sync_client_factory: Final = sync_client_factory + self._async_client_factory: Final = async_client_factory + + def get_supported_openai_params( + self, model: str + ) -> list[str]: # mutable-ok: base class contract returns a plain list + return ["model", "prompt", "input_reference", "seconds", "size", "extra_headers"] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> _VideoParams: + unsupported: Final = unsupported_video_fields( + video_create_optional_params, + model, + _REPLICATE_PROVIDER, + drop_params, + frozenset({"generate_audio", "aspect_ratio", "seed", "duration"}), + ) + supported: Final = frozenset(self.get_supported_openai_params(model)) + profile: Final = _profile_for_model(model) + input_reference: Final = video_create_optional_params.get("input_reference") + if "input_reference" in video_create_optional_params and not isinstance(input_reference, str): + raise ValueError("Replicate needs a public image URL for input_reference") + reference_params: Final[Mapping[str, object]] = ( + MappingProxyType({profile.reference_key: input_reference}) + if isinstance(input_reference, str) + else MappingProxyType({}) + ) + duration_params: Final[Mapping[str, object]] = ( + MappingProxyType({"duration": _duration_value(video_create_optional_params["seconds"])}) + if "seconds" in video_create_optional_params + else MappingProxyType({}) + ) + size_params: Final[Mapping[str, object]] = ( + _size_params(video_create_optional_params["size"], profile) + if "size" in video_create_optional_params + else MappingProxyType({}) + ) + return { + **reference_params, + **duration_params, + **size_params, + **{ + key: value + for key, value in video_create_optional_params.items() + if key not in supported and key not in unsupported and key != "user" + }, + } + + def validate_environment( + self, + headers: _VideoHeaders, + model: str, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> _VideoHeaders: + return replicate_auth_headers( + headers, api_key or (litellm_params.api_key if litellm_params is not None else None) + ) + + def get_complete_url(self, model: str, api_base: str | None, litellm_params: _VideoParams) -> str: + return replicate_api_base(api_base) + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: _VideoParams, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + ) -> tuple[_VideoParams, RequestFiles, str]: + prediction_input: Final = { + **{key: value for key, value in video_create_optional_request_params.items() if key != "model"}, + "prompt": prompt, + } + request: Final = replicate_prediction_request(api_base, model, prediction_input) + return dict(request.body), [], request.url + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: object, + custom_llm_provider: str | None = None, + request_data: Mapping[str, object] | None = None, + ) -> VideoObject: + return _video_object(replicate_prediction(raw_response), model, custom_llm_provider, int(time.time())) + + @staticmethod + def _decode_video_id(video_id: str) -> tuple[str, str | None]: + decoded: Final = decode_video_id_with_provider(video_id) + return decoded.get("video_id", video_id), decoded.get("model_id") + + @classmethod + def _prediction_url_for_video(cls, video_id: str, api_base: str) -> str: + prediction_id, model_id = cls._decode_video_id(video_id) + prediction_url: Final = httpx.URL(replicate_prediction_url(api_base, prediction_id)) + return str(prediction_url.copy_with(fragment=model_id) if model_id else prediction_url) + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + ) -> tuple[str, _VideoParams]: + return self._prediction_url_for_video(video_id, api_base), {} + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: object, + custom_llm_provider: str | None = None, + client: HTTPHandler | None = None, + ) -> VideoObject: + prediction: Final = _owned_prediction(raw_response) + model: Final = raw_response.request.url.fragment or replicate_prediction_string(prediction, "model") + return _video_object(prediction, model, custom_llm_provider, 0) + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + variant: str | None = None, + ) -> tuple[str, _VideoStringParams]: + return self._prediction_url_for_video(video_id, api_base), {} + + @staticmethod + def _video_url(raw_response: httpx.Response) -> str: + prediction: Final = _owned_prediction(raw_response) + failure: Final = replicate_prediction_failure(prediction) + if failure is not None: + raise ReplicateError(status_code=raw_response.status_code, message=failure, headers=raw_response.headers) + urls: Final = replicate_output_urls(prediction.get("output")) + if not urls: + raise ReplicateError( + status_code=raw_response.status_code, + message=f"Replicate prediction has no video output yet (status {replicate_prediction_string(prediction, 'status')})", + headers=raw_response.headers, + ) + return urls[0] + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: object, + client: "HTTPHandler | None" = None, + ) -> bytes: + download_client: Final = client if client is not None else self._sync_client_factory() + return replicate_download(download_client, self._video_url(raw_response)).content + + async def async_transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: object, + client: "AsyncHTTPHandler | None" = None, + ) -> bytes: + download_client: Final = client if client is not None else self._async_client_factory() + return (await async_replicate_download(download_client, self._video_url(raw_response))).content + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + extra_body: Mapping[str, object] | None = None, + ) -> tuple[str, _VideoParams]: + raise NotImplementedError("video remix is not supported for Replicate") + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: object, + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("video remix is not supported for Replicate") + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: Mapping[str, object] | None = None, + ) -> tuple[str, _VideoParams]: + raise NotImplementedError("video listing is not supported for Replicate") + + def transform_video_list_response( + self, + raw_response: httpx.Response, + logging_obj: object, + custom_llm_provider: str | None = None, + ) -> _VideoStringParams: + raise NotImplementedError("video listing is not supported for Replicate") + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + ) -> tuple[str, _VideoParams]: + raise NotImplementedError("video delete is not supported for Replicate") + + def transform_video_delete_response(self, raw_response: httpx.Response, logging_obj: object) -> VideoObject: + raise NotImplementedError("video delete is not supported for Replicate") + + def transform_video_create_character_request( + self, + name: str, + video: object, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + ) -> tuple[str, _VideoFiles]: + raise NotImplementedError("video character creation is not supported for Replicate") + + def transform_video_create_character_response( + self, + raw_response: httpx.Response, + logging_obj: object, + ) -> CharacterObject: + raise NotImplementedError("video character creation is not supported for Replicate") + + def transform_video_get_character_request( + self, + character_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + ) -> tuple[str, _VideoParams]: + raise NotImplementedError("video character retrieval is not supported for Replicate") + + def transform_video_get_character_response( + self, + raw_response: httpx.Response, + logging_obj: object, + ) -> CharacterObject: + raise NotImplementedError("video character retrieval is not supported for Replicate") + + def transform_video_edit_request( + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + video_file: FileContent | None = None, + extra_body: Mapping[str, object] | None = None, + prefetched_source_data: Mapping[str, object] | None = None, + ) -> tuple[str, Mapping[str, object], RequestFiles | None]: + raise NotImplementedError("video edit is not supported for Replicate") + + def transform_video_edit_response( + self, + raw_response: httpx.Response, + logging_obj: object, + custom_llm_provider: str | None = None, + request_data: Mapping[str, object] | None = None, + ) -> VideoObject: + raise NotImplementedError("video edit is not supported for Replicate") + + def transform_video_extension_request( + self, + prompt: str, + video_id: str, + seconds: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: _VideoHeaders, + extra_body: Mapping[str, object] | None = None, + ) -> tuple[str, _VideoParams]: + raise NotImplementedError("video extension is not supported for Replicate") + + def transform_video_extension_response( + self, + raw_response: httpx.Response, + logging_obj: object, + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("video extension is not supported for Replicate") + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: _VideoHeaders | httpx.Headers, + ) -> BaseLLMException: + return ReplicateError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index a852ae8b1a1..c20f0df6768 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.optional_imports import ensure_optional_import +from litellm.llms.base_llm.embedding.transformation import extra_body_fields from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, pop_aws_auth_params from litellm.llms.bedrock.web_identity_session_policy import SAGEMAKER_SESSION_POLICY, WebIdentitySessionPolicy from litellm.llms.custom_httpx.http_handler import ( @@ -594,8 +595,11 @@ class SagemakerLLM(BaseAWSLLM): inference_component_name: Final = embedding_params.get("model_id") if inference_component_name is not None and not isinstance(inference_component_name, str): raise SagemakerError(status_code=400, message="model_id must be the inference component name as a string") - body_params: Final = {k: v for k, v in embedding_params.items() if k != "model_id"} - request_data: Final = provider_config.transform_embedding_request(endpoint_name, input, body_params, {}) + body_params: Final = {k: v for k, v in embedding_params.items() if k not in ("model_id", "extra_body")} + request_data: Final = { + **extra_body_fields(embedding_params), + **provider_config.transform_embedding_request(endpoint_name, input, body_params, {}), + } data: Final = json.dumps(request_data).encode("utf-8") invoke_args: Final = _InvokeEndpointArgs( EndpointName=endpoint_name, diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 7a7afcb83f1..478291bd7db 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -66,6 +66,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( + embed_content_config_fields, transform_openai_input_gemini_embed_content, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig @@ -104,6 +105,11 @@ _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = ( ("taskType", "task_type"), ("title", "title"), ) +_EMBED_CONTENT_CONFIG_REQUEST_FIELD_BY_GEMINI_PARAM: Final = ( + ("documentOcr", "document_ocr"), + ("audioTrackExtraction", "audio_track_extraction"), + ("autoTruncate", "auto_truncate"), +) _VERTEX_BATCH_FANNED_OUT_KEY_PATTERN: Final = re.compile(r"(?P[^#]*)#(?P\d+)/(?P\d+)") _JSONL_NEWLINE: Final = b"\n" _BATCH_OUTPUT_FIRST_ROW_PEEK_LIMIT_BYTES: Final = 32 * 1024 * 1024 @@ -659,6 +665,16 @@ def _vertex_batch_embeddings_key(custom_id: str, index: int, total: int) -> str: return encoded_custom_id if total < 2 else f"{encoded_custom_id}#{index}/{total}" +def _vertex_embed_content_config(embed_content_request: Mapping[str, object]) -> Mapping[str, object]: + embed_content_config: Final = embed_content_config_fields(embed_content_request) + config_fields: Final = { + request_field: embed_content_config[gemini_param] + for gemini_param, request_field in _EMBED_CONTENT_CONFIG_REQUEST_FIELD_BY_GEMINI_PARAM + if gemini_param in embed_content_config + } + return {"embed_content_config": config_fields} if config_fields else {} + + def _vertex_embeddings_row(key: str | None, embed_content_request: Mapping[str, object]) -> Mapping[str, object]: """ One Vertex Gemini Embedding batch input row. @@ -674,6 +690,7 @@ def _vertex_embeddings_row(key: str | None, embed_content_request: Mapping[str, for gemini_param, request_field in _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM if gemini_param in embed_content_request }, + **_vertex_embed_content_config(embed_content_request), } if key is None: return {"request": request} @@ -694,9 +711,10 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( `GenerateContentRequest`) and that the `custom_id` round-trips through the top-level `key`. An `EmbedContentRequest` returns exactly one vector, so an entry whose `input` is an array fans out into one row per element and is reassembled on the way back. - The docs put the per-row config in an `embed_content_config` sibling of `request`, - but the API rejects that key outright and fails the whole batch job, so the config - fields go inside the `EmbedContentRequest` itself. + An `embed_content_config` sibling of `request` is rejected outright and fails the whole + batch job, so the config fields go inside the `EmbedContentRequest` itself. The fields + that only exist on its `embed_content_config` (`document_ocr`, `audio_track_extraction`) + go in an `embed_content_config` inside `request`, alongside `auto_truncate`. API Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/batch-prediction-genai-embeddings """ diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 96a785c06c9..2fa6df8ca75 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -5,11 +5,13 @@ Why separate file? Make it easy to see how transformation works """ from collections.abc import Iterator, Mapping, Sequence +from types import MappingProxyType from typing import Annotated, Final, Literal, cast from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError +from litellm.llms.base_llm.embedding.transformation import extra_body_fields from litellm.llms.vertex_ai.common_utils import GEMINI_FILES_API_URI_PREFIX, gemini_video_metadata_from_openai from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.vertex_ai import ( @@ -377,16 +379,59 @@ def _build_part_for_input( _SUPPORTED_EMBED_PARAMS: Final = {"outputDimensionality", "taskType", "title"} +_GEMINI_PARAM_BY_OPENAI_PARAM: Final = (("dimensions", "outputDimensionality"), ("task_type", "taskType")) +_EMBED_CONTENT_CONFIG_FIELD: Final = "embedContentConfig" +_STRING_KEYED_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_EMBED_CONTENT_CONFIG_FIELD_BY_PARAM: Final = MappingProxyType( + { + "document_ocr": "documentOcr", + "documentOcr": "documentOcr", + "audio_track_extraction": "audioTrackExtraction", + "audioTrackExtraction": "audioTrackExtraction", + "auto_truncate": "autoTruncate", + "autoTruncate": "autoTruncate", + } +) + + +def embed_content_config_fields(request_body: Mapping[str, object]) -> Mapping[str, object]: + """The `embedContentConfig` object of a Gemini embedding request body, empty when the body has none.""" + config: Final = request_body.get(_EMBED_CONTENT_CONFIG_FIELD) + return _STRING_KEYED_FIELDS.validate_python(config) if isinstance(config, Mapping) else {} + + +def _embed_content_config(optional_params: Mapping[str, object]) -> Mapping[str, object]: + """The named config params under `embedContentConfig`, on top of any config the caller sent in `extra_body`.""" + named_config: Final = { + field: optional_params[param] + for param, field in _EMBED_CONTENT_CONFIG_FIELD_BY_PARAM.items() + if optional_params.get(param) is not None + } + if not named_config: + return {} + return { + _EMBED_CONTENT_CONFIG_FIELD: { + **embed_content_config_fields(extra_body_fields(optional_params)), + **named_config, + } + } def _filter_embed_params(optional_params: dict) -> dict: """Map and filter optional_params to only include Gemini embedding fields.""" - gemini_params: Final = optional_params.copy() - if "dimensions" in gemini_params: - gemini_params["outputDimensionality"] = gemini_params.pop("dimensions") - if "task_type" in gemini_params: - gemini_params["taskType"] = gemini_params.pop("task_type") - return {k: v for k, v in gemini_params.items() if k in _SUPPORTED_EMBED_PARAMS} + params: Final = _STRING_KEYED_FIELDS.validate_python(optional_params) + gemini_params: Final = { + **params, + **{ + gemini_param: params[openai_param] + for openai_param, gemini_param in _GEMINI_PARAM_BY_OPENAI_PARAM + if openai_param in params + }, + } + return { + **{k: v for k, v in gemini_params.items() if k in _SUPPORTED_EMBED_PARAMS}, + **_embed_content_config(params), + } def transform_openai_input_gemini_content( @@ -414,6 +459,7 @@ def transform_openai_input_gemini_content( gemini_model_name: Final = f"models/{model}" gemini_params: Final = _filter_embed_params(optional_params) + extra_body: Final = extra_body_fields(optional_params) input_list: Final = _top_level_elements(input) requests: Final[list[EmbedContentRequest]] = [] @@ -432,7 +478,11 @@ def transform_openai_input_gemini_content( content=ContentType(parts=parts), **gemini_params, ) - requests.append(request) + requests.append( + cast( # cast-ok: extra_body carries raw Gemini fields the TypedDict does not list + EmbedContentRequest, {**extra_body, **request} + ) + ) return VertexAIBatchEmbeddingsRequestBody(requests=requests) @@ -472,6 +522,7 @@ def transform_openai_input_gemini_embed_content( parts.append(_build_part_for_input(element, resolved_files=resolved_files, drop_params=drop_params)) request_body: Final[dict] = { + **extra_body_fields(optional_params), "content": ContentType(parts=parts), **gemini_params, } @@ -550,10 +601,10 @@ def _usage_from_embed_content_response( ), ) - text_tokens: Final = _tokens_for_modality(details, "TEXT") audio_tokens: Final = _tokens_for_modality(details, "AUDIO") - image_tokens: Final = _tokens_for_modality(details, "IMAGE") + image_tokens: Final = _tokens_for_modality(details, "IMAGE") + _tokens_for_modality(details, "DOCUMENT") video_tokens: Final = _tokens_for_modality(details, "VIDEO") + text_tokens: Final = sum(detail["tokenCount"] for detail in details) - audio_tokens - image_tokens - video_tokens return Usage( prompt_tokens=prompt_tokens, diff --git a/litellm/llms/vertex_ai/interactions/transformation.py b/litellm/llms/vertex_ai/interactions/transformation.py index b8e13059917..d81818d021d 100644 --- a/litellm/llms/vertex_ai/interactions/transformation.py +++ b/litellm/llms/vertex_ai/interactions/transformation.py @@ -146,4 +146,4 @@ class VertexAIInteractionsConfig(VertexBase, GoogleAIStudioInteractionsConfig): litellm_params: GenericLiteLLMParams, headers: Mapping[str, str], ) -> tuple[str, dict]: # mutable-ok: BaseInteractionsAPIConfig declares a plain-dict request body - return self._interaction_by_id_request(interaction_id, api_base, litellm_params, url_suffix=":cancel") + return self._interaction_by_id_request(interaction_id, api_base, litellm_params, url_suffix="/cancel") diff --git a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py index c282ae22ea6..7c276b43dc7 100644 --- a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( ) from litellm.utils import count_characters, is_base64_encoded -from ...base_llm.embedding.transformation import BaseEmbeddingConfig +from ...base_llm.embedding.transformation import BaseEmbeddingConfig, extra_body_fields from ..common_utils import VertexAIError @@ -189,8 +189,11 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig): request_data["instances"] = [vertex_request_instance] + extra_body: Final = extra_body_fields(optional_params) if "outputDimensionality" in optional_params: - request_data["parameters"] = {"dimension": optional_params["outputDimensionality"]} + request_data["parameters"] = {**extra_body, "dimension": optional_params["outputDimensionality"]} + elif extra_body: + request_data["parameters"] = dict(extra_body) return cast(dict, request_data) diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index 96fccf007a5..5d99ce6d90f 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,10 +1,11 @@ import types from collections.abc import Mapping -from typing import Final, Literal +from typing import Final, Literal, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict +from litellm.llms.base_llm.embedding.transformation import split_extra_body from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage @@ -164,7 +165,12 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): vertex_text_embedding_input_list.append(embedding_input) vertex_request["instances"] = vertex_text_embedding_input_list - vertex_request["parameters"] = EmbeddingParameters(**optional_params) + body_params, extra_body = split_extra_body(optional_params) + vertex_request["parameters"] = ( + cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list + EmbeddingParameters, {**extra_body, **body_params} + ) + ) if labels: vertex_request["labels"] = labels @@ -205,7 +211,12 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): vertex_text_embedding_input_list.append(embedding_input) vertex_request["instances"] = vertex_text_embedding_input_list - vertex_request["parameters"] = TextEmbeddingFineTunedParameters(**optional_params) + body_params, extra_body = split_extra_body(optional_params) + vertex_request["parameters"] = ( + cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list + TextEmbeddingFineTunedParameters, {**extra_body, **body_params} + ) + ) # Remove 'shared_session' from parameters if present if vertex_request["parameters"] is not None and "shared_session" in vertex_request["parameters"]: del vertex_request["parameters"]["shared_session"] diff --git a/litellm/main.py b/litellm/main.py index e4110455e24..6798242773b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -237,6 +237,7 @@ from .litellm_core_utils.core_helpers import ( drops_unsupported_params, normalize_drop_params, safe_deep_copy, + without, ) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, @@ -807,27 +808,29 @@ async def acompletion( prompt_management_logger, AnthropicCacheControlHook ) - if ( - isinstance(litellm_logging_obj, LiteLLMLoggingObj) + prompt_hook_result: Final = ( + cast( # cast-ok: the hook returns a bare dict of request params + "tuple[str, list[AllMessageValues], Mapping[str, object]]", + await litellm_logging_obj.async_get_chat_completion_prompt( + model=model, + messages=messages, + non_default_params=kwargs, + prompt_id=prompt_id, + prompt_variables=kwargs.get("prompt_variables", None), + prompt_management_logger=prompt_management_logger, + tools=tools, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + request_kwargs=kwargs, + ), + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and prompt_management_logger is not None and not each_fallback_writes_its_own_cache_dialect - ): - ( - model, - messages, - _, - ) = await litellm_logging_obj.async_get_chat_completion_prompt( - model=model, - messages=messages, - non_default_params=kwargs, - prompt_id=prompt_id, - prompt_variables=kwargs.get("prompt_variables", None), - prompt_management_logger=prompt_management_logger, - tools=tools, - prompt_label=kwargs.get("prompt_label", None), - prompt_version=kwargs.get("prompt_version", None), - request_kwargs=kwargs, - ) + else None + ) + if prompt_hook_result is not None: + model, messages, _ = prompt_hook_result ######################################################### # if the chat completion logging hook removed all tools, # set tools to None @@ -896,8 +899,14 @@ async def acompletion( "shared_session": shared_session, "enable_json_schema_validation": enable_json_schema_validation, } + request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs + prompt_params: Final[Mapping[str, object]] = prompt_hook_result[2] if prompt_hook_result is not None else {} + prompt_overrides: Final = { + key: value + for key, value in prompt_params.items() + if key not in request_kwargs or request_kwargs[key] is not value + } if custom_llm_provider is None: - request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs _, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -936,7 +945,9 @@ async def acompletion( try: # Use a partial function to pass your keyword arguments kwargs.pop("acompletion", None) - func: Final = partial(completion, **completion_kwargs, **kwargs) + func: Final = partial( + completion, **{**completion_kwargs, **prompt_overrides}, **without(request_kwargs, *prompt_overrides) + ) # Add the context to the function ctx: Final = contextvars.copy_context() @@ -977,7 +988,7 @@ async def acompletion( messages=messages, optional_params={ k: v - for k, v in completion_kwargs.items() + for k, v in {**completion_kwargs, **prompt_overrides}.items() if v is not None and k not in ( @@ -1010,7 +1021,7 @@ async def acompletion( model=model, custom_llm_provider=custom_llm_provider, original_exception=e, - completion_kwargs=completion_kwargs, + completion_kwargs={**completion_kwargs, **prompt_overrides}, extra_kwargs=kwargs, ) @@ -9083,7 +9094,7 @@ def speech( custom_llm_provider=custom_llm_provider, ) response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None - if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None: + if custom_llm_provider in ("edenai", "dashscope", "replicate") and text_to_speech_provider_config is not None: litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base response = base_llm_http_handler.text_to_speech_handler( model=model, diff --git a/litellm/mcp_client/client.py b/litellm/mcp_client/client.py index 0461f1f06ff..475d334330f 100644 --- a/litellm/mcp_client/client.py +++ b/litellm/mcp_client/client.py @@ -101,7 +101,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_TIMEOUT, ) from litellm.llms.custom_httpx.http_handler import get_ssl_configuration -from litellm.mcp_client.tools import list_tools_with_pagination +from litellm.mcp_client.tools import list_tools_result_with_pagination, list_tools_with_pagination from litellm.proxy.mcp_server.mcp_context import get_active_mcp_request_ctx from litellm.proxy.mcp_server.mcp_debug import capture_upstream_error_response from litellm.proxy.mcp_server.result_conversion import ( @@ -1071,6 +1071,9 @@ class MCPClient: return await self.run_with_session(lambda session: session.list_tools(params=params), quiet_on_error=True) async def list_tools(self, raise_on_error: bool = False) -> list[MCPTool]: + return (await self.list_tools_result(raise_on_error=raise_on_error)).tools + + async def list_tools_result(self, raise_on_error: bool = False) -> ListToolsResult: """List available tools from the server. Args: @@ -1085,16 +1088,20 @@ class MCPClient: try: # A per-server timeout above the global default extends the whole-walk deadline listing_deadline: Final = max(self.timeout, MCP_TOOL_LISTING_TIMEOUT) - tools: Final = await self.run_with_session( - partial(list_tools_with_pagination, listing_deadline=listing_deadline, require_complete=raise_on_error), + listing: Final = await self.run_with_session( + partial( + list_tools_result_with_pagination, + listing_deadline=listing_deadline, + require_complete=raise_on_error, + ), quiet_on_error=raise_on_error, ) - tool_count: Final = len(tools) - tool_names: Final = tuple(tool.name for tool in tools) + tool_count: Final = len(listing.tools) + tool_names: Final = tuple(tool.name for tool in listing.tools) verbose_logger.info( "MCP client listed %s tools from %s: %s", tool_count, self.server_url or "stdio", tool_names ) - return tools + return listing except asyncio.CancelledError: verbose_logger.warning("MCP client list_tools was cancelled") raise @@ -1124,7 +1131,7 @@ class MCPClient: if raise_on_error: raise # Return empty list instead of raising to allow graceful degradation - return [] + return ListToolsResult(tools=[]) @staticmethod def error_tool_result(exc: Exception) -> MCPCallToolResult: diff --git a/litellm/mcp_client/tools.py b/litellm/mcp_client/tools.py index 4f204b3629e..94b40a33469 100644 --- a/litellm/mcp_client/tools.py +++ b/litellm/mcp_client/tools.py @@ -1,9 +1,11 @@ import json +import time +from collections.abc import Sequence from typing import Final, Literal import anyio from mcp import ClientSession, MCPError -from mcp.types import INTERNAL_ERROR, PaginatedRequestParams +from mcp.types import INTERNAL_ERROR, ListToolsResult, PaginatedRequestParams from mcp.types import CallToolRequestParams as MCPCallToolRequestParams from mcp.types import CallToolResult as MCPCallToolResult from mcp.types import Tool as MCPTool @@ -18,6 +20,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT, ) +from litellm.proxy.mcp_server.result_conversion import age_freshness, aggregate_freshness from litellm.types.llms.anthropic import AnthropicMessagesTool from litellm.types.utils import ChatCompletionMessageToolCall @@ -101,9 +104,21 @@ def transform_mcp_tool_to_anthropic_tool(mcp_tool: MCPTool) -> AnthropicMessages ) +def _complete_listing(tools: Sequence[MCPTool], pages: Sequence[tuple[ListToolsResult, float]]) -> ListToolsResult: + now: Final = time.monotonic() + freshness: Final = aggregate_freshness(tuple(age_freshness(page, now - received) for page, received in pages)) + return ListToolsResult(tools=list(tools), ttl_ms=freshness.ttl_ms, cache_scope=freshness.cache_scope) + + async def list_tools_with_pagination( session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False ) -> list[MCPTool]: # mutable-ok: list return contract + return (await list_tools_result_with_pagination(session, listing_deadline, require_complete=require_complete)).tools + + +async def list_tools_result_with_pagination( + session: ClientSession, listing_deadline: float | None = None, *, require_complete: bool = False +) -> ListToolsResult: """Collect tools from every tools/list page by following nextCursor. Stops and returns the tools collected so far when the upstream repeats a @@ -111,8 +126,10 @@ async def list_tools_with_pagination( buggy or slow upstream yields a partial catalog instead of an error. listing_deadline overrides the default whole-walk deadline; callers with a per-server timeout above the global default pass it through here. + Only a complete walk carries the upstream ttlMs, so a partial catalog is never cached. """ tools: Final[list[MCPTool]] = [] # mutable-ok: accumulates each page's tools + pages: Final[list[tuple[ListToolsResult, float]]] = [] # mutable-ok: bounded pagination evidence seen_cursors: Final[set[str]] = set() # mutable-ok: guards against cursor loops cursor: str | None = None # rebind-ok: advances to each page's nextCursor # The per-request session read timeout restarts on every page, so a multi-page @@ -131,10 +148,11 @@ async def list_tools_with_pagination( else await session.list_tools(params=PaginatedRequestParams(cursor=cursor)) ) tools.extend(result.tools) + pages.append((result, time.monotonic())) next_cursor = result.next_cursor if not isinstance(next_cursor, str) or not next_cursor: - return tools + return _complete_listing(tools, pages) if next_cursor in seen_cursors: verbose_logger.warning( "MCP server repeated a tools/list cursor while listing tools; returning %s tools collected so far", @@ -144,7 +162,7 @@ async def list_tools_with_pagination( raise MCPError( code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: repeated cursor" ) - return tools + return ListToolsResult(tools=tools) seen_cursors.add(next_cursor) cursor = next_cursor @@ -155,7 +173,7 @@ async def list_tools_with_pagination( ) if require_complete: raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: pagination limit") - return tools + return ListToolsResult(tools=tools) verbose_logger.warning( "MCP server tools/list pagination exceeded the %s second listing deadline; returning %s tools collected so far", @@ -164,7 +182,7 @@ async def list_tools_with_pagination( ) if require_complete: raise MCPError(code=INTERNAL_ERROR, message="Upstream tool discovery is incomplete: listing deadline") - return tools + return ListToolsResult(tools=tools) async def load_mcp_tools( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f76ec44b0fe..9c0c65406b1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -168,6 +168,11 @@ "max_output_tokens": 4096 }, "sample_spec": { + "output_cost_per_video_clip": { + "5s": { + "720p": 0.0 + } + }, "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, "computer_use_output_cost_per_1k_tokens": 0.0, @@ -1211,7 +1216,9 @@ "supports_tool_choice": true, "supports_vision": true, "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07 + "cache_read_input_token_cost": 3e-07, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "anthropic.claude-3-5-sonnet-20241022-v2:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -1229,7 +1236,9 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "anthropic.claude-3-7-sonnet-20240620-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -1267,7 +1276,9 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, @@ -3885,8 +3896,8 @@ "supports_vision": true }, "azure_ai/claude-haiku-4-5": { - "deprecation_date": "2026-11-15", - "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "deprecation_date": "2027-03-16", + "deprecation_source": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", "successor_model": "azure_ai/claude-haiku-5-5", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, @@ -3910,8 +3921,8 @@ "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, "azure_ai/claude-opus-4-5": { - "deprecation_date": "2026-11-24", - "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "deprecation_date": "2027-03-16", + "deprecation_source": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", "successor_model": "azure_ai/claude-opus-5-5", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -9142,11 +9153,7 @@ "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "input_cost_per_token_above_272k_tokens_flex": 1e-05, - "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, - "input_cost_per_token_flex": 5e-06, - "output_cost_per_token_flex": 2.5e-05, - "cache_read_input_token_cost_flex": 5e-07, - "cache_creation_input_token_cost_flex": 6.25e-06 + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 }, "azure/gpt-astra-latest": { "alias_of": "azure/gpt-6-astra" @@ -9245,15 +9252,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true, - "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, - "cache_creation_input_token_cost_flex": 6.25e-08, - "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, - "cache_read_input_token_cost_flex": 5e-09, - "input_cost_per_token_above_272k_tokens_flex": 1e-07, - "input_cost_per_token_flex": 5e-08, - "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, - "output_cost_per_token_flex": 2.5e-07 + "supports_xhigh_reasoning_effort": true }, "azure/gpt-luna-latest": { "alias_of": "azure/gpt-6-luna" @@ -12969,8 +12968,8 @@ "supports_vision": true }, "azure/whisper-1": { - "deprecation_date": "2027-02-26", - "deprecation_source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", + "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", @@ -15197,7 +15196,9 @@ "supports_tool_choice": true, "supports_vision": true, "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 + "cache_creation_input_token_cost": 3.75e-06, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -22684,6 +22685,9 @@ "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, "cache_creation_input_token_cost_priority": 1.35002e-06, + "deprecation_date": "2026-11-19", + "deprecation_source": "https://docs.databricks.com/aws/en/machine-learning/retired-models-policy", + "successor_model": "databricks/databricks-gemini-3-8-flash", "supports_audio_input": true, "supports_video_input": true }, @@ -25930,7 +25934,9 @@ "supports_tool_choice": true, "supports_vision": true, "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 + "cache_creation_input_token_cost": 3.75e-06, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "eu.anthropic.claude-3-5-sonnet-20241022-v2:0": { "input_cost_per_token": 3e-06, @@ -25948,7 +25954,9 @@ "supports_tool_choice": true, "supports_vision": true, "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 + "cache_creation_input_token_cost": 3.75e-06, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "eu.anthropic.claude-3-7-sonnet-20250219-v1:0": { "input_cost_per_token": 3e-06, @@ -25967,7 +25975,9 @@ "supports_tool_choice": true, "supports_vision": true, "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 + "cache_creation_input_token_cost": 3.75e-06, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, @@ -26230,6 +26240,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/text-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/text-to-video" + }, "fal_ai/bytedance/seedance-2.5/image-to-video": { "litellm_provider": "fal_ai", "mode": "video_generation", @@ -26248,6 +26261,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/image-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/image-to-video" + }, "fal_ai/bytedance/seedance-2.5/reference-to-video": { "litellm_provider": "fal_ai", "mode": "video_generation", @@ -26266,6 +26282,9 @@ "video" ] }, + "fal_ai/bytedance/seedance-latest/reference-to-video": { + "alias_of": "fal_ai/bytedance/seedance-2.5/reference-to-video" + }, "fal_ai/minimax/h3/text-to-video": { "litellm_provider": "fal_ai", "metadata": {"comment":"per second of generated video by resolution; output_cost_per_second is the default resolution 2k rate"}, @@ -26475,6 +26494,46 @@ "/v1/images/generations" ] }, + "fal_ai/google/nano-banana-2.1": { + "litellm_provider": "fal_ai", + "metadata": { + "comment": "fal bills this endpoint by tokens and reports the charge in the x-fal-billable-units response header, priced at output_cost_per_billable_unit dollars per unit. The per-image prices are fal's representative costs by the request's resolution field (1K default, 2K, 4K) for a short prompt, used when the header is absent" + }, + "mode": "image_generation", + "output_cost_per_billable_unit": 1.0, + "output_cost_per_image": 0.04, + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.059, + "output_cost_per_image_4K": 0.134, + "source": "https://fal.ai/models/google/nano-banana-2.1", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/google/nano-banana-2.1/edit": { + "litellm_provider": "fal_ai", + "metadata": { + "comment": "fal bills this endpoint by tokens and reports the charge in the x-fal-billable-units response header, priced at output_cost_per_billable_unit dollars per unit. The per-image prices are fal's representative costs by the request's resolution field (1K default, 2K, 4K) for a short prompt and two 1K input images, used when the header is absent" + }, + "mode": "image_generation", + "output_cost_per_billable_unit": 1.0, + "output_cost_per_image": 0.044, + "output_cost_per_image_1K": 0.044, + "output_cost_per_image_2K": 0.063, + "output_cost_per_image_4K": 0.138, + "source": "https://fal.ai/models/google/nano-banana-2.1/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/google/nano-banana-latest": { + "alias_of": "fal_ai/google/nano-banana-2.1" + }, + "fal_ai/google/nano-banana-latest/edit": { + "alias_of": "fal_ai/google/nano-banana-2.1/edit" + }, "fal_ai/openai/gpt-image-2": { "litellm_provider": "fal_ai", "metadata": { @@ -28601,7 +28660,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", - "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -29075,7 +29134,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", - "successor_model": "fireworks_ai/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/deepseek-v4p1-flash", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -31148,7 +31207,7 @@ "text" ], "supports_audio_input": true, - "supports_function_calling": false, + "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -31160,7 +31219,7 @@ "supports_url_context": false, "supports_video_input": true, "supports_vision": true, - "supports_web_search": false, + "supports_web_search": true, "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { @@ -31936,6 +31995,8 @@ "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, + "output_cost_per_image_token_priority": 5.4e-05, "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -32372,7 +32433,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-2.5-flash-preview-tts": { - "deprecation_date": "2026-11-17", + "deprecation_date": "2026-12-18", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 5e-07, @@ -33485,7 +33546,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, - "supports_function_calling": false, + "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -33497,12 +33558,12 @@ "supports_url_context": false, "supports_video_input": true, "supports_vision": true, - "supports_web_search": false, + "supports_web_search": true, "supports_native_streaming": true }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-11-17", + "deprecation_date": "2026-12-18", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_audio_token": 7e-07, @@ -36264,6 +36325,7 @@ "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, + "successor_model": "gpt-audio-1.5", "supports_audio_input": true, "supports_audio_output": true, "supports_function_calling": true, @@ -36284,6 +36346,7 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "successor_model": "gpt-realtime-2.1-mini", "supports_audio_input": true, "supports_audio_output": true, "supports_function_calling": true, @@ -36488,7 +36551,7 @@ "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -36512,7 +36575,7 @@ "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -36650,7 +36713,7 @@ "low/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -36664,7 +36727,7 @@ "low/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36678,7 +36741,7 @@ "low/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36692,7 +36755,7 @@ "medium/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -36706,7 +36769,7 @@ "medium/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -36720,7 +36783,7 @@ "medium/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -36734,7 +36797,7 @@ "high/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -36748,7 +36811,7 @@ "high/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -36762,7 +36825,7 @@ "high/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -36776,7 +36839,7 @@ "standard/1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -36790,7 +36853,7 @@ "standard/1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36804,7 +36867,7 @@ "standard/1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36818,7 +36881,7 @@ "1024-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -36832,7 +36895,7 @@ "1024-x-1536/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36846,7 +36909,7 @@ "1536-x-1024/gpt-image-1.5": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -36869,7 +36932,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "low/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36883,7 +36946,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "low/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -36897,7 +36960,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "medium/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.034, @@ -36911,7 +36974,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "medium/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.05, @@ -36925,7 +36988,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "medium/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.05, @@ -36939,7 +37002,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "high/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.133, @@ -36953,7 +37016,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "high/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.2, @@ -36967,7 +37030,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "high/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.2, @@ -36981,7 +37044,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "standard/1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.009, @@ -36995,7 +37058,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "standard/1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -37009,7 +37072,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "standard/1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -37023,7 +37086,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "1024-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.009, @@ -37037,7 +37100,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "1024-x-1536/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -37051,7 +37114,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "1536-x-1024/gpt-image-1.5-2025-12-16": { "input_cost_per_image": 0.013, @@ -37065,7 +37128,7 @@ "supports_pdf_input": true, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst" + "successor_model": "gpt-image-2.6-sunburst" }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, @@ -37074,7 +37137,7 @@ "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, @@ -38767,7 +38830,6 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -38776,7 +38838,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_above_272k_tokens_batches": 0.000135, "reasoning_contexts": [ "auto", "current_turn", @@ -39392,7 +39453,7 @@ "gpt-5-pro": { "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -39444,7 +39505,7 @@ "gpt-5-pro-2025-10-06": { "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "openai", @@ -39500,7 +39561,7 @@ "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, @@ -39721,7 +39782,7 @@ "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-terra", + "successor_model": "gpt-6.1-terra", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -39782,7 +39843,7 @@ "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-terra", + "successor_model": "gpt-6.1-terra", "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -39842,7 +39903,7 @@ "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-luna", + "successor_model": "gpt-6-luna", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, @@ -39900,7 +39961,7 @@ "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-luna", + "successor_model": "gpt-6-luna", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, @@ -39979,7 +40040,7 @@ "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image_token": 2.5e-06, "input_cost_per_image_token_batches": 1.25e-06, "input_cost_per_token": 2e-06, @@ -42368,7 +42429,7 @@ "low/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -42379,7 +42440,7 @@ "low/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -42390,7 +42451,7 @@ "low/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -42401,7 +42462,7 @@ "medium/1024-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -42412,7 +42473,7 @@ "medium/1024-x-1536/gpt-image-1-mini": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -42423,7 +42484,7 @@ "medium/1536-x-1024/gpt-image-1-mini": { "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -43788,6 +43849,7 @@ "cache_read_input_token_cost": 1e-08 }, "mistral/mistral-embed-2312": { + "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, @@ -44242,6 +44304,7 @@ "supports_vision": true }, "mistral/mistral-embed": { + "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, @@ -44270,7 +44333,8 @@ "mode": "chat", "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ - "none", + "low", + "medium", "high" ], "source": "https://docs.mistral.ai/models/mistral-large-4-0", @@ -44291,7 +44355,8 @@ "mode": "chat", "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ - "none", + "low", + "medium", "high" ], "source": "https://docs.mistral.ai/models/mistral-large-4-0", @@ -46075,7 +46140,8 @@ "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2026-12-11", - "successor_model": "gpt-5.6-sol", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -46199,7 +46265,7 @@ "cache_read_input_token_cost_priority": 8.75e-07, "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, @@ -46298,7 +46364,7 @@ "o3-pro": { "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -46345,7 +46411,7 @@ "o3-pro-2025-06-10": { "deprecation_date": "2026-12-11", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-5.6-sol", + "successor_model": "gpt-6.1-sol", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "openai", @@ -50694,7 +50760,9 @@ "supports_tool_choice": true, "supports_vision": true, "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 + "cache_creation_input_token_cost": 3.75e-06, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "us.anthropic.claude-3-5-sonnet-20241022-v2:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -50712,7 +50780,9 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "us.anthropic.claude-3-7-sonnet-20250219-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -50731,7 +50801,9 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-13", + "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html" }, "us.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, @@ -51403,7 +51475,7 @@ "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, - "max_output_tokens": 163840, + "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -51423,7 +51495,7 @@ "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, - "max_output_tokens": 163840, + "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -54449,6 +54521,20 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, + "vertex_ai/deep-research-preview-04-2026": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "vertex_ai", + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "source": "https://cloud.google.com/model-garden/pricing", + "web_search_billing_unit": "per_query" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, @@ -56741,9 +56827,11 @@ "supported_output_modalities": [ "video" ], - "metadata": { - "comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled" - } + "metadata": {"comment":"Output: 20/30/68/110 credits per second at 480p/720p/1080p/4K @ $0.01 per credit; input video billed additionally at 10/15/34/57 credits per input second and the 80 credit minimum per task are not modeled"}, + "output_cost_per_second_4k": 1.1 + }, + "runwayml/seedance-latest": { + "alias_of": "runwayml/seedance2_5" }, "runwayml/hailuo3": { "litellm_provider": "runwayml", @@ -60460,7 +60548,7 @@ "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", - "successor_model": "gpt-image-2.5-sunburst", + "successor_model": "gpt-image-2.6-sunburst", "input_cost_per_image_token": 8e-06, "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, @@ -60528,7 +60616,7 @@ "gemini_native_audio": true }, "gemini-2.5-flash-native-audio-preview-12-2025": { - "deprecation_date": "2026-11-17", + "deprecation_date": "2027-03-31", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, @@ -60561,8 +60649,6 @@ "supports_web_search": true }, "gemini-3.1-flash-live-preview": { - "deprecation_date": "2026-11-17", - "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, @@ -60720,7 +60806,7 @@ "gemini_native_audio": true }, "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { - "deprecation_date": "2026-11-17", + "deprecation_date": "2027-03-31", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, @@ -60756,8 +60842,6 @@ "supports_reasoning": true }, "gemini/gemini-3.1-flash-live-preview": { - "deprecation_date": "2026-11-17", - "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-live", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, @@ -60798,7 +60882,7 @@ "supports_reasoning": true }, "gemini/gemini-3.1-flash-tts-preview": { - "deprecation_date": "2026-11-17", + "deprecation_date": "2026-12-18", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 1e-06, @@ -60821,10 +60905,6 @@ "supports_web_search": false }, "gemini/gemini-3.8-flash-tts": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_batches": 6.25e-08, - "cache_read_input_token_cost_flex": 2.5e-08, - "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, @@ -60842,13 +60922,9 @@ "supported_endpoints": [ "/v1/audio/speech" ], - "supports_prompt_caching": true + "supports_prompt_caching": false }, "gemini/gemini-3.8-flash-lite-tts": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_batches": 6.25e-08, - "cache_read_input_token_cost_flex": 2.5e-08, - "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, @@ -60866,10 +60942,10 @@ "supported_endpoints": [ "/v1/audio/speech" ], - "supports_prompt_caching": true + "supports_prompt_caching": false }, "gemini-2.5-flash-preview-tts": { - "deprecation_date": "2026-11-17", + "deprecation_date": "2026-12-18", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "successor_model": "gemini/gemini-3.8-flash-tts", "input_cost_per_token": 5e-07, @@ -63280,9 +63356,28 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ], - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-06-30", + "deprecation_source": "https://docs.soniox.com/stt/models", + "successor_model": "soniox/stt-async-v5" }, "soniox/stt-async-v5": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_input_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 2.77778e-05, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true, + "deprecation_date": "2026-12-15", + "deprecation_source": "https://docs.soniox.com/stt/models", + "successor_model": "soniox/stt-async-v6" + }, + "soniox/stt-async-v6": { "litellm_provider": "soniox", "max_output_tokens": 8000, "max_input_tokens": 8000, @@ -63296,6 +63391,29 @@ "supports_audio_input": true }, "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "deprecation_date": "2026-12-15", + "deprecation_source": "https://docs.soniox.com/stt/models", + "successor_model": "soniox/stt-rt-v6" + }, + "soniox/stt-rt-v6": { "input_cost_per_second": 3.333e-05, "litellm_provider": "soniox", "metadata": { @@ -63353,7 +63471,10 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2025-10-31", + "deprecation_source": "https://docs.soniox.com/stt/models", + "successor_model": "soniox/stt-rt-v3" }, "soniox/stt-rt-v3": { "input_cost_per_second": 3.333e-05, @@ -63373,7 +63494,10 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-02-28", + "deprecation_source": "https://docs.soniox.com/stt/models", + "successor_model": "soniox/stt-rt-v4" }, "soniox/stt-rt-v3-preview": { "input_cost_per_second": 3.333e-05, @@ -63413,7 +63537,10 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "deprecation_date": "2026-06-30", + "deprecation_source": "https://docs.soniox.com/stt/models", + "successor_model": "soniox/stt-rt-v5" }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", @@ -67729,6 +67856,7 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, + "supports_reasoning": true, "supports_vision": false, "source": "https://deepinfra.com/pricing" }, @@ -68151,6 +68279,7 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, + "supports_reasoning": true, "supports_vision": false, "source": "https://deepinfra.com/pricing" }, @@ -68196,6 +68325,7 @@ "supports_function_calling": true, "supports_tool_choice": true, "supports_response_schema": true, + "supports_reasoning": true, "supports_vision": true, "source": "https://deepinfra.com/pricing" }, @@ -68411,6 +68541,72 @@ "supports_vision": true, "source": "https://deepinfra.com/pricing" }, + "deepinfra/zai-org/GLM-5.3-Flash": { + "max_output_tokens": 1048576, + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_video_input": true, + "source": "https://deepinfra.com/zai-org/GLM-5.3-Flash" + }, + "deepinfra/XiaomiMiMo/MiMo-V2.6-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 2.8e-07, + "cache_read_input_token_cost": 2.8e-09, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_video_input": true, + "supports_audio_input": true, + "source": "https://deepinfra.com/XiaomiMiMo/MiMo-V2.6-Flash" + }, + "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Super-VL": { + "max_input_tokens": 262144, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_video_input": true, + "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Super-VL" + }, + "deepinfra/tencent/Hy4-preview": { + "max_output_tokens": 131072, + "max_input_tokens": 1048576, + "input_cost_per_token": 8.34e-07, + "output_cost_per_token": 2.501e-06, + "cache_read_input_token_cost": 4.2e-08, + "supports_prompt_caching": true, + "litellm_provider": "deepinfra", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": false, + "source": "https://deepinfra.com/tencent/Hy4-preview" + }, "gemini/gemini-omni-1.1-flash": { "deprecation_date": "2026-12-03", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", @@ -68562,10 +68758,12 @@ "xai/grok-imagine-image-quality": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68581,10 +68779,12 @@ "xai/grok-imagine-image-quality-20260403": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68600,10 +68800,12 @@ "xai/grok-imagine-image-quality-latest": { "deprecation_date": "2026-11-02", "deprecation_source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -68619,10 +68821,12 @@ "xai/grok-imagine-image-pro": { "deprecation_date": "2026-05-15", "deprecation_source": "https://docs.x.ai/developers/migration/may-15-retirement", - "input_cost_per_image": 0.05, + "input_cost_per_image": 0.04, "litellm_provider": "xai", "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", + "output_cost_per_image_1K": 0.04, + "output_cost_per_image_2K": 0.06, + "source": "https://docs.x.ai/developers/migration/imagine-image-quality-nov-2", "successor_model": "xai/grok-imagine-image-2.0", "supported_endpoints": [ "/v1/images/generations" @@ -69326,6 +69530,7 @@ "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_mid_conversation_system": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true @@ -69383,6 +69588,7 @@ "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, "supports_mid_conversation_system": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true @@ -69767,6 +69973,18 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/deepseek-v4.1-flash-1017": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 6e-09, + "litellm_provider": "azure_ai", + "mode": "chat", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "supports_prompt_caching": true, + "supports_vision": false, + "source": "https://ai.azure.com/catalog/models/DeepSeek-V4.1-Flash-1017" + }, "azure_ai/deepseek-v4.1-flash": { "cache_read_input_token_cost": 6e-09, "input_cost_per_token": 3e-07, @@ -71780,6 +71998,22 @@ "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-hydrogen": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 7.5e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, + "perplexity/google/gemini-4-carbon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -74962,36 +75196,57 @@ } }, "nace/drex-v1.0": { - "input_cost_per_token": 4e-08, + "input_cost_per_token": 3e-08, "litellm_provider": "nace", - "max_input_tokens": 32768, + "max_input_tokens": 262144, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://console.nace.ai/docs/drex/reference/models" + "source": "https://console.nace.ai/docs/drex/reference/models", + "supports_vision": true }, "nace/drex-v1.5": { - "input_cost_per_token": 5e-08, + "input_cost_per_token": 3e-08, "litellm_provider": "nace", - "max_input_tokens": 131072, + "max_input_tokens": 262144, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://console.nace.ai/docs/drex/reference/models" + "source": "https://console.nace.ai/docs/drex/reference/models", + "supports_vision": true }, "nace/drex-v2.0": { - "input_cost_per_token": 6e-08, + "input_cost_per_token": 3e-08, "litellm_provider": "nace", - "max_input_tokens": 131072, + "max_input_tokens": 262144, "mode": "evaluation", "output_cost_per_token": 0.0, "source": "https://console.nace.ai/docs/drex/reference/models" }, "nace/drex-latest": { - "input_cost_per_token": 6e-08, + "input_cost_per_token": 3e-08, "litellm_provider": "nace", - "max_input_tokens": 131072, + "max_input_tokens": 262144, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://console.nace.ai/docs/drex/reference/models" + "source": "https://console.nace.ai/docs/drex/reference/models", + "supports_vision": true + }, + "nace/drex-max": { + "input_cost_per_token": 3e-08, + "litellm_provider": "nace", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/guides/drex-max", + "supports_vision": true + }, + "nace/drex-max-v1.1": { + "input_cost_per_token": 3e-08, + "litellm_provider": "nace", + "max_input_tokens": 262144, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/guides/drex-max", + "supports_vision": true }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, @@ -75985,6 +76240,7 @@ "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_mid_conversation_system": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -77065,6 +77321,14 @@ "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/anthropic/claude-fable-5-2": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 8e-06, + "output_cost_per_token": 4e-05, + "cache_read_input_token_cost": 2e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/anthropic/claude-fable-5-1": { "litellm_provider": "perplexity", "mode": "responses", @@ -77862,7 +78126,8 @@ "output_cost_per_token": 2.75e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-amazon-nova-2-5-sonic.md", "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_function_calling": true }, "azure/model-router": { "comment": "The flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it", @@ -77878,8 +78143,8 @@ "supports_vision": true }, "azure/whisper": { - "deprecation_date": "2027-02-26", - "deprecation_source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", + "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", @@ -77926,6 +78191,9 @@ "tpm": 4000000.0, "web_search_billing_unit": "per_query" }, + "gemini/nano-banana-latest": { + "alias_of": "gemini/gemini-nano-banana-2.1" + }, "gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, @@ -78020,6 +78288,9 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, + "vertex_ai/nano-banana-latest": { + "alias_of": "vertex_ai/gemini-nano-banana-2.1" + }, "azure_ai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -79156,7 +79427,6 @@ "vertex_ai/gemini-4-argon": { "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, - "cache_read_input_token_cost_flex": 5e-08, "cache_read_input_token_cost_priority": 1.8e-07, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 2e-06, @@ -79164,7 +79434,6 @@ "input_cost_per_audio_token_priority": 3.6e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, - "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 3.6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, @@ -79173,7 +79442,6 @@ "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, - "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05, "prompt_cache_min_tokens": 6144.0, "regional_endpoint_uplift_multiplier": 1.1, @@ -79216,7 +79484,6 @@ "vertex_ai/gemini-4-argon-cyber-permissive": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, - "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 4e-06, @@ -79224,7 +79491,6 @@ "input_cost_per_audio_token_priority": 7.2e-06, "input_cost_per_token": 4e-06, "input_cost_per_token_batches": 2e-06, - "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 7.2e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, @@ -79233,7 +79499,6 @@ "output_cost_per_reasoning_token": 2e-05, "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, - "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 3.6e-05, "prompt_cache_min_tokens": 6144.0, "regional_endpoint_uplift_multiplier": 1.1, @@ -79800,6 +80065,7 @@ "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_mid_conversation_system": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -83601,7 +83867,7 @@ "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://api.fireworks.ai/v1/serverless/models", - "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "successor_model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -83788,6 +84054,7 @@ "output_cost_per_token_priority": 6.25e-07, "source": "https://docs.fireworks.ai/serverless/pricing.md", "supports_function_calling": false, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true @@ -86827,6 +87094,15 @@ "output_cost_per_token": 1e-05, "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" }, + "gmi/google/gemini-4-carbon": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "gmi", + "max_input_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, "gmi/google/gemma-4-26b-a4b-it": { "input_cost_per_token": 1.3e-07, "litellm_provider": "gmi", @@ -87160,6 +87436,18 @@ "output_cost_per_token": 6e-06, "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" }, + "gmi/xai/grok4.7": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "gmi", + "max_input_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://ie-api.gmicloud.ai/v1/artifact/get_public_artifacts" + }, "gmi/zai-org/GLM-4.5-Air-FP8": { "input_cost_per_token": 2e-07, "litellm_provider": "gmi", @@ -87256,6 +87544,7 @@ "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_mid_conversation_system": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -87279,12 +87568,17 @@ }, "gpt-image-2.6-sunburst": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -87295,12 +87589,17 @@ }, "gpt-image-2.6-flare": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_image_token": 8e-06, + "input_cost_per_image_token_batches": 4e-06, "input_cost_per_token": 5e-06, + "input_cost_per_token_batches": 2.5e-06, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image_token": 3e-05, + "output_cost_per_image_token_batches": 1.5e-05, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/images/generations", @@ -88562,8 +88861,10 @@ "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://api.together.ai/v1/models", + "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_vision": true }, "gemini/gemini-4-carbon": { @@ -95051,6 +95352,438 @@ "video" ] }, + "fal_ai/fal-ai/auto-caption": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.1, + "source": "https://fal.ai/models/fal-ai/auto-caption", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/hunyuan-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.4, + "source": "https://fal.ai/models/fal-ai/hunyuan-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/hunyuan-video-image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.4, + "source": "https://fal.ai/models/fal-ai/hunyuan-video-image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/infinity-star/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.07, + "source": "https://fal.ai/models/fal-ai/infinity-star/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.02, + "source": "https://fal.ai/models/fal-ai/ltx-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-video-13b-distilled": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.04, + "source": "https://fal.ai/models/fal-ai/ltx-video-13b-distilled", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-video-13b-distilled/extend": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.04, + "source": "https://fal.ai/models/fal-ai/ltx-video-13b-distilled/extend", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-video-13b-distilled/multiconditioning": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.04, + "source": "https://fal.ai/models/fal-ai/ltx-video-13b-distilled/multiconditioning", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ltx-video/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.02, + "source": "https://fal.ai/models/fal-ai/ltx-video/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/hailuo-2.3-fast/pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.33, + "source": "https://fal.ai/models/fal-ai/minimax/hailuo-2.3-fast/pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01-director": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01-director", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01-director/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01-director/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01-live": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01-live", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01-live/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01-live/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01-subject-reference": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01-subject-reference", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/minimax/video-01/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.5, + "source": "https://fal.ai/models/fal-ai/minimax/video-01/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ovi": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.2, + "source": "https://fal.ai/models/fal-ai/ovi", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/ovi/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.2, + "source": "https://fal.ai/models/fal-ai/ovi/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-i2v-lora": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.75, + "source": "https://fal.ai/models/fal-ai/wan-i2v-lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-pro/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.8, + "source": "https://fal.ai/models/fal-ai/wan-pro/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan-t2v-lora": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.75, + "source": "https://fal.ai/models/fal-ai/wan-t2v-lora", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-5b/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.15, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-5b/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/wan/v2.2-5b/text-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.15, + "source": "https://fal.ai/models/fal-ai/wan/v2.2-5b/text-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/moonvalley/marey/motion-transfer": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 2, + "source": "https://fal.ai/models/moonvalley/marey/motion-transfer", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/moonvalley/marey/pose-transfer": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 2, + "source": "https://fal.ai/models/moonvalley/marey/pose-transfer", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/mureka/api/generate/lyrics-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.1, + "source": "https://fal.ai/models/mureka/api/generate/lyrics-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, "fal_ai/fal-ai/wan-22-vace-fun-a14b/inpainting": { "litellm_provider": "fal_ai", "metadata": {"comment":"per second of generated video by resolution; the page's resolution default is auto or unstated, so output_cost_per_second is the highest tier (720p); tiers without a catalog field: 580p $0.075"}, @@ -97753,6 +98486,9 @@ "output_cost_per_image_4K": 0.1134, "source": "https://replicate.com/google/nano-banana-2.1" }, + "replicate/google/nano-banana-latest": { + "alias_of": "replicate/google/nano-banana-2.1" + }, "replicate/heygen/avatar-iv": { "litellm_provider": "replicate", "mode": "video_generation", @@ -100825,5 +101561,301 @@ "supports_vision": true, "supports_web_search": false, "tpm": 1000000 + }, + "runwayml/kling_4": { + "litellm_provider": "runwayml", + "metadata": {"comment":"Promo rate through 2026-11-30 UTC: 12.56 credits per second (720p) and 18.8 (1080p) without input video @ $0.01 per credit = $0.1256 and $0.188 per second; list rate 15.7/23.5 credits per second applies from 2026-12-01 UTC (ruling runwayml-kling-4-promo-rate). The with-input-video rate (7.52/11.28 credits per output second plus the same per second of the greater of input and output duration) and rounding up to the next credit are not modeled"}, + "mode": "video_generation", + "output_cost_per_second": 0.1256, + "output_cost_per_second_1080p": 0.188, + "source": "https://docs.dev.runwayml.com/guides/pricing/", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/bytedance/seedream/v5.5/text-to-image": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per image at the default image_size auto_2K; 1K and 1.5K outputs are $0.03 and custom dimensions are $0.03 up to and including 2,610,000 pixels and $0.0405 above, so 1.5K has no catalog field"}, + "mode": "image_generation", + "output_cost_per_image": 0.0405, + "output_cost_per_image_1K": 0.03, + "output_cost_per_image_2K": 0.0405, + "source": "https://fal.ai/models/bytedance/seedream/v5.5/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/bytedance/seedream/v5.5/edit": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per output image with one input image at the default image_size auto_2K ($0.0405 per output plus $0.012 per input image per output); 1K and 1.5K outputs are $0.03 plus the input fee and each additional input image adds $0.012 per output, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.0525, + "output_cost_per_image_1K": 0.042, + "output_cost_per_image_2K": 0.0525, + "source": "https://fal.ai/models/bytedance/seedream/v5.5/edit", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/bytedance/seedream/v5.5/layerize": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per returned output image (the base image and every separated layer) at the default image_size auto, where a base image above 2,610,000 pixels bills $0.0405 per output and one at or under that area bills $0.03; fal also bills $0.012 for the input image once per request and returns up to max_images (default 30) outputs, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.0405, + "output_cost_per_image_1K": 0.03, + "output_cost_per_image_2K": 0.0405, + "source": "https://fal.ai/models/bytedance/seedream/v5.5/layerize", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/flux-pro/v1/vto": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per try-on with one 1 MP input image and one 1 MP output ($0.0375 for the first input megapixel plus $0.005 per output megapixel); fal bills $0.005 per additional input megapixel and rounds each input image and the output up to whole megapixels separately, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.0425, + "source": "https://fal.ai/models/fal-ai/flux-pro/v1/vto", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/patina/material": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per request at the schema defaults (1024x1024 square_hd, all 5 maps, no upscaling), the page's own example; fal bills $0.01 per request plus $0.02 per megapixel plus $0.01 per megapixel per map type, and 2x or 4x upscaling adds $0.004 or $0.016 per pre-upscaling megapixel per map, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.08, + "source": "https://fal.ai/models/fal-ai/patina/material", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/patina/material/extract": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per request at the schema defaults (1024x1024 input, all 5 maps, no upscaling), the page's own example; fal bills $0.10 per request plus $0.02 per megapixel plus $0.01 per megapixel per map type, and 2x or 4x upscaling adds $0.004 or $0.016 per pre-upscaling megapixel per map, which is not modeled"}, + "mode": "image_generation", + "output_cost_per_image": 0.17, + "source": "https://fal.ai/models/fal-ai/patina/material/extract", + "supported_endpoints": [ + "/v1/images/edits", + "/v1/images/generations" + ], + "supports_vision": true + }, + "fal_ai/fal-ai/vidu/template-to-video": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second derived from the $0.20 Standard template price over the fixed 4s clip the sibling vidu endpoints state; Premium templates are $0.30 and Advanced templates $0.50 per video, which has no catalog field"}, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "source": "https://fal.ai/models/fal-ai/vidu/template-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "gemini/gemini-4-hydrogen": { + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_priority": 1.35e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 2.7e-06, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_flex": 3.75e-06, + "output_cost_per_token_priority": 1.35e-05, + "prompt_cache_min_tokens": 4096.0, + "rpm": 2000.0, + "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, + "source": "https://aistudio.google.com/docs/pricing#gemini-4-hydrogen", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_computer_use": true, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000.0, + "web_search_billing_unit": "per_query" + }, + "vertex_ai/gemini-4-hydrogen": { + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, + "cache_read_input_token_cost_priority": 1.35e-07, + "google_maps_grounding_cost_per_query": 0.014, + "input_cost_per_audio_token": 1.5e-06, + "input_cost_per_audio_token_batches": 7.5e-07, + "input_cost_per_audio_token_priority": 2.7e-06, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_batches": 7.5e-07, + "input_cost_per_token_priority": 2.7e-06, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_reasoning_token": 7.5e-06, + "output_cost_per_token": 7.5e-06, + "output_cost_per_token_batches": 3.75e-06, + "output_cost_per_token_priority": 1.35e-05, + "prompt_cache_min_tokens": 4096.0, + "regional_endpoint_uplift_multiplier": 1.1, + "search_context_cost_per_query": {"search_context_size_high":0.014,"search_context_size_low":0.014,"search_context_size_medium":0.014}, + "search_free_requests": {"count":5000,"period":"month","shared_group":"gemini-3"}, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_audio_output": false, + "supports_computer_use": true, + "supports_continuation": true, + "supports_function_calling": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "together_ai/thinkingmachines/Inkling-2-Small": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1e-06, + "source": "https://api.together.ai/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fal_ai/google/gemini-omni-flash/v1.2/upscale": { + "litellm_provider": "fal_ai", + "metadata": {"comment":"per second of upscaled output video by target resolution; the 1080p tier and output_cost_per_second take the higher source-dependent rate ($0.0819 from a 360p source) because the catalog selects by target resolution alone and the source resolution has no field, so a 720p source upscaled to 1080p ($0.041 per second) is overcounted, the same convention as fal_ai/blackforestlabs/flux-video-upscale"}, + "mode": "video_generation", + "output_cost_per_second": 0.0819, + "output_cost_per_second_1080p": 0.0819, + "output_cost_per_second_4k": 0.1638, + "output_cost_per_second_720p": 0.041, + "source": "https://fal.ai/models/google/gemini-omni-flash/v1.2/upscale", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "video" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/decart/lucy-5b/image-to-video": { + "litellm_provider": "fal_ai", + "mode": "video_generation", + "output_cost_per_video": 0.15, + "source": "https://fal.ai/models/decart/lucy-5b/image-to-video", + "supported_endpoints": [ + "/v1/videos" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "fal_ai/fal-ai/decart/lucy-5b/image-to-video": { + "alias_of": "fal_ai/decart/lucy-5b/image-to-video" + }, + "github_copilot/gemini-4-hydrogen": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 1.5e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 200000, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 7.5e-06, + "source": "https://github.blog/changelog/2026-11-13-gemini-4-hydrogen-in-github-copilot", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_vision": true } } diff --git a/litellm/models/access_group.py b/litellm/models/access_group.py index fb0ab211bbc..cec0c1ef27e 100644 --- a/litellm/models/access_group.py +++ b/litellm/models/access_group.py @@ -20,6 +20,7 @@ class LiteLLM_AccessGroupTable(LiteLLMPydanticObjectBase): mcp_tool_permissions: Mapping[str, Sequence[str]] | None = None access_agent_ids: list[str] = [] access_skill_names: tuple[str, ...] = () + access_vector_store_ids: tuple[str, ...] = () assigned_team_ids: list[str] = [] assigned_key_ids: list[str] = [] created_at: datetime | None = None diff --git a/litellm/models/alert.py b/litellm/models/alert.py index f96e042ff6b..4db127bf121 100644 --- a/litellm/models/alert.py +++ b/litellm/models/alert.py @@ -41,6 +41,24 @@ class DeprecationEvidence(BaseModel): ) +class AlertSummary(BaseModel): + model_config = ConfigDict(frozen=True) + + id: AlertID + type: AlertType + resource_type: AlertResourceType + resource_name: str + created_at: AwareDatetime + evidence: MetricEvidence + + +class AlertRecipient(BaseModel): + model_config = ConfigDict(frozen=True) + + user_id: str + email: str + + class AlertDetection(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") diff --git a/litellm/models/budget.py b/litellm/models/budget.py index c9922b41f79..30cdfffce58 100644 --- a/litellm/models/budget.py +++ b/litellm/models/budget.py @@ -7,12 +7,46 @@ Canonical definition for ``litellm_budgettable``. Re-exported from from collections.abc import Mapping from datetime import datetime, timezone -from typing import Final +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias -from pydantic import ConfigDict +from pydantic import AfterValidator, ConfigDict, Field, model_validator +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.base import LiteLLMPydanticObjectBase +if TYPE_CHECKING: + from litellm.proxy.common_utils.timezone_utils import BudgetResetSettings + + +class BudgetLimit(LiteLLMPydanticObjectBase): + model_config = ConfigDict(frozen=True) + + budget_duration: Literal["1mo", "12mo"] + max_budget: float = Field(ge=0, allow_inf_nan=False) + reset_at: datetime | None = None + + +def unique_budget_limits(limits: tuple[BudgetLimit, ...]) -> tuple[BudgetLimit, ...]: + if len({limit.budget_duration for limit in limits}) != len(limits): + raise ValueError("Each budget duration may appear only once") + return limits + + +BudgetLimits: TypeAlias = Annotated[tuple[BudgetLimit, ...], AfterValidator(unique_budget_limits)] + + +def initialize_budget_limits( + limits: tuple[BudgetLimit, ...], settings: "BudgetResetSettings", now: datetime +) -> tuple[BudgetLimit, ...]: + from litellm.proxy.common_utils.timezone_utils import compute_budget_reset_at + + return tuple( + limit + if limit.reset_at is not None + else limit.model_copy(update={"reset_at": compute_budget_reset_at(limit.budget_duration, settings, now)}) + for limit in limits + ) + class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase): """Represents user-controllable params for a LiteLLM_BudgetTable record. @@ -39,6 +73,14 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase): model_config = ConfigDict(protected_namespaces=()) + @model_validator(mode="before") + @classmethod + def reject_concurrent_windows(cls, value: object) -> object: + values: Final = as_str_mapping(value) + if values is not None and values.get("budget_limits"): + raise ValueError("Concurrent budget windows are supported only by customer hierarchies") + return value + def active_temp_budget_increase(self, now: datetime) -> float: if self.temp_budget_increase is None or self.temp_budget_expiry is None: return 0.0 diff --git a/litellm/models/performance_alert.py b/litellm/models/performance_alert.py index 1773aa460dc..1cae91e5379 100644 --- a/litellm/models/performance_alert.py +++ b/litellm/models/performance_alert.py @@ -85,6 +85,7 @@ class AlertMonitoring(BaseModel): model_config = ConfigDict(frozen=True) enabled: bool = Field(description="Whether performance monitoring is enabled. Defaults to true.") + model_deprecations_enabled: bool = Field(default=True, description="Whether model deprecation alerts are enabled.") alerts_used: int = Field(ge=0, le=FREE_ALERT_LIMIT, description="Lifetime free allowance consumed, capped at five.") alert_limit: int | None = Field(description="Free allowance, or null for Enterprise.") disabled_by_config: bool = Field(description="Whether performance_alerts_enabled is false in proxy settings.") @@ -100,9 +101,16 @@ class AlertMonitoringUpdate(BaseModel): enabled: bool = Field(strict=True, description="Enable or disable performance monitoring for this gateway.") +class ModelDeprecationMonitoringUpdate(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model_deprecations_enabled: bool = Field(strict=True, description="Enable or disable model deprecation alerts.") + + class AlertMonitoringStore(Protocol): async def monitoring(self) -> AlertMonitoring: ... async def set_enabled(self, enabled: bool) -> AlertMonitoring: ... + async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: ... class PerformanceAlertStore(Protocol): diff --git a/litellm/models/skill_submission.py b/litellm/models/skill_submission.py index 09d722fa69f..59587588a35 100644 --- a/litellm/models/skill_submission.py +++ b/litellm/models/skill_submission.py @@ -11,7 +11,7 @@ from litellm.models.plugin_upload import ( ) from litellm.types.proxy.skill_scanning import SkillScan -SubmissionStatus: TypeAlias = Literal["pending_review", "approved", "rejected"] +SubmissionStatus: TypeAlias = Literal["pending_review", "approved", "rejected", "withdrawn"] SubmissionOperation: TypeAlias = Literal["create", "update"] @@ -38,6 +38,7 @@ class SkillSubmission: reviewed_by: str | None reviewed_at: datetime | None rejection_reason: str | None + revision: int = 0 @dataclass(frozen=True, slots=True) @@ -99,6 +100,14 @@ class SkillSubmissionStore(Protocol): async def get_archive(self, submission_id: str, submitted_by: str | None) -> bytes | None: ... + async def edit( + self, submission_id: str, owner: str, revision: int, upload: PluginPublication + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict | SubmissionLimitExceeded: ... + + async def withdraw( + self, submission_id: str, owner: str, revision: int + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict: ... + async def review( self, submission_id: str, @@ -107,4 +116,5 @@ class SkillSubmissionStore(Protocol): reason: str | None, *, publication_base_url: str | None = None, + expected_revision: int | None = None, ) -> SubmissionReviewResult: ... diff --git a/litellm/models/team.py b/litellm/models/team.py index 8edf10703b1..6d526528589 100644 --- a/litellm/models/team.py +++ b/litellm/models/team.py @@ -137,6 +137,10 @@ class LiteLLM_TeamTableCachedObj(LiteLLM_TeamTable): last_refreshed_at: float | None = None +class LiteLLM_TeamRebuiltFromToken(LiteLLM_TeamTableCachedObj): # frozen-ok: same mutability as its cached base + """Auth's stand-in, built from the key's fields when the team row was unreadable. It has no access groups.""" + + class LiteLLM_DeletedTeamTable(LiteLLM_TeamTable): """Audit record for deleted teams; mirrors the team plus deletion metadata.""" diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261111120000_add_access_group_vector_store_ids/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261111120000_add_access_group_vector_store_ids/migration.sql new file mode 100644 index 00000000000..fe06185e1f0 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261111120000_add_access_group_vector_store_ids/migration.sql @@ -0,0 +1,4 @@ +SET LOCAL lock_timeout = '10s'; + +-- AlterTable +ALTER TABLE "LiteLLM_AccessGroupTable" ADD COLUMN IF NOT EXISTS "access_vector_store_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261112000000_customer_hierarchy/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261112000000_customer_hierarchy/migration.sql new file mode 100644 index 00000000000..90358d05893 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261112000000_customer_hierarchy/migration.sql @@ -0,0 +1,133 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_CustomerOrganizationTable" ( + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "budget_reset_at" TIMESTAMP(3), + "customer_org_id" TEXT NOT NULL, + "alias" TEXT, + "default_budget_id" TEXT, + "budget_ids" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + "budget_names" JSONB NOT NULL DEFAULT '{}', + "budget_id" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT NOT NULL, + + CONSTRAINT "LiteLLM_CustomerOrganizationTable_pkey" PRIMARY KEY ("customer_org_id") +); + +CREATE TABLE IF NOT EXISTS "LiteLLM_CustomerTeamTable" ( + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "budget_reset_at" TIMESTAMP(3), + "customer_team_id" TEXT NOT NULL, + "alias" TEXT, + "parent_org_id" TEXT NOT NULL, + "budget_id" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT NOT NULL, + + CONSTRAINT "LiteLLM_CustomerTeamTable_pkey" PRIMARY KEY ("customer_team_id") +); + +CREATE TABLE IF NOT EXISTS "LiteLLM_CustomerMemberTable" ( + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "budget_reset_at" TIMESTAMP(3), + "customer_member_id" TEXT NOT NULL, + "alias" TEXT, + "parent_team_id" TEXT NOT NULL, + "budget_id" TEXT, + "budget_template_id" TEXT, + "budget_override_expires_at" TIMESTAMP(3), + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT NOT NULL, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT NOT NULL, + + CONSTRAINT "LiteLLM_CustomerMemberTable_pkey" PRIMARY KEY ("customer_member_id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_parent_org_id_idx" ON "LiteLLM_CustomerTeamTable"("parent_org_id"); + +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_parent_team_id_idx" ON "LiteLLM_CustomerMemberTable"("parent_team_id"); + +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_template_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_template_id"); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerOrganizationTable_default_budget_id_fkey' + AND conrelid = '"LiteLLM_CustomerOrganizationTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerOrganizationTable" ADD CONSTRAINT "LiteLLM_CustomerOrganizationTable_default_budget_id_fkey" FOREIGN KEY ("default_budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerTeamTable_parent_org_id_fkey' + AND conrelid = '"LiteLLM_CustomerTeamTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerTeamTable" ADD CONSTRAINT "LiteLLM_CustomerTeamTable_parent_org_id_fkey" FOREIGN KEY ("parent_org_id") REFERENCES "LiteLLM_CustomerOrganizationTable"("customer_org_id") ON DELETE RESTRICT ON UPDATE CASCADE; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerTeamTable_budget_id_fkey' + AND conrelid = '"LiteLLM_CustomerTeamTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerTeamTable" ADD CONSTRAINT "LiteLLM_CustomerTeamTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerMemberTable_parent_team_id_fkey' + AND conrelid = '"LiteLLM_CustomerMemberTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" ADD CONSTRAINT "LiteLLM_CustomerMemberTable_parent_team_id_fkey" FOREIGN KEY ("parent_team_id") REFERENCES "LiteLLM_CustomerTeamTable"("customer_team_id") ON DELETE RESTRICT ON UPDATE CASCADE; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerMemberTable_budget_id_fkey' + AND conrelid = '"LiteLLM_CustomerMemberTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" ADD CONSTRAINT "LiteLLM_CustomerMemberTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE RESTRICT ON UPDATE CASCADE; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerOrganizationTable_budget_id_fkey' + AND conrelid = '"LiteLLM_CustomerOrganizationTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerOrganizationTable" ADD CONSTRAINT "LiteLLM_CustomerOrganizationTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerMemberTable_budget_template_id_fkey' + AND conrelid = '"LiteLLM_CustomerMemberTable"'::regclass + ) THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" ADD CONSTRAINT "LiteLLM_CustomerMemberTable_budget_template_id_fkey" FOREIGN KEY ("budget_template_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE RESTRICT ON UPDATE CASCADE; + END IF; +END $$; diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261112120000_rename_customer_member_budget_columns/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261112120000_rename_customer_member_budget_columns/migration.sql new file mode 100644 index 00000000000..9d93f5530c1 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261112120000_rename_customer_member_budget_columns/migration.sql @@ -0,0 +1,36 @@ +SET LOCAL lock_timeout = '10s'; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = to_regclass('"LiteLLM_CustomerMemberTable"') + AND attname = 'budget_template_id' AND NOT attisdropped + ) THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = to_regclass('"LiteLLM_CustomerMemberTable"') + AND attname = 'budget_override_id' AND NOT attisdropped + ) THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" RENAME COLUMN "budget_id" TO "budget_override_id"; + END IF; + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerMemberTable_budget_id_fkey' + AND conrelid = to_regclass('"LiteLLM_CustomerMemberTable"') + ) THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" RENAME CONSTRAINT "LiteLLM_CustomerMemberTable_budget_id_fkey" TO "LiteLLM_CustomerMemberTable_budget_override_id_fkey"; + END IF; + ALTER TABLE "LiteLLM_CustomerMemberTable" RENAME COLUMN "budget_template_id" TO "budget_id"; + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'LiteLLM_CustomerMemberTable_budget_template_id_fkey' + AND conrelid = to_regclass('"LiteLLM_CustomerMemberTable"') + ) THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" RENAME CONSTRAINT "LiteLLM_CustomerMemberTable_budget_template_id_fkey" TO "LiteLLM_CustomerMemberTable_budget_id_fkey"; + END IF; +END $$; + +ALTER INDEX IF EXISTS "LiteLLM_CustomerMemberTable_budget_template_id_idx" RENAME TO "LiteLLM_CustomerMemberTable_budget_id_idx"; diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql new file mode 100644 index 00000000000..f2746f41854 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql @@ -0,0 +1,47 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend" ( + "id" TEXT NOT NULL, + "date" TEXT NOT NULL, + "customer_org_id" TEXT NOT NULL, + "customer_team_id" TEXT NOT NULL DEFAULT '', + "customer_member_id" TEXT NOT NULL DEFAULT '', + "model" TEXT NOT NULL DEFAULT '', + "model_group" TEXT NOT NULL DEFAULT '', + "custom_llm_provider" TEXT NOT NULL DEFAULT '', + "mcp_namespaced_tool_name" TEXT NOT NULL DEFAULT '', + "endpoint" TEXT NOT NULL DEFAULT '', + "api_key" TEXT NOT NULL DEFAULT '', + "prompt_tokens" BIGINT NOT NULL DEFAULT 0, + "completion_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_read_input_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_creation_input_tokens" BIGINT NOT NULL DEFAULT 0, + "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0, + "api_requests" BIGINT NOT NULL DEFAULT 0, + "successful_requests" BIGINT NOT NULL DEFAULT 0, + "failed_requests" BIGINT NOT NULL DEFAULT 0, + "total_response_time_ms" BIGINT NOT NULL DEFAULT 0, + "timed_requests" BIGINT NOT NULL DEFAULT 0, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_DailyCustomerMemberSpend_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_org_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_org_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_team_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_team_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_member_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_member_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_api_key_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("api_key"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_model_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("model"); diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113120000_add_reusable_budget_limits/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113120000_add_reusable_budget_limits/migration.sql new file mode 100644 index 00000000000..2ad1f40c8ad --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113120000_add_reusable_budget_limits/migration.sql @@ -0,0 +1,3 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "budget_limits" JSONB; diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113120000_drop_customer_budget_reset_at/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113120000_drop_customer_budget_reset_at/migration.sql new file mode 100644 index 00000000000..32c25cd5bf2 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113120000_drop_customer_budget_reset_at/migration.sql @@ -0,0 +1,5 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_CustomerOrganizationTable" DROP COLUMN IF EXISTS "budget_reset_at"; +ALTER TABLE "LiteLLM_CustomerTeamTable" DROP COLUMN IF EXISTS "budget_reset_at"; +ALTER TABLE "LiteLLM_CustomerMemberTable" DROP COLUMN IF EXISTS "budget_reset_at"; diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113130000_customer_hierarchy_schema_cleanup/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113130000_customer_hierarchy_schema_cleanup/migration.sql new file mode 100644 index 00000000000..afdeeac3af4 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113130000_customer_hierarchy_schema_cleanup/migration.sql @@ -0,0 +1,51 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "name" TEXT; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'LiteLLM_CustomerOrganizationTable' AND column_name = 'budget_ids') THEN + ALTER TABLE "LiteLLM_CustomerOrganizationTable" RENAME COLUMN "budget_ids" TO "budget_template_ids"; + END IF; + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'LiteLLM_CustomerOrganizationTable' AND column_name = 'default_budget_id') THEN + ALTER TABLE "LiteLLM_CustomerOrganizationTable" RENAME COLUMN "default_budget_id" TO "default_budget_template_id"; + END IF; + IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_CustomerOrganizationTable_default_budget_id_fkey') THEN + ALTER TABLE "LiteLLM_CustomerOrganizationTable" RENAME CONSTRAINT "LiteLLM_CustomerOrganizationTable_default_budget_id_fkey" TO "LiteLLM_CustomerOrganizationTable_default_budget_template_id_fkey"; + END IF; +END $$; +ALTER TABLE "LiteLLM_CustomerOrganizationTable" DROP COLUMN IF EXISTS "budget_names"; + +ALTER TABLE "LiteLLM_CustomerTeamTable" ADD COLUMN IF NOT EXISTS "budget_override_id" TEXT; +ALTER TABLE "LiteLLM_CustomerTeamTable" ADD COLUMN IF NOT EXISTS "budget_override_expires_at" TIMESTAMP(3); +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_CustomerTeamTable_budget_override_id_fkey') THEN + ALTER TABLE "LiteLLM_CustomerTeamTable" ADD CONSTRAINT "LiteLLM_CustomerTeamTable_budget_override_id_fkey" FOREIGN KEY ("budget_override_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE RESTRICT ON UPDATE CASCADE; + END IF; +END $$; + +ALTER TABLE "LiteLLM_CustomerMemberTable" ADD COLUMN IF NOT EXISTS "parent_org_id" TEXT; +ALTER TABLE "LiteLLM_CustomerMemberTable" ADD COLUMN IF NOT EXISTS "parent_team_ids" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'LiteLLM_CustomerMemberTable' AND column_name = 'parent_team_id') THEN + -- data-migration-ok: customer tables exist only on the v1.111.0 dev line and hold a handful of rows + UPDATE "LiteLLM_CustomerMemberTable" m SET "parent_org_id" = t."parent_org_id", "parent_team_ids" = ARRAY[m."parent_team_id"] FROM "LiteLLM_CustomerTeamTable" t WHERE m."parent_team_id" = t."customer_team_id" AND m."parent_org_id" IS NULL; + END IF; + IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_CustomerMemberTable_parent_team_id_fkey') THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" DROP CONSTRAINT "LiteLLM_CustomerMemberTable_parent_team_id_fkey"; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_CustomerMemberTable_parent_org_id_fkey') THEN + ALTER TABLE "LiteLLM_CustomerMemberTable" ADD CONSTRAINT "LiteLLM_CustomerMemberTable_parent_org_id_fkey" FOREIGN KEY ("parent_org_id") REFERENCES "LiteLLM_CustomerOrganizationTable"("customer_org_id") ON DELETE RESTRICT ON UPDATE CASCADE; + END IF; +END $$; +ALTER TABLE "LiteLLM_CustomerMemberTable" ALTER COLUMN "parent_org_id" SET NOT NULL; +DROP INDEX IF EXISTS "LiteLLM_CustomerMemberTable_parent_team_id_idx"; +DROP INDEX IF EXISTS "LiteLLM_CustomerMemberTable_budget_id_idx"; +ALTER TABLE "LiteLLM_CustomerMemberTable" DROP COLUMN IF EXISTS "parent_team_id"; + +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_parent_org_id_idx" ON "LiteLLM_CustomerMemberTable"("parent_org_id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerOrganizationTable_spend_idx" ON "LiteLLM_CustomerOrganizationTable"("spend"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_spend_idx" ON "LiteLLM_CustomerTeamTable"("spend"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_spend_idx" ON "LiteLLM_CustomerMemberTable"("spend"); diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113140000_customer_budget_increase_requests/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113140000_customer_budget_increase_requests/migration.sql new file mode 100644 index 00000000000..de0a9e86645 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113140000_customer_budget_increase_requests/migration.sql @@ -0,0 +1,28 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "cap_policy" TEXT; + +CREATE TABLE IF NOT EXISTS "LiteLLM_CustomerBudgetIncreaseRequest" ( + "request_id" TEXT NOT NULL, + "customer_member_id" TEXT NOT NULL, + "requested_max_budget" DOUBLE PRECISION NOT NULL, + "expires_at" TIMESTAMP(3), + "reason" TEXT, + "status" TEXT NOT NULL DEFAULT 'pending_review', + "submitted_by" TEXT NOT NULL, + "submitted_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "reviewed_by" TEXT, + "reviewed_at" TIMESTAMP(3), + "review_notes" TEXT, + CONSTRAINT "LiteLLM_CustomerBudgetIncreaseRequest_pkey" PRIMARY KEY ("request_id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerBudgetIncreaseRequest_status_idx" ON "LiteLLM_CustomerBudgetIncreaseRequest"("status"); +CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerBudgetIncreaseRequest_customer_member_id_idx" ON "LiteLLM_CustomerBudgetIncreaseRequest"("customer_member_id"); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_CustomerBudgetIncreaseRequest_customer_member_id_fkey') THEN + ALTER TABLE "LiteLLM_CustomerBudgetIncreaseRequest" ADD CONSTRAINT "LiteLLM_CustomerBudgetIncreaseRequest_customer_member_id_fkey" FOREIGN KEY ("customer_member_id") REFERENCES "LiteLLM_CustomerMemberTable"("customer_member_id") ON DELETE CASCADE ON UPDATE CASCADE; + END IF; +END $$; diff --git a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma index 498f86cc40f..08c4b127de5 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma @@ -11,7 +11,14 @@ generator client { // Budget / Rate Limits for an org model LiteLLM_BudgetTable { + customer_organization_defaults LiteLLM_CustomerOrganizationTable[] @relation("CustomerOrganizationDefaultBudget") + customer_organizations LiteLLM_CustomerOrganizationTable[] @relation("CustomerOrganizationBudget") + customer_teams LiteLLM_CustomerTeamTable[] @relation("CustomerTeamBudget") + customer_members LiteLLM_CustomerMemberTable[] @relation("CustomerMemberBudget") + customer_member_overrides LiteLLM_CustomerMemberTable[] @relation("CustomerMemberBudgetOverride") + customer_team_overrides LiteLLM_CustomerTeamTable[] @relation("CustomerTeamBudgetOverride") budget_id String @id @default(uuid()) + name String? max_budget Float? soft_budget Float? max_parallel_requests Int? @@ -22,6 +29,8 @@ model LiteLLM_BudgetTable { model_tpm_limit Json? model_rpm_limit Json? budget_duration String? + budget_limits Json? + cap_policy String? budget_reset_at DateTime? allowed_models String[] @default([]) // per-member model scope; empty = inherit team models temp_budget_increase Float? @@ -40,6 +49,81 @@ model LiteLLM_BudgetTable { organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization } +model LiteLLM_CustomerOrganizationTable { + spend Float @default(0.0) + budget_template_ids String[] @default([]) + budget_id String? + budget LiteLLM_BudgetTable? @relation("CustomerOrganizationBudget", fields: [budget_id], references: [budget_id]) + customer_org_id String @id @default(uuid()) + alias String? + default_budget_template_id String? + default_budget_template LiteLLM_BudgetTable? @relation("CustomerOrganizationDefaultBudget", fields: [default_budget_template_id], references: [budget_id]) + teams LiteLLM_CustomerTeamTable[] + members LiteLLM_CustomerMemberTable[] + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String + @@index([spend]) +} + +model LiteLLM_CustomerTeamTable { + spend Float @default(0.0) + customer_team_id String @id @default(uuid()) + alias String? + parent_org_id String + parent_org LiteLLM_CustomerOrganizationTable @relation(fields: [parent_org_id], references: [customer_org_id]) + budget_id String? + budget LiteLLM_BudgetTable? @relation("CustomerTeamBudget", fields: [budget_id], references: [budget_id]) + budget_override_id String? + budget_override LiteLLM_BudgetTable? @relation("CustomerTeamBudgetOverride", fields: [budget_override_id], references: [budget_id], onDelete: Restrict) + budget_override_expires_at DateTime? + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String + @@index([parent_org_id]) + @@index([spend]) +} + +model LiteLLM_CustomerMemberTable { + spend Float @default(0.0) + customer_member_id String @id @default(uuid()) + alias String? + parent_org_id String + parent_org LiteLLM_CustomerOrganizationTable @relation(fields: [parent_org_id], references: [customer_org_id]) + parent_team_ids String[] @default([]) + budget_id String? + budget LiteLLM_BudgetTable? @relation("CustomerMemberBudget", fields: [budget_id], references: [budget_id], onDelete: Restrict) + budget_override_id String? + budget_override LiteLLM_BudgetTable? @relation("CustomerMemberBudgetOverride", fields: [budget_override_id], references: [budget_id], onDelete: Restrict) + budget_override_expires_at DateTime? + budget_increase_requests LiteLLM_CustomerBudgetIncreaseRequest[] + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String + @@index([parent_org_id]) + @@index([spend]) +} + +model LiteLLM_CustomerBudgetIncreaseRequest { + request_id String @id @default(uuid()) + customer_member_id String + customer_member LiteLLM_CustomerMemberTable @relation(fields: [customer_member_id], references: [customer_member_id], onDelete: Cascade) + requested_max_budget Float + expires_at DateTime? + reason String? + status String @default("pending_review") + submitted_by String + submitted_at DateTime @default(now()) + reviewed_by String? + reviewed_at DateTime? + review_notes String? + @@index([status]) + @@index([customer_member_id]) +} + // Models on proxy model LiteLLM_CredentialsTable { credential_id String @id @default(uuid()) @@ -975,6 +1059,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? @@ -1630,6 +1753,7 @@ model LiteLLM_AccessGroupTable { mcp_tool_permissions Json? access_agent_ids String[] @default([]) access_skill_names String[] @default([]) + access_vector_store_ids String[] @default([]) assigned_team_ids String[] @default([]) assigned_key_ids String[] @default([]) diff --git a/litellm/proxy-extras/litellm_proxy_extras/utils.py b/litellm/proxy-extras/litellm_proxy_extras/utils.py index f8625088d8e..2b988df14f6 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm/proxy-extras/litellm_proxy_extras/utils.py @@ -643,37 +643,63 @@ class ProxyExtrasDBManager: logger.warning(f"Failed to resolve migration {migration_name}: {e.stderr}") @staticmethod - def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None: + def _legacy_schema_object_exists(query: str, subject: str, connect: LensCheckConnect | None) -> bool: database_url: Final = os.environ.get("DATABASE_URL") if not database_url: - return + return False try: import psycopg except ImportError as exc: - raise RuntimeError("Install psycopg to verify Lens data safety before prisma db push.") from exc + raise RuntimeError(f"Install psycopg to verify {subject} data safety before prisma db push.") from exc open_connection: Final = connect if connect is not None else psycopg.connect try: with open_connection( ProxyExtrasDBManager._strip_prisma_query_params(database_url), connect_timeout=10, autocommit=True ) as connection: - legacy: Final = connection.execute( - "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " - "WHERE n.nspname=%s AND c.relname IN ('LiteLLM_Engine', 'LiteLLM_EngineRun', 'LiteLLM_EngineWorker') " - "LIMIT 1", - (ProxyExtrasDBManager._prisma_schema_param(database_url) or "public",), - ).fetchone() + return ( + connection.execute( + query, (ProxyExtrasDBManager._prisma_schema_param(database_url) or "public",) + ).fetchone() + is not None + ) except psycopg.Error as exc: raise RuntimeError( - "Cannot verify Lens data safety; refusing prisma db push. " + f"Cannot verify {subject} data safety; refusing prisma db push. " f"The database check failed: {_redact_credentials(str(exc)).strip()}" ) from exc - if legacy is not None: + + @staticmethod + def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None: + if ProxyExtrasDBManager._legacy_schema_object_exists( + "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace " + "WHERE n.nspname=%s AND c.relname IN ('LiteLLM_Engine', 'LiteLLM_EngineRun', 'LiteLLM_EngineWorker') " + "LIMIT 1", + "Lens", + connect, + ): raise RuntimeError( "Legacy Lens tables exist. prisma db push would drop saved Lens data. " "Apply the shipped 20261001100000_rename_lens migration to this database schema before retrying. " "Deployments using migration history can upgrade without --use_prisma_db_push instead." ) + @staticmethod + def raise_if_customer_member_budget_rename_pending(connect: LensCheckConnect | None = None) -> None: + if ProxyExtrasDBManager._legacy_schema_object_exists( + "SELECT 1 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid " + "JOIN pg_namespace n ON n.oid=c.relnamespace " + "WHERE n.nspname=%s AND c.relname='LiteLLM_CustomerMemberTable' " + "AND a.attname='budget_template_id' AND NOT a.attisdropped LIMIT 1", + "customer member budget", + connect, + ): + raise RuntimeError( + "LiteLLM_CustomerMemberTable still has the budget_template_id column. Syncing the schema would drop it " + "and treat every customer member budget override as the member's budget. Apply the shipped " + "20261112120000_rename_customer_member_budget_columns migration to this database schema " + "before retrying. Deployments with migration history upgrade without this step." + ) + @staticmethod def spend_logs_is_partitioned() -> bool: """True when the connected database's LiteLLM_SpendLogs is a @@ -1037,6 +1063,7 @@ class ProxyExtrasDBManager: if not use_migrate: ProxyExtrasDBManager.raise_if_lens_rename_pending() + ProxyExtrasDBManager.raise_if_customer_member_budget_rename_pending() if ProxyExtrasDBManager.spend_logs_is_partitioned(): raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) original_dir = os.getcwd() @@ -1623,6 +1650,7 @@ class ProxyExtrasDBManager: logger.info( "Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system" ) + ProxyExtrasDBManager.raise_if_customer_member_budget_rename_pending() ProxyExtrasDBManager._create_baseline_migration(schema_path) logger.info("Baseline migration created, resolving all migrations") ProxyExtrasDBManager._resolve_all_migrations(migrations_dir, schema_path) @@ -1715,6 +1743,7 @@ class ProxyExtrasDBManager: raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) # Use prisma db push with increased timeout ProxyExtrasDBManager.raise_if_lens_rename_pending() + ProxyExtrasDBManager.raise_if_customer_member_budget_rename_pending() prisma_toolchain.run_prisma( [_get_prisma_command(), "db", "push", "--accept-data-loss"], timeout=prisma_command_timeout(), diff --git a/litellm/proxy-extras/pyproject.toml b/litellm/proxy-extras/pyproject.toml index 06d77e751c7..1d5f3038ac5 100644 --- a/litellm/proxy-extras/pyproject.toml +++ b/litellm/proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.114" +version = "0.4.116" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.114" +version = "0.4.116" version_files = [ "pyproject.toml:^version", "../../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b99fc8b61e1..5cba49bf884 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -939,6 +939,20 @@ ], "title": "Access Skill Names" }, + "access_vector_store_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Vector Store Ids" + }, "assigned_key_ids": { "anyOf": [ { @@ -1128,6 +1142,21 @@ "title": "Access Skill Names", "type": "array" }, + "access_vector_store_ids": { + "default": [], + "items": { + "type": "string" + }, + "title": "Access Vector Store Ids", + "type": "array" + }, + "access_vector_stores": { + "items": { + "$ref": "#/components/schemas/AccessGroupResource" + }, + "title": "Access Vector Stores", + "type": "array" + }, "assigned_key_ids": { "items": { "type": "string" @@ -1227,6 +1256,7 @@ "assigned_key_ids", "access_mcp_servers", "access_agents", + "access_vector_stores", "assigned_teams", "assigned_keys", "created_at", @@ -1304,6 +1334,20 @@ ], "title": "Access Skill Names" }, + "access_vector_store_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Vector Store Ids" + }, "assigned_key_ids": { "anyOf": [ { @@ -59670,6 +59714,38 @@ "skill_hub": { "components": { "schemas": { + "Body_edit_submission_archive_skill_hub_submissions__submission_id__upload_put": { + "properties": { + "expected_revision": { + "minimum": 0.0, + "title": "Expected Revision", + "type": "integer" + }, + "file": { + "contentMediaType": "application/octet-stream", + "description": "Replacement Agent Skills ZIP with the same declared skill name", + "title": "File", + "type": "string" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Version" + } + }, + "required": [ + "file", + "expected_revision" + ], + "title": "Body_edit_submission_archive_skill_hub_submissions__submission_id__upload_put", + "type": "object" + }, "Body_replace_skill_archive_skill_hub_skills__name__upload_put": { "properties": { "expected_updated_at": { @@ -60227,6 +60303,18 @@ "SkillHubRejection": { "additionalProperties": false, "properties": { + "expected_revision": { + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Expected Revision" + }, "reason": { "anyOf": [ { @@ -60536,6 +60624,12 @@ ], "title": "Base Updated At" }, + "can_edit": { + "default": false, + "description": "Whether the caller may edit or withdraw this pending submission", + "title": "Can Edit", + "type": "boolean" + }, "created_at": { "format": "date-time", "title": "Created At", @@ -60594,6 +60688,13 @@ ], "title": "Reviewed By" }, + "revision": { + "default": 0, + "description": "Proposal revision used to prevent stale edits and reviews", + "minimum": 0.0, + "title": "Revision", + "type": "integer" + }, "skill_name": { "title": "Skill Name", "type": "string" @@ -60602,7 +60703,8 @@ "enum": [ "pending_review", "approved", - "rejected" + "rejected", + "withdrawn" ], "title": "Status", "type": "string" @@ -60653,6 +60755,95 @@ "title": "SkillHubSubmissionDetail", "type": "object" }, + "SkillHubSubmissionEdit": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "maxLength": 1024, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Domain" + }, + "expected_revision": { + "description": "Revision read from the submission detail", + "minimum": 0.0, + "title": "Expected Revision", + "type": "integer" + }, + "expected_updated_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Published updated_at read by the editor. A stale value returns 409", + "title": "Expected Updated At" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Namespace" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Explicit higher SemVer, or omit/null for an automatic patch bump", + "title": "Version" + } + }, + "required": [ + "expected_revision" + ], + "title": "SkillHubSubmissionEdit", + "type": "object" + }, "SkillHubSubmissionList": { "properties": { "count": { @@ -60679,6 +60870,22 @@ "title": "SkillHubSubmissionList", "type": "object" }, + "SkillHubSubmissionRevision": { + "additionalProperties": false, + "properties": { + "expected_revision": { + "description": "Revision read from the submission detail", + "minimum": 0.0, + "title": "Expected Revision", + "type": "integer" + } + }, + "required": [ + "expected_revision" + ], + "title": "SkillHubSubmissionRevision", + "type": "object" + }, "SkillHubVersion": { "properties": { "content_kind": { @@ -62118,7 +62325,8 @@ "enum": [ "pending_review", "approved", - "rejected" + "rejected", + "withdrawn" ], "type": "string" }, @@ -62188,7 +62396,7 @@ }, "/skill-hub/submissions/{submission_id}": { "get": { - "description": "Read an immutable proposal and its currently visible published skill.", + "description": "Read a proposal revision and its currently visible published skill.", "operationId": "get_submission_skill_hub_submissions__submission_id__get", "parameters": [ { @@ -62232,6 +62440,62 @@ "tags": [ "skill_hub" ] + }, + "put": { + "description": "Replace a pending proposal's metadata, preserving files. Only its original submitting user may edit.", + "operationId": "edit_submission_skill_hub_submissions__submission_id__put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmissionEdit" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Submission", + "tags": [ + "skill_hub" + ] } }, "/skill-hub/submissions/{submission_id}/approve": { @@ -62249,6 +62513,23 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SkillHubSubmissionRevision" + }, + { + "type": "null" + } + ], + "title": "Body" + } + } + } + }, "responses": { "200": { "content": { @@ -62506,6 +62787,122 @@ "skill_hub" ] } + }, + "/skill-hub/submissions/{submission_id}/upload": { + "put": { + "description": "Replace a pending proposal's files using the upload limits. Only its original submitting user may edit.", + "operationId": "edit_submission_archive_skill_hub_submissions__submission_id__upload_put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/Body_edit_submission_archive_skill_hub_submissions__submission_id__upload_put" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Edit Submission Archive", + "tags": [ + "skill_hub" + ] + } + }, + "/skill-hub/submissions/{submission_id}/withdraw": { + "put": { + "description": "Withdraw a pending proposal without publishing it. Only its original submitting user may withdraw.", + "operationId": "withdraw_submission_skill_hub_submissions__submission_id__withdraw_put", + "parameters": [ + { + "in": "path", + "name": "submission_id", + "required": true, + "schema": { + "title": "Submission Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmissionRevision" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillHubSubmission" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Withdraw Submission", + "tags": [ + "skill_hub" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 74ab928f93a..124e6c68a85 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_langfuse_span_scope_value, validate_no_callback_env_reference, ) +from litellm.models.budget import BudgetLimits from litellm.proxy.auth.access_group_rules import AccessGroupRule, _validate_rules from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled from litellm.types.agents import ( @@ -470,6 +471,8 @@ class LiteLLMRoutes(enum.Enum): "/skill-hub/submissions", "/skill-hub/submissions/{submission_id}", "/skill-hub/submissions/{submission_id}/archive", + "/skill-hub/submissions/{submission_id}/upload", + "/skill-hub/submissions/{submission_id}/withdraw", "/skill-hub/submissions/{submission_id}/approve", "/skill-hub/submissions/{submission_id}/reject", "/skill-hub/submissions/{submission_id}/scan", @@ -722,6 +725,18 @@ class LiteLLMRoutes(enum.Enum): "/jwt/key/mapping/delete", "/jwt/key/mapping/list", "/jwt/key/mapping/info", + "/management/v1/customer_organizations", + "/management/v1/customer_organizations/daily/activity/aggregated", + "/management/v1/customer_organizations/{customer_org_id}", + "/management/v1/customer_organizations/daily/activity", + "/management/v1/customer_organizations/{customer_org_id}/default_budgets", + "/management/v1/customer_organizations/{customer_org_id}/default_budgets/{budget_id}", + "/management/v1/customer_teams", + "/management/v1/customer_teams/{customer_team_id}", + "/management/v1/customer_teams/daily/activity", + "/management/v1/customer_members", + "/management/v1/customer_members/{customer_member_id}", + "/management/v1/customer_members/daily/activity", ] + key_management_routes + mcp_management_routes @@ -877,6 +892,7 @@ class LiteLLMRoutes(enum.Enum): ) self_managed_routes = [ + "/management/v1/customer_budget_increase_requests", "/lens", "/lens/{path:path}", "/telemetry/ui_events", @@ -1842,6 +1858,15 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase): ) +class BudgetWriteRequest(BudgetNewRequest): + model_config = ConfigDict(frozen=True) + + budget_limits: BudgetLimits | None = Field( + default=None, + description="Independent budget windows. Omit to preserve, or pass null or an empty list to clear.", + ) + + class BudgetRequest(LiteLLMPydanticObjectBase): budgets: list[str] @@ -2346,6 +2371,14 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): default=None, description="List of HTTP methods this endpoint handles (e.g., ['GET', 'POST']). If None or empty, all methods (GET, POST, PUT, DELETE, PATCH) are supported for backward compatibility. This allows the same path to have different targets for different HTTP methods.", ) + display_name: str | None = Field( + default=None, + description="Human-readable name for this endpoint. The Model Hub lists the endpoint under this name, falling back to the path when unset.", + ) + show_in_model_hub: bool = Field( + default=False, + description="If True, this endpoint is published on the public Model Hub as a row in mode 'passthrough'.", + ) class PassThroughEndpointResponse(LiteLLMPydanticObjectBase): @@ -2928,6 +2961,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + redact_tokens_in_spend_logs: bool = Field( + False, + description="Set True to mask token-shaped secrets in stored spend log request and response content. Default is False.", + ) spend_logs_prompt_storage: Literal["latest_turn"] | None = Field( None, description="With store_prompts_in_spend_logs on, 'latest_turn' stores only the current turn of a stateless request (/chat/completions, /v1/messages, and /v1/responses without previous_response_id): the model and the latest user message, with the system prompt, tool definitions and earlier turns dropped. Requests that continue a stored conversation and the response are stored as they are today. Unset stores the full request.", @@ -3833,6 +3870,13 @@ class SpendLogsRouterMetadata(TypedDict): class SpendLogsMetadata(TypedDict): + user_api_key_customer_org_id: ReadOnly[NotRequired[str | None]] + user_api_key_customer_org_alias: ReadOnly[NotRequired[str | None]] + user_api_key_customer_team_id: ReadOnly[NotRequired[str | None]] + user_api_key_customer_team_alias: ReadOnly[NotRequired[str | None]] + user_api_key_customer_member_id: ReadOnly[NotRequired[str | None]] + user_api_key_customer_member_alias: ReadOnly[NotRequired[str | None]] + user_api_key_customer_usage: ReadOnly[NotRequired[str | None]] actor_agent_id: ReadOnly[NotRequired[str | None]] target_agent_id: ReadOnly[NotRequired[str | None]] billing_agent_id: ReadOnly[NotRequired[str | None]] @@ -4474,6 +4518,7 @@ class TeamAccessGroupModelGrant(LiteLLMPydanticObjectBase): mcp_server_ids: tuple[str, ...] = () agent_ids: tuple[str, ...] = () skill_names: tuple[str, ...] = () + vector_store_ids: tuple[str, ...] = () class TeamInfoMember(Member): @@ -4513,6 +4558,7 @@ class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable): access_group_mcp_server_ids: list[str] | None = None access_group_agent_ids: list[str] | None = None access_group_skill_names: tuple[str, ...] | None = None + access_group_vector_store_ids: tuple[str, ...] | None = None # Listed group ids with no row; skill grants are suspended while this is non-empty. unresolved_access_group_ids: tuple[str, ...] | None = None access_group_details: tuple[TeamAccessGroupModelGrant, ...] | None = None @@ -4630,7 +4676,7 @@ class LitellmMetadataFromRequestHeaders(TypedDict, total=False): Headers a user can pass that will get added to litellm metadata for the request """ - spend_logs_metadata: dict | None + spend_logs_metadata: ReadOnly[object] agent_id: str | None trace_id: str | None session_id: str | None @@ -5376,6 +5422,12 @@ class DailyAgentSpendTransaction(BaseDailySpendTransaction): agent_id: str +class DailyCustomerMemberSpendTransaction(BaseDailySpendTransaction): + customer_member_id: ReadOnly[str] + customer_team_id: ReadOnly[str] + customer_org_id: ReadOnly[str] + + class DBSpendUpdateTransactions(TypedDict): """ Internal Data Structure for buffering spend updates in Redis or in memory before committing them to the database @@ -5392,6 +5444,9 @@ class DBSpendUpdateTransactions(TypedDict): tag_list_transactions: dict[str, float] | None agent_list_transactions: dict[str, float] | None model_access_group_list_transactions: ReadOnly[dict[str, float] | None] + customer_org_list_transactions: ReadOnly[dict[str, float] | None] # mutable-ok: aggregated like siblings + customer_team_list_transactions: ReadOnly[dict[str, float] | None] # mutable-ok: aggregated like siblings + customer_member_list_transactions: ReadOnly[dict[str, float] | None] # mutable-ok: aggregated like siblings class SpendUpdateQueueItem(TypedDict, total=False): diff --git a/litellm/proxy/alerting_endpoints/alerts.py b/litellm/proxy/alerting_endpoints/alerts.py index a6ed06c3b96..9c701208e0d 100644 --- a/litellm/proxy/alerting_endpoints/alerts.py +++ b/litellm/proxy/alerting_endpoints/alerts.py @@ -12,7 +12,7 @@ from litellm.models.alert import ( AlertView, AlertViewPage, ) -from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate +from litellm.models.performance_alert import AlertMonitoring, AlertMonitoringUpdate, ModelDeprecationMonitoringUpdate from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import AlertMonitoringRepositoryDep, AlertRepositoryDep @@ -60,7 +60,7 @@ async def list_alerts( return AlertViewPage(data=await store.enrich(page.data), next_cursor=page.next_cursor) -@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read performance monitoring settings") +@router.get("/alerts/monitoring", response_model=AlertMonitoring, summary="Read alert monitoring settings") async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) -> AlertMonitoring: """Read the gateway's opt-out setting and free allowance. Requires a gateway admin or view-only admin.""" _authorize(user, write=False) @@ -69,14 +69,16 @@ async def get_monitoring(user: Auth, repository: AlertMonitoringRepositoryDep) - return await repository.monitoring() -@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable performance monitoring") +@router.patch("/alerts/monitoring", response_model=AlertMonitoring, summary="Enable or disable alert monitoring") async def update_monitoring( - data: AlertMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep + data: AlertMonitoringUpdate | ModelDeprecationMonitoringUpdate, user: Auth, repository: AlertMonitoringRepositoryDep ) -> AlertMonitoring: """Change monitoring for every worker using this database. Only gateway admins can write. Keeps saved alerts.""" _authorize(user, write=True) if repository is None: raise HTTPException(status_code=503, detail="Alerts require a database connection") + if isinstance(data, ModelDeprecationMonitoringUpdate): + return await repository.set_model_deprecations_enabled(data.model_deprecations_enabled) if data.enabled and (await repository.monitoring()).disabled_by_config: raise HTTPException(status_code=409, detail="Performance monitoring is disabled in proxy settings") return await repository.set_enabled(data.enabled) diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py index 4775498dec3..b0f77158d4b 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_skill_access.py @@ -2,20 +2,25 @@ Claude Code marketplace visibility: enabled plugins are public, disabled plugins are private and resolve only for proxy admins or keys granted them via ``object_permission.skills`` or via ``access_skill_names`` on an access group -attached to the key or its team. +attached to the key or its team. Dashboard sessions inherit their user's and live teams' grants. """ from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Protocol, TypeAlias +from pydantic import TypeAdapter + +from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UI_TEAM_ID, LiteLLM_ObjectPermissionTable, UserAPIKeyAuth from litellm.proxy.common_utils.resource_ownership import is_proxy_admin +from litellm.proxy.management.teams.authz import is_team_member +from litellm.proxy.mcp_server.ui_session_utils import is_ui_session_credential if TYPE_CHECKING: from prisma.types import LiteLLM_ClaudeCodePluginTableWhereInput - from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable class _SkillRecord(Protocol): @@ -113,6 +118,84 @@ async def _team_object(user_api_key_dict: UserAPIKeyAuth, check_db_only: bool) - return None if team_obj.blocked else team_obj +async def _team_group_skills(team: "LiteLLM_TeamTable | None", check_db_only: bool) -> frozenset[str]: + from litellm.proxy.auth.auth_checks import get_skill_names_from_access_groups + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + return await get_skill_names_from_access_groups( + access_group_ids=tuple(team.access_group_ids or ()) if team is not None else (), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, + strict_errors=True, + ) + + +async def _ui_team_skills(caller: UserAPIKeyAuth, team_id: str) -> frozenset[str]: + try: + team: Final = await _team_object(caller.model_copy(update={"team_id": team_id}), check_db_only=True) + if team is None or not is_team_member(caller, team): + return frozenset() + inherited: Final = await _team_group_skills(team, check_db_only=True) + except Exception: # noqa: BLE001 # a failed team source grants nothing without removing independent sources + verbose_proxy_logger.warning("Cannot resolve dashboard team's skill grants") + return frozenset() + return _skills_of(team.object_permission) | inherited + + +async def _ui_user_skills(user: "LiteLLM_UserTable", caller: UserAPIKeyAuth) -> frozenset[str]: + from litellm.proxy.auth.auth_checks import get_object_permission + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if not user.object_permission_id: + return _skills_of(user.object_permission) + try: + permission: Final = await get_object_permission( + object_permission_id=user.object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=caller.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_db_only=True, + ) + except Exception: # noqa: BLE001 # an unreadable direct grant must not fall back to cached permission + verbose_proxy_logger.warning("Cannot resolve dashboard user's skill permission") + return frozenset() + return _skills_of(permission) + + +async def _ui_skill_visibility(caller: UserAPIKeyAuth) -> SkillVisibility: + from litellm.proxy.auth.auth_checks import get_user_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if prisma_client is None: + return PUBLIC_ONLY + try: + user: Final = await get_user_object( + user_id=caller.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=caller.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_db_only=True, + ) + except Exception: # noqa: BLE001 # missing or unreadable user identity cannot establish private grants + verbose_proxy_logger.warning("Cannot resolve dashboard user's skill grants") + return PUBLIC_ONLY + if user is None: + return PUBLIC_ONLY + metadata: Final = TypeAdapter(dict[str, object]).validate_python( + user.model_dump(include={"metadata"})["metadata"] or {} + ) + if metadata.get("scim_active") is False: + return PUBLIC_ONLY + own: Final = await _ui_user_skills(user, caller) + teams: Final = [await _ui_team_skills(caller, team_id) for team_id in frozenset(user.teams) if team_id] + return SkillVisibility(granted=own.union(*teams), sees_private=False) + + async def access_group_skills(user_api_key_dict: UserAPIKeyAuth) -> AccessGroupSkills: """Resolve ``access_skill_names`` of the groups attached to the key and to its team. @@ -122,9 +205,8 @@ async def access_group_skills(user_api_key_dict: UserAPIKeyAuth) -> AccessGroupS from litellm.proxy.auth.auth_checks import ( AccessGroupNotFoundError, get_authorized_resources_from_key_access_groups, - get_skill_names_from_access_groups, ) - from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + from litellm.proxy.proxy_server import prisma_client if prisma_client is None: return NO_ACCESS_GROUP_SKILLS @@ -140,14 +222,7 @@ async def access_group_skills(user_api_key_dict: UserAPIKeyAuth) -> AccessGroupS strict_errors=True, ) ) - team_skills: Final = await get_skill_names_from_access_groups( - access_group_ids=tuple(team_obj.access_group_ids or ()) if team_obj is not None else (), - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - check_db_only=check_db_only, - strict_errors=True, - ) + team_skills: Final = await _team_group_skills(team_obj, check_db_only) return AccessGroupSkills(key=key_skills, team=team_skills) except AccessGroupNotFoundError: return AccessGroupSkills(unresolved=True) @@ -160,7 +235,9 @@ async def resolve_skill_visibility( user_api_key_dict: UserAPIKeyAuth | None, load_access_group_skills: AccessGroupSkillsLoader = access_group_skills, ) -> SkillVisibility: - """``skill_visibility`` with the access-group grants of the key and its team folded in.""" + """Resolve credential-scoped grants, or live user/team grants for a dashboard session.""" if user_api_key_dict is None or is_proxy_admin(user_api_key_dict): return skill_visibility(user_api_key_dict) + if is_ui_session_credential(user_api_key_dict): + return await _ui_skill_visibility(user_api_key_dict) return skill_visibility(user_api_key_dict, await load_access_group_skills(user_api_key_dict)) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c7b698b7f18..e1cd3edbefe 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -54,6 +54,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( is_base64_encoded_unified_id, ) from litellm.models.project import LiteLLM_ProjectTable +from litellm.models.team import LiteLLM_TeamRebuiltFromToken from litellm.proxy._types import ( MODEL_LIST_ROUTES, RBAC_ROLES, @@ -384,6 +385,9 @@ class _PrismaMaxBudgetRow(Protocol): @property def max_budget(self) -> float | None: ... + @property + def budget_limits(self) -> object: ... + class _PrismaModelAccessGroupBudgetRow(Protocol): access_group_name: str @@ -1361,7 +1365,12 @@ async def common_checks( ) ## 2.1 If user can call model (if personal key) - if _model and team_object is None and user_object is not None: + if ( + _model + and team_object is None + and user_object is not None + and not (_is_dashboard_session(valid_token) and personal_key_team_model_access_applies(valid_token)) + ): with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"): await can_user_call_model( model=_model, @@ -1811,6 +1820,9 @@ async def get_default_end_user_budget( ) return None + from litellm.proxy.management_helpers.utils import reject_budget_windows + + reject_budget_windows(budget_record.dict()) _budget_obj: Final = LiteLLM_BudgetTable.model_validate(budget_record.dict()) # Cache the budget for 60 seconds await user_api_key_cache.async_set_cache( @@ -1822,6 +1834,8 @@ async def get_default_end_user_budget( return _budget_obj + except HTTPException: + raise except Exception as e: verbose_proxy_logger.error("Error fetching default end user budget: %s", e) return None @@ -1876,6 +1890,9 @@ async def get_team_member_default_budget( verbose_proxy_logger.warning("Team-default member budget not found in database: %s", budget_id) return None + from litellm.proxy.management_helpers.utils import reject_budget_windows + + reject_budget_windows(budget_record.dict()) budget: Final = LiteLLM_BudgetTable.model_validate(budget_record.dict()) await user_api_key_cache.async_set_cache( key=cache_key, @@ -2507,6 +2524,8 @@ async def _fetch_uncached_model_access_group_budgets( model_type=ModelAccessGroupBudget, ttl=get_management_object_ttl(user_api_key_cache), ) + except HTTPException: + raise except Exception as e: # noqa: BLE001 # fail-safe: a budget fetch error must yield "no budget rows", never break auth verbose_proxy_logger.debug("Error batch fetching model access group budgets from database: %s", e) return () @@ -2515,7 +2534,11 @@ async def _fetch_uncached_model_access_group_budgets( def _model_access_group_budget(row: _PrismaModelAccessGroupBudgetRow) -> ModelAccessGroupBudget: + from litellm.proxy.management_helpers.utils import reject_budget_windows + budget_table: Final = row.litellm_budget_table + if budget_table is not None: + reject_budget_windows({"budget_limits": budget_table.budget_limits}) return ModelAccessGroupBudget( access_group_name=row.access_group_name, spend=row.spend or 0.0, @@ -3247,7 +3270,7 @@ async def cache_team_object( if team_table.team_alias: alias_key: Final = f"team_alias:{team_table.team_alias}" try: - pipelined_delete: Final = await user_api_key_cache.async_delete_cache_pre_call(alias_key) + pipelined_delete: Final = user_api_key_cache.declare_delete(alias_key) if pipelined_delete is None: await user_api_key_cache.async_delete_cache(key=alias_key) else: @@ -5553,6 +5576,27 @@ async def can_key_call_model( Raises: - Exception: If token not allowed to call model """ + if _is_dashboard_session(valid_token) and personal_key_team_model_access_applies(valid_token): + from litellm.proxy.proxy_server import ( + prisma_client as dashboard_prisma_client, + ) + from litellm.proxy.proxy_server import ( + proxy_logging_obj, + user_api_key_cache, + ) + + owner: Final = await _personal_key_owner( + valid_token, dashboard_prisma_client, user_api_key_cache, proxy_logging_obj + ) + return await can_personal_key_call_model_via_teams( + model=model, + user_object=owner, + valid_token=valid_token, + llm_router=llm_router, + prisma_client=dashboard_prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token) try: return can_object_call_model( @@ -5619,12 +5663,25 @@ async def can_key_call_resolved_model( valid_token: UserAPIKeyAuth, llm_router: RegistryReader | None, ) -> None: + await _can_key_call_resolved_model(model, llm_model_list, valid_token.dashboard_model_access_auth(), llm_router) + + +async def _can_key_call_resolved_model( + model: str, + llm_model_list: Sequence[object] | None, + valid_token: UserAPIKeyAuth, + llm_router: RegistryReader | None, +) -> None: + from litellm.proxy.auth.customer_hierarchy_auth import check_customer_hierarchy_model_access from litellm.proxy.proxy_server import ( prisma_client, proxy_logging_obj, user_api_key_cache, ) + check_customer_model_access(model=model, valid_token=valid_token, llm_router=llm_router) + if valid_token.customer_hierarchy is not None: + check_customer_hierarchy_model_access(valid_token.customer_hierarchy, model, llm_router) skip_key_model_check: Final = valid_token.config or ( isinstance(valid_token.models, list) and SpecialModelNames.all_team_models.value in valid_token.models ) @@ -5807,6 +5864,17 @@ async def can_team_access_model( raise +def _group_grants_key( + group: LiteLLM_AccessGroupTable, valid_token: UserAPIKeyAuth, team_object: LiteLLM_TeamTable | None +) -> bool: + key_team_id: Final = valid_token.team_id or (team_object.team_id if team_object is not None else None) + return ( + bool(key_team_id and key_team_id in (group.assigned_team_ids or [])) + or bool(valid_token.token and valid_token.token in (group.assigned_key_ids or [])) + or group.access_group_id in valid_token.access_group_rule_grants + ) + + async def get_authorized_resources_from_key_access_groups( valid_token: UserAPIKeyAuth | None, team_object: LiteLLM_TeamTable | None, @@ -5835,9 +5903,6 @@ async def get_authorized_resources_from_key_access_groups( if _prisma_client is None or _user_api_key_cache is None: return [] - key_team_id: Final = valid_token.team_id or (team_object.team_id if team_object is not None else None) - key_token: Final = valid_token.token - authorized_resources: Final[list[str]] = [] for ag_id in key_access_group_ids: try: @@ -5856,9 +5921,7 @@ async def get_authorized_resources_from_key_access_groups( if check_db_only or strict_errors: raise continue - team_authorized = bool(key_team_id and key_team_id in (ag.assigned_team_ids or [])) - key_authorized = bool(key_token and key_token in (ag.assigned_key_ids or [])) - if team_authorized or key_authorized or ag_id in valid_token.access_group_rule_grants: + if _group_grants_key(ag, valid_token, team_object): authorized_resources.extend(getattr(ag, resource_field, []) or []) return list(set(authorized_resources)) @@ -5922,9 +5985,9 @@ def can_project_access_model( ) -def can_customer_access_model( - model: str | list[str], - end_user_object: LiteLLM_EndUserTable, +def _can_customer_scope_call_model( + model: str | Sequence[str], + models: Sequence[str], llm_router: RegistryReader | None, valid_token: UserAPIKeyAuth | None, ) -> Literal[True]: @@ -5936,12 +5999,12 @@ def can_customer_access_model( team_target: Final = ( _live_team_alias_target(name, team_model_aliases, team_id, llm_router) if team_model_aliases else name ) - if team_target != name and name in (end_user_object.models or ()): + if team_target != name and name in models: return can_object_call_model( model=team_target, llm_router=llm_router, - models=end_user_object.models, + models=list(models), key_model_aliases=key_model_aliases, object_type="customer", ) @@ -5951,6 +6014,22 @@ def can_customer_access_model( return True +def can_customer_access_model( + model: str | list[str], + end_user_object: LiteLLM_EndUserTable, + llm_router: RegistryReader | None, + valid_token: UserAPIKeyAuth | None, +) -> Literal[True]: + return _can_customer_scope_call_model(model, end_user_object.models or (), llm_router, valid_token) + + +def check_customer_model_access( + model: str | Sequence[str], valid_token: UserAPIKeyAuth, llm_router: RegistryReader | None +) -> None: + for scope in valid_token.customer_model_scopes: + _can_customer_scope_call_model(model, scope, llm_router, valid_token) + + async def can_user_call_model( model: str | list[str], llm_router: Router | None, @@ -5984,9 +6063,11 @@ async def can_user_call_model( def personal_key_team_model_access_applies(valid_token: UserAPIKeyAuth | None) -> bool: return ( - bool(litellm.personal_key_model_access_from_teams) - and valid_token is not None - and valid_token.team_id is None + valid_token is not None + and ( + _is_dashboard_session(valid_token) + or (bool(litellm.personal_key_model_access_from_teams) and valid_token.team_id is None) + ) and valid_token.user_id is not None and valid_token.api_key != LITELLM_PROXY_MASTER_KEY_ALIAS and valid_token.user_role != LitellmUserRoles.PROXY_ADMIN @@ -5999,10 +6080,41 @@ def personal_key_inherits_team_grants(valid_token: UserAPIKeyAuth | None) -> boo return ( personal_key_team_model_access_applies(valid_token) and valid_token is not None + and valid_token.team_id is None and valid_token.api_key is not None ) +_DASHBOARD_USER_MODELS_ADAPTER: Final[TypeAdapter[Mapping[str, tuple[str, ...]]]] = TypeAdapter( + Mapping[str, tuple[str, ...]] +) + + +def _dashboard_user_grants_model( + model: str, + valid_token: UserAPIKeyAuth, + user_object: LiteLLM_UserTable | None, + llm_router: RegistryReader | None, +) -> bool: + if not _is_dashboard_session(valid_token) or user_object is None: + return False + try: + own_models: Final = _DASHBOARD_USER_MODELS_ADAPTER.validate_python(user_object.model_dump(include={"models"}))[ + "models" + ] + if SpecialModelNames.no_default_models.value in own_models: + return False + return can_object_call_model( + model=model, + models=list(own_models), + llm_router=llm_router, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + object_type="user", + ) + except (ProxyException, ValidationError): + return False + + @dataclass(frozen=True, slots=True) class _PersonalKeyTeamLoad: team: LiteLLM_TeamTableCachedObj | None @@ -6112,6 +6224,14 @@ async def _team_grants_personal_key_model( proxy_logging_obj=proxy_logging_obj, key_model_aliases=key_model_aliases, ) + if _is_dashboard_session(valid_token): + return prisma_client is not None and await _personal_key_team_within_budget( + team=team_object, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) except ProxyException: return False except Exception as e: # noqa: BLE001 # fail closed: a team check that errors grants nothing @@ -6194,6 +6314,11 @@ async def _personal_key_team_within_budget( *, check_db_only: bool = False, ) -> bool: + if ( + valid_token.dashboard_session_origin is not None + and valid_token.dashboard_session_billing_team_id == team.team_id + ): + return True source: Final = valid_token.model_copy( update={"team_id": team.team_id, "org_id": valid_token.org_id or team.organization_id} ) @@ -6352,6 +6477,77 @@ async def personal_key_owner_teams( ).teams +async def dashboard_model_billing_auth( + auth: UserAPIKeyAuth, + model: str | Sequence[str] | None, + llm_router: RegistryReader | None, +) -> UserAPIKeyAuth: + from litellm.proxy.auth.team_grants import team_grant_billing_auth_update + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if not _is_dashboard_session(auth) or not personal_key_team_model_access_applies(auth) or model is None: + return auth + origin: Final = auth.dashboard_session_origin or auth + owner: Final = await _personal_key_owner(origin, prisma_client, user_api_key_cache, proxy_logging_obj) + if not isinstance(model, str): + if not all(_dashboard_user_grants_model(entry, origin, owner, llm_router) for entry in model): + raise HTTPException(status_code=403, detail="Dashboard batches cannot use team-only models") + return auth + fields: Final = frozenset( + name for name in UserAPIKeyAuth.model_fields if name.startswith(("team_", "organization_")) or name == "org_id" + ) + original_billing: Final = TypeAdapter(Mapping[str, object]).validate_python(origin.model_dump(include=set(fields))) + base: Final = auth.model_copy(update=original_billing) + if _dashboard_user_grants_model(model, origin, owner, llm_router): + if auth.dashboard_session_origin is None: + base.bind_dashboard_session(origin, model=model) + return base + team: Final = await _dashboard_model_billing_team(base, model, llm_router) + membership: Final = ( + await get_team_membership( + user_id=origin.user_id, + team_id=team.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if origin.user_id is not None and prisma_client is not None + else None + ) + update: Final = await team_grant_billing_auth_update(base, team, membership) + billed: Final = base.model_copy(update=update) + if auth.dashboard_session_origin is None: + billed.bind_dashboard_session(origin, model=model) + return billed + + +async def _dashboard_model_billing_team( + origin: UserAPIKeyAuth, model: str, llm_router: RegistryReader | None +) -> LiteLLM_TeamTableCachedObj: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + teams: Final = await _personal_key_listing_teams(origin, prisma_client, user_api_key_cache, proxy_logging_obj) + loaded: Final = tuple(team for team in teams if team is not None) + allowed: Final = await asyncio.gather( + *( + _team_grants_personal_key_model( + model, team, origin, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj + ) + for team in loaded + ) + ) + eligible: Final = tuple(team for team, granted in zip(loaded, allowed, strict=True) if granted) + if not eligible: + raise ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=model), + internal_message=f"No eligible owner team grants dashboard model {model}", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=status.HTTP_403_FORBIDDEN, + ) + return min(eligible, key=lambda source: source.team_id) + + async def personal_key_team_visible_models( models: Sequence[str], user_api_key_dict: UserAPIKeyAuth, @@ -6359,19 +6555,51 @@ async def personal_key_team_visible_models( prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache | None, proxy_logging_obj: ProxyLogging | None, + only_model_access_groups: bool = False, + team_object: LiteLLM_TeamTableCachedObj | None = None, + return_wildcard_routes: bool = False, ) -> tuple[str, ...]: - teams: Final = await _personal_key_listing_teams( - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, + owner: Final = ( + await _personal_key_owner(user_api_key_dict, prisma_client, user_api_key_cache, proxy_logging_obj) + if _is_dashboard_session(user_api_key_dict) + and team_object is None + and user_api_key_cache is not None + and proxy_logging_obj is not None + else None + ) + candidates: Final = ( + tuple( + dict.fromkeys( + ( + *models, + *(name for name in llm_router.registry().names() if return_wildcard_routes or "*" not in name), + ) + ) + ) + if _is_dashboard_session(user_api_key_dict) + and llm_router is not None + and not only_model_access_groups + and team_object is None + else models + ) + teams: Final = tuple( + (team_object,) + if team_object is not None + else await _personal_key_listing_teams( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) ) return tuple( [ model - for model in (models if teams else ()) - if ( - prisma_client is not None + for model in candidates + if (team_object is None and _dashboard_user_grants_model(model, user_api_key_dict, owner, llm_router)) + or ( + bool(teams) + and prisma_client is not None and user_api_key_cache is not None and proxy_logging_obj is not None and await _loaded_teams_allow_model( @@ -6410,6 +6638,8 @@ async def can_personal_key_call_model_via_teams( else () ) for requested_model in requested_models: + if _dashboard_user_grants_model(requested_model, valid_token, user_object, llm_router): + continue if user_object is None or not await _loaded_teams_allow_model( model=requested_model, teams=teams, @@ -6514,7 +6744,9 @@ def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool def _is_dashboard_session(valid_token: UserAPIKeyAuth | None) -> bool: - return valid_token is not None and valid_token.team_id == UI_TEAM_ID + return valid_token is not None and ( + valid_token.team_id == UI_TEAM_ID or valid_token.dashboard_session_origin is not None + ) def _is_search_tool_strict_identity(valid_token: UserAPIKeyAuth) -> bool: @@ -8095,10 +8327,104 @@ def _legacy_requested_vector_store_ids( ) -def _restricts_vector_stores(grants: Sequence[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None]]) -> bool: +def _restricts_vector_stores(grants: Sequence[tuple[GrantLayer, _VectorStorePermissionsRow | None]]) -> bool: return any(grant is not None and bool(grant.vector_stores) for _, grant in grants) +async def access_group_vector_stores( + valid_token: UserAPIKeyAuth | None, + team_object: LiteLLM_TeamTable | None, + unreadable: Literal["skip", "raise"] = "skip", +) -> Mapping[GrantLayer, tuple[str, ...]]: + """The stores the key's and its team's access groups list, per layer. + A deleted group lists nothing. So does a group that cannot be read, unless `unreadable` is "raise".""" + key_groups: Final = await get_access_group_tables( + (valid_token.access_group_ids or ()) if valid_token is not None else (), unreadable=unreadable + ) + team_groups: Final = await get_access_group_tables( + (team_object.access_group_ids or ()) if team_object is not None else (), unreadable=unreadable + ) + listed_key_groups: Final = ( + tuple(group for group in key_groups if _group_grants_key(group, valid_token, team_object)) + if valid_token is not None + else () + ) + return MappingProxyType( + { + "key": tuple(chain.from_iterable(group.access_vector_store_ids for group in listed_key_groups)), + "team": tuple(chain.from_iterable(group.access_vector_store_ids for group in team_groups)), + } + ) + + +async def _team_read_again( + valid_token: UserAPIKeyAuth | None, team_object: LiteLLM_TeamTable | None +) -> LiteLLM_TeamTable | None: + """Auth's stand-in for a team it could not read has none of the team's groups, so the team is read again.""" + team_id: Final = valid_token.team_id if valid_token is not None else None + if ( + not isinstance(team_object, LiteLLM_TeamRebuiltFromToken) + or valid_token is None + or team_id is None + or team_id == UI_TEAM_ID + ): + return team_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + return await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=valid_token.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + + +@dataclass(frozen=True, slots=True) +class _OwnAndGroupVectorStores: + vector_stores: Sequence[str] + + +def _with_group_stores( + grant: LiteLLM_ObjectPermissionTable | None, group_stores: Sequence[str] +) -> _VectorStorePermissionsRow | None: + if not group_stores: + return grant + own_stores: Final = (grant.vector_stores if grant is not None else None) or () + return _OwnAndGroupVectorStores([*dict.fromkeys((*own_stores, *group_stores))]) + + +async def _vector_store_grants( + valid_token: UserAPIKeyAuth | None, + team_object: LiteLLM_TeamTable | None, + user_object: LiteLLM_UserTable | None, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + strict_layers: frozenset[GrantLayer] = frozenset(), +) -> tuple[tuple[GrantLayer, _VectorStorePermissionsRow | None], ...]: + """Each identity's own grant. The stores its access groups list count as if they were on its own list. + A strict team layer can only lose grants without its groups, so it keeps auth's stand-in team.""" + try: + team: Final = team_object if "team" in strict_layers else await _team_read_again(valid_token, team_object) + group_stores: Final = await access_group_vector_stores(valid_token, team, unreadable="raise") + except Exception as e: + # Skipping an unreadable team or group could empty a list and lift the restriction. + raise ProxyException( + message="Could not read the team or its access groups to check vector store access", + type=ProxyErrorTypes.internal_server_error, + param="vector_store", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) from e + return tuple( + [ + (layer, _with_group_stores(grant, group_stores.get(layer, ()))) + for layer, grant in await _identity_grants( + valid_token, team_object, user_object, prisma_client, user_api_key_cache + ) + ] + ) + + def _request_drop_policies( request_body: Mapping[str, object], llm_router: Router | None, valid_token: UserAPIKeyAuth | None ) -> tuple[DropPolicy, ...]: @@ -8161,7 +8487,7 @@ async def vector_store_access_check( Requested stores come from `vector_store_ids`, `tools[].vector_store_ids`, the tools of `additional_tools` input items and the RAG `retrieval_config.vector_store_id`. Grants come from each identity's - `object_permission.vector_stores`. + `object_permission.vector_stores`, plus the stores its access groups list, which count as if they were on it. Both modes read the same request fields, with or without a vector store registry, and answer 400 when the top-level `vector_store_ids` is not a list of strings, which legacy mode does only when the @@ -8209,7 +8535,7 @@ async def vector_store_access_check( ) if isinstance(requested_ids, _UnreadableVectorStoreIds): if _restricts_vector_stores( - await _identity_grants(valid_token, team_object, None, prisma_client, user_api_key_cache) + await _vector_store_grants(valid_token, team_object, None, prisma_client, user_api_key_cache) ): raise _malformed_vector_store_ids() return True @@ -8223,12 +8549,13 @@ async def vector_store_access_check( # Check if the object (key, team, org) has access to the vector store ######################################################### strict_layers: Final = _strict_grant_layers(deny_by_default, valid_token, team_object) - grants: Final = await _identity_grants( + grants: Final = await _vector_store_grants( valid_token, team_object, user_object if "user" in strict_layers else None, prisma_client, user_api_key_cache, + strict_layers, ) for layer, grant in grants: if layer in strict_layers: diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 2ec2274db2b..cadaba7bc44 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -90,6 +90,14 @@ def _get_user_agent(request: Request) -> str | None: return request.headers.get("user-agent") +def _get_spend_logs_metadata(request: Request) -> Mapping[str, object] | None: + if "headers" not in request.scope: + return None + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + return as_str_mapping(LiteLLMProxyRequestSetup.get_spend_logs_metadata_from_request_headers(request.headers)) + + def _with_client_context( request_data: dict[str, object], requester_ip: str | None, user_agent: str | None ) -> dict[str, object]: @@ -108,6 +116,27 @@ def _with_client_context( return {**request_data, key: {**base, **stamped}} +def _metadata_bucket(request_data: Mapping[str, object], key: str) -> Mapping[str, object]: + metadata: Final = request_data.get(key) + return metadata if isinstance(metadata, Mapping) else EMPTY_MAPPING + + +def _with_spend_logs_metadata( + request_data: dict[str, object], route: str, spend_logs_metadata: Mapping[str, object] | None +) -> dict[str, object]: + """The failure spend row reads `metadata` on every route, while callbacks on + `litellm_metadata` routes read that bucket, so the header goes into both.""" + if spend_logs_metadata is None: + return request_data + from litellm.proxy.litellm_pre_call_utils import metadata_variable_name_for_request + + keys: Final = frozenset(("metadata", metadata_variable_name_for_request(route, request_data))) + return { + **request_data, + **{key: {**_metadata_bucket(request_data, key), "spend_logs_metadata": spend_logs_metadata} for key in keys}, + } + + def _escape_control_chars(value: str) -> str: return "".join(ch if ch.isprintable() else ch.encode("unicode_escape").decode("ascii") for ch in value) @@ -146,6 +175,7 @@ class UserAPIKeyAuthExceptionHandler: parent_otel_span: Span | None, api_key: str, resolved_identity: UserAPIKeyAuth | None = None, + dispatched_call_type: str | None = None, ) -> UserAPIKeyAuth: """ Handles Connection Errors when reading a Virtual Key from LiteLLM DB @@ -253,11 +283,16 @@ class UserAPIKeyAuthExceptionHandler: # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( - request_data=_with_client_context(request_context, requester_ip, user_agent), + request_data=_with_spend_logs_metadata( + _with_client_context(request_context, requester_ip, user_agent), + normalize_request_route(route), + _get_spend_logs_metadata(request), + ), original_exception=e, user_api_key_dict=user_api_key_dict, error_type=ProxyErrorTypes.auth_error, route=route, + dispatched_call_type=dispatched_call_type, ) # Use transformed exception if callback returned one, otherwise use original if transformed_exception is not None: diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 37dba77ffde..827dbeee9b7 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -2429,12 +2429,14 @@ _MODEL_ROUTING_ROUTE_MARKERS: Final = ( "/evals", "/fine_tuning", "/videos", + "/interactions/", ) _MODEL_ROUTING_HEADER_OR_QUERY_ROUTE_MARKERS: Final = ( "/files", "/batches", "/skills", "/evals", + "/interactions/", ) _MODEL_ROUTING_QUERY_TARGET_MODEL_ROUTE_MARKERS: Final = ( "/files", @@ -2608,6 +2610,19 @@ def video_access_group( return creating_group +_INTERACTION_BY_ID_ROUTE_MARKERS: Final = ("/interactions/",) + + +def _group_of_deployment_selector(selector: object, llm_router: Router | None) -> object: + if not isinstance(selector, str) or llm_router is None: + return selector + registry: Final = llm_router.registry() + if selector in registry.index("model_name"): + return selector + deployment: Final = registry.deployment(selector) + return selector if deployment is None else deployment.model_name + + def _resolve_model_id_with_router(model_id: str | None, llm_router: Router | None) -> str | None: if model_id is None or llm_router is None: return model_id @@ -2686,9 +2701,15 @@ def _extract_model_candidates_from_request( uses_completion_model_sources: Final = _route_matches_any_marker( route=route, markers=_MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS ) + selects_by_interaction_id: Final = _route_matches_any_marker(route=route, markers=_INTERACTION_BY_ID_ROUTE_MARKERS) body_model: Final = request_data.get("model") - _append_model_candidates(candidates, body_model) + _append_model_candidates( + candidates, + _group_of_deployment_selector(body_model, llm_router) + if selects_by_interaction_id and isinstance(body_model, str) + else body_model, + ) if uses_body_target_model_sources or not body_model: _append_model_candidates(candidates, request_data.get("target_model_names")) if _route_matches_any_marker(route=route, markers=_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS): @@ -2700,14 +2721,14 @@ def _extract_model_candidates_from_request( if uses_model_routing_sources: if uses_header_or_query_model_sources: - _append_model_candidates( - candidates, + for selector in ( _get_case_insensitive_mapping_value(request_query_params, "model"), - ) - _append_model_candidates( - candidates, _get_case_insensitive_mapping_value(request_headers, MODEL_ROUTING_HEADER_NAME), - ) + ): + _append_model_candidates( + candidates, + _group_of_deployment_selector(selector, llm_router) if selects_by_interaction_id else selector, + ) if uses_query_target_model_sources: _append_model_candidates( candidates, diff --git a/litellm/proxy/auth/customer_hierarchy_auth.py b/litellm/proxy/auth/customer_hierarchy_auth.py new file mode 100644 index 00000000000..11b6ae2c488 --- /dev/null +++ b/litellm/proxy/auth/customer_hierarchy_auth.py @@ -0,0 +1,790 @@ +"""Request-path resolution of the customer organization, team and member a request runs under. + +A request names its customer member with the ``x-litellm-customer-member-id`` header. The member row +carries its team and that team's organization, so one lookup yields every level whose budget and rate +limits apply to the request. A request may instead, or additionally, name the organization by its alias +with ``x-litellm-customer-org-alias``: alone it charges the organization level only, next to the member +header it must agree with the member's organization. +""" + +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias + +from fastapi import HTTPException, status +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +import litellm +from litellm._internal_context import with_service_target +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.auth.auth_checks import can_object_call_model +from litellm.proxy.auth.auth_object_prefetch import AUTH_OBJECTS_TARGET +from litellm.proxy.auth.customer_member_auto_create import ensure_customer_member, ensure_customer_organization +from litellm.proxy.auth.customer_spend import current_customer_spend +from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast +from litellm.proxy.common_utils.customer_usage import CustomerIdentifier +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, get_management_object_ttl +from litellm.proxy.management_helpers.customer_hierarchy import ( + ErrorCode, + Failure, + PrismaRecord, + customer_row_reset_at, + effective_member_budget, + related_budget, + utc_now, +) +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import CustomerTeamRepository +from litellm.types.proxy.auth.customer_hierarchy import ( + CustomerHierarchyAuth, + CustomerHierarchyIds, + CustomerRequestIdentity, + CustomerScopeAuth, + CustomerScopeKey, +) +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + INCREASE_REQUEST_ROUTE, + Budget, + CustomerMember, + Organization, +) + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + from litellm.router_api import RegistryReader + +CUSTOMER_MEMBER_ID_HEADER: Final = "x-litellm-customer-member-id" +CUSTOMER_ORG_ALIAS_HEADER: Final = "x-litellm-customer-org-alias" +CUSTOMER_TEAM_ID_HEADER: Final = "x-litellm-customer-team-id" +_ORGANIZATION_INCLUDE: Final = {"budget": True} +_TEAM_INCLUDE: Final = {"budget": True, "budget_override": True} +_MEMBER_INCLUDE: Final = {"budget": True, "budget_override": True, "parent_org": {"include": {"budget": True}}} +_EXPIRY: Final[TypeAdapter[datetime | None]] = TypeAdapter(datetime | None) +_TEAM_IDS: Final = TypeAdapter(tuple[str, ...]) +_SPEND: Final = TypeAdapter(float) + + +class _OrganizationRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + customer_org_id: str + alias: str | None = None + spend: float = 0.0 + budget: Mapping[str, object] | None = None + + +class _TeamRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + customer_team_id: str + alias: str | None = None + spend: float = 0.0 + budget: Mapping[str, object] | None = None + budget_override: Mapping[str, object] | None = None + budget_override_expires_at: datetime | None = None + + +class _MemberRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + customer_member_id: str + alias: str | None = None + spend: float = 0.0 + budget: Mapping[str, object] | None = None + budget_override: Mapping[str, object] | None = None + budget_override_expires_at: datetime | None = None + parent_org: _OrganizationRow + parent_team_ids: tuple[str, ...] = () + teams: tuple[_TeamRow, ...] = () + + +@dataclass(frozen=True, slots=True) +class AmbiguousOrganizationAlias: + """More than one customer organization carries the alias, so the request cannot be charged to one of them.""" + + alias: str + + +class CustomerBudgetExceededError(litellm.BudgetExceededError): + def __init__(self, hierarchy: CustomerHierarchyAuth, error: litellm.BudgetExceededError) -> None: + super().__init__( + current_cost=error.current_cost, + max_budget=error.max_budget, + message=error.message, + llm_provider=error.llm_provider, + entity_type=error.entity_type, + entity_id=error.entity_id, + ) + self.customer_hierarchy: Final = hierarchy + + +def customer_member_cache_key(customer_member_id: str) -> str: + return f"customer_member_id:{customer_member_id}" + + +def customer_org_alias_cache_key(customer_org_alias: str) -> str: + return f"customer_org_alias:{customer_org_alias}" + + +def _header_value(headers: Mapping[str, object] | None, header: str) -> str | None: + if headers is None: + return None + values: Final = tuple( + value.strip() + for name, value in headers.items() + if name.lower() == header and isinstance(value, str) and value.strip() + ) + return values[0] if values else None + + +def customer_member_id_from_headers(headers: Mapping[str, object] | None) -> str | None: + return _header_value(headers, CUSTOMER_MEMBER_ID_HEADER) + + +def customer_org_alias_from_headers(headers: Mapping[str, object] | None) -> str | None: + return _header_value(headers, CUSTOMER_ORG_ALIAS_HEADER) + + +def customer_team_id_from_headers(headers: Mapping[str, object] | None) -> str | None: + return _header_value(headers, CUSTOMER_TEAM_ID_HEADER) + + +def customer_identity_from_headers(headers: Mapping[str, object] | None) -> CustomerRequestIdentity | None: + """The customer the request named, or None when it carried neither customer header.""" + identity: Final = CustomerRequestIdentity( + customer_member_id=customer_member_id_from_headers(headers), + customer_org_alias=customer_org_alias_from_headers(headers), + customer_team_id=customer_team_id_from_headers(headers), + ) + if identity.customer_member_id is None and identity.customer_org_alias is None: + return None + return identity + + +def _scope( + key: CustomerScopeKey, entity_id: str, spend: float, budget: Budget | None, budget_reset_at: datetime | None = None +) -> CustomerScopeAuth: + if budget is None: + return CustomerScopeAuth(key=key, entity_id=entity_id, spend=spend, budget_reset_at=budget_reset_at) + return CustomerScopeAuth( + key=key, + entity_id=entity_id, + spend=spend, + budget_reset_at=budget_reset_at, + max_budget=budget.max_budget, + budget_limits=budget.budget_limits or (), + tpm_limit=budget.tpm_limit, + rpm_limit=budget.rpm_limit, + allowed_models=budget.allowed_models or None, + model_tpm_limit=budget.model_tpm_limit, + model_rpm_limit=budget.model_rpm_limit, + cap_policy=budget.cap_policy, + ) + + +def _team_scope(team: _TeamRow, now: datetime) -> CustomerScopeAuth: + budget: Final = related_budget(team.budget, now) + override: Final = related_budget(team.budget_override, now, budget.max_budget if budget is not None else None) + return _scope( + "customer_team", + team.customer_team_id, + team.spend, + effective_member_budget(budget, override, team.budget_override_expires_at, now), + customer_row_reset_at(team, now), + ).model_copy(update={"alias": team.alias}) + + +def customer_hierarchy_from_member_row(row: Mapping[str, object], now: datetime) -> CustomerHierarchyAuth: + """The org, team and member scopes of one member row loaded with ``_MEMBER_INCLUDE`` plus its ``teams`` rows. + The member's limits are its effective budget: the selected organization template under any unexpired override. + ``scopes`` charges the first team in ``parent_team_ids``; ``select_customer_team`` swaps in another.""" + member: Final = _MemberRow.model_validate(row) + organization: Final = member.parent_org + template: Final = related_budget(member.budget, now) + override: Final = related_budget(member.budget_override, now, template.max_budget if template is not None else None) + teams_by_id: Final = {team.customer_team_id: team for team in member.teams} + team_scopes: Final = tuple( + _team_scope(teams_by_id[team_id], now) for team_id in member.parent_team_ids if team_id in teams_by_id + ) + return CustomerHierarchyAuth( + customer_member_id=member.customer_member_id, + customer_member_alias=member.alias, + customer_team_id=team_scopes[0].entity_id if team_scopes else None, + customer_team_alias=team_scopes[0].alias if team_scopes else None, + customer_org_id=organization.customer_org_id, + customer_org_alias=organization.alias, + scopes=( + _scope( + "customer_org", + organization.customer_org_id, + organization.spend, + related_budget(organization.budget, now), + customer_row_reset_at(organization, now), + ), + *team_scopes[:1], + _scope( + "customer_member", + member.customer_member_id, + member.spend, + effective_member_budget(template, override, member.budget_override_expires_at, now), + customer_row_reset_at(member, now), + ), + ), + team_scopes=team_scopes, + ) + + +def select_customer_team(hierarchy: CustomerHierarchyAuth, customer_team_id: str) -> CustomerHierarchyAuth | None: + """The same hierarchy charged to ``customer_team_id``, or None when the member is not in that team.""" + selected: Final = next((scope for scope in hierarchy.team_scopes if scope.entity_id == customer_team_id), None) + if selected is None: + return None + return hierarchy.model_copy( + update={ + "customer_team_id": customer_team_id, + "customer_team_alias": selected.alias, + "scopes": tuple(selected if scope.key == "customer_team" else scope for scope in hierarchy.scopes), + } + ) + + +def customer_hierarchy_from_organization_row(row: Mapping[str, object], now: datetime) -> CustomerHierarchyAuth: + """The organization scope alone, for a request that named an organization alias and no member.""" + organization: Final = _OrganizationRow.model_validate(row) + return CustomerHierarchyAuth( + customer_org_id=organization.customer_org_id, + customer_org_alias=organization.alias, + scopes=( + _scope( + "customer_org", + organization.customer_org_id, + organization.spend, + related_budget(organization.budget, now), + customer_row_reset_at(organization, now), + ), + ), + ) + + +def _member_table(prisma_client: "PrismaClient") -> TableActions[PrismaRecord]: + return prisma_client.db.litellm_customermembertable # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportAny] # prisma tables are generated at build time + + +def _team_table(prisma_client: "PrismaClient") -> TableActions[PrismaRecord]: + return prisma_client.db.litellm_customerteamtable # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportAny] # generated client + + +def _organization_table(prisma_client: "PrismaClient") -> TableActions[PrismaRecord]: + return prisma_client.db.litellm_customerorganizationtable # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportAny] # prisma tables are generated at build time + + +@with_service_target(AUTH_OBJECTS_TARGET) +async def get_customer_hierarchy( + customer_member_id: str, + prisma_client: "PrismaClient", + user_api_key_cache: UserApiKeyCache, + now: datetime | None = None, +) -> CustomerHierarchyAuth | None: + cache_key: Final = customer_member_cache_key(customer_member_id) + cached: Final = await user_api_key_cache.async_get_cache(key=cache_key, model_type=CustomerHierarchyAuth) + if cached is not None: + return cached + lease: Final = await user_api_key_cache.lease(cache_key) + row: Final = await _member_table(prisma_client).find_unique( + where={"customer_member_id": customer_member_id}, include=_MEMBER_INCLUDE + ) + if row is None: + return None + loaded_at: Final = now if now is not None else utc_now() + team_ids: Final = _TEAM_IDS.validate_python(row.model_dump().get("parent_team_ids") or ()) + teams: Final = ( + await _team_table(prisma_client).find_many( + # bounded-ok: team_ids is one member's parent_team_ids, a handful of teams written by the management API + where={"customer_team_id": {"in": list(team_ids)}}, + include=_TEAM_INCLUDE, + ) + if team_ids + else () + ) + row_values: Final = {**row.model_dump(), "teams": tuple(team.model_dump() for team in teams)} + hierarchy: Final = customer_hierarchy_from_member_row(row_values, loaded_at) + _ = await user_api_key_cache.async_set_leased( + lease, hierarchy, _cache_ttl(row_values, loaded_at, get_management_object_ttl(user_api_key_cache)) + ) + return hierarchy + + +async def get_customer_scopes_for_accounting( + ids: CustomerHierarchyIds, + prisma_client: "PrismaClient", + user_api_key_cache: UserApiKeyCache, +) -> tuple[CustomerScopeAuth, ...]: + """Resolve policies for the stamped charge identities without running admission checks.""" + now: Final = utc_now() + hierarchy: Final = ( + await get_customer_hierarchy(ids.customer_member_id, prisma_client, user_api_key_cache, now) + if ids.customer_member_id is not None + else None + ) + known: Final = () if hierarchy is None else hierarchy.scopes + identities: Final[tuple[tuple[CustomerScopeKey, str | None], ...]] = ( + ("customer_org", ids.customer_org_id), + ("customer_team", ids.customer_team_id), + ("customer_member", ids.customer_member_id), + ) + scopes: Final = tuple( + [ + await _accounting_scope(key, entity_id, known, prisma_client, now) + for key, entity_id in identities + if entity_id is not None + ] + ) + return tuple(scope for scope in scopes if scope is not None) + + +async def _accounting_scope( + key: CustomerScopeKey, + entity_id: str, + known: tuple[CustomerScopeAuth, ...], + prisma_client: "PrismaClient", + now: datetime, +) -> CustomerScopeAuth | None: + cached: Final = next((scope for scope in known if scope.key == key and scope.entity_id == entity_id), None) + if cached is not None or key == "customer_member": + return cached + # Organization-only requests and a member moved since admission still charge + # the original team/organization, never the member's new parents. + table: Final[TableActions[PrismaRecord]] = ( + _organization_table(prisma_client) if key == "customer_org" else CustomerTeamRepository(prisma_client).table + ) + row: Final = await table.find_unique(where={f"{key}_id": entity_id}, include=_ORGANIZATION_INCLUDE) + if row is None: + return None + values: Final = row.model_dump() + return _scope( + key, entity_id, _SPEND.validate_python(values.get("spend", 0)), related_budget(values.get("budget"), now) + ) + + +@with_service_target(AUTH_OBJECTS_TARGET) +async def get_customer_organization_hierarchy( + customer_org_alias: str, + prisma_client: "PrismaClient", + user_api_key_cache: UserApiKeyCache, + now: datetime | None = None, +) -> CustomerHierarchyAuth | AmbiguousOrganizationAlias | None: + """The organization-only hierarchy behind an alias, None when no organization carries it.""" + cache_key: Final = customer_org_alias_cache_key(customer_org_alias) + cached: Final = await user_api_key_cache.async_get_cache(key=cache_key, model_type=CustomerHierarchyAuth) + if cached is not None: + return cached + lease: Final = await user_api_key_cache.lease(cache_key) + rows: Final = await _organization_table(prisma_client).find_many( + where={"alias": customer_org_alias}, include=_ORGANIZATION_INCLUDE, take=2 + ) + if not rows: + return None + if len(rows) > 1: + return AmbiguousOrganizationAlias(customer_org_alias) + hierarchy: Final = customer_hierarchy_from_organization_row( + rows[0].model_dump(), now if now is not None else utc_now() + ) + _ = await user_api_key_cache.async_set_leased(lease, hierarchy, get_management_object_ttl(user_api_key_cache)) + return hierarchy + + +def _cache_ttl(row: Mapping[str, object], now: datetime, default_ttl: float) -> float: + """A cached hierarchy may not outlive the member's or any of its teams' budget override, or an expired + override would keep granting its limits until the management-object TTL ran out.""" + member: Final = _MemberRow.model_validate(row) + expiries: Final = tuple( + expires_at + for expires_at in ( + member.budget_override_expires_at, + *(team.budget_override_expires_at for team in member.teams), + ) + if expires_at is not None and expires_at > now + ) + if not expiries: + return default_ttl + return max(1.0, min(default_ttl, (min(expiries) - now).total_seconds())) + + +def cap_policy_hint(hierarchy: CustomerHierarchyAuth, key: str, entity_id: str) -> str: + """What the paused caller can do, from the budget of the level that blocked it.""" + policy: Final = next( + (scope.cap_policy for scope in hierarchy.scopes if scope.key == key and scope.entity_id == entity_id), None + ) + if policy == "pause_allow_increase": + return f" Request a temporary increase through POST {INCREASE_REQUEST_ROUTE}." + return " Spend resumes when the budget window resets." + + +def check_customer_hierarchy_budget(hierarchy: CustomerHierarchyAuth) -> None: + """Raises ``BudgetExceededError`` for the first level, org first, whose spend has reached its ``max_budget``.""" + for scope in hierarchy.scopes: + if scope.max_budget is not None and scope.spend >= scope.max_budget: + raise litellm.BudgetExceededError( + current_cost=scope.spend, + max_budget=scope.max_budget, + message=( + f"ExceededBudget: {scope.key}={scope.entity_id} over budget. " + f"Spend={scope.spend}, Budget={scope.max_budget}." + f"{cap_policy_hint(hierarchy, scope.key, scope.entity_id)}" + ), + entity_type=scope.key, + entity_id=scope.entity_id, + ) + + +async def check_customer_hierarchy_windows( + hierarchy: CustomerHierarchyAuth, + now: datetime | None = None, +) -> None: + from itertools import chain + + from litellm.proxy.proxy_server import get_current_spend, spend_counter_store + from litellm.proxy.spend_tracking.budget_reservation import customer_window_counters + from litellm.proxy.spend_tracking.spend_counter_batch import spend_counter_batch_scope + + counters: Final = tuple( + chain.from_iterable( + customer_window_counters(scope.key, scope.entity_id, scope.budget_limits, now=now) + for scope in hierarchy.scopes + ) + ) + with spend_counter_batch_scope( + spend_counter_store, counter_keys=frozenset(counter.counter_key for counter in counters) + ): + for counter in counters: + if counter.window_start is None: + raise ProxyException( + message="Customer budget window is unavailable.", + type=ProxyErrorTypes.internal_server_error, + param="budget_limits", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + try: + spend = await get_current_spend( + counter_key=counter.counter_key, + fallback_spend=0.0, + max_budget=counter.max_budget, + window_entity_type=counter.entity_type, + window_entity_id=counter.entity_id, + window_duration=counter.window_duration, + window_start=counter.window_start, + require_authoritative_window=True, + ) + except HTTPException as error: + raise ProxyException( + message="Customer budget window spend is unavailable.", + type=ProxyErrorTypes.internal_server_error, + param="budget_limits", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) from error + if spend >= counter.max_budget: + raise litellm.BudgetExceededError( + current_cost=spend, + max_budget=counter.max_budget, + message=( + f"ExceededBudget: {counter.entity_type}={counter.entity_id} over {counter.window_duration} budget." + f"{cap_policy_hint(hierarchy, counter.entity_type, counter.entity_id)}" + ), + entity_type=counter.entity_type, + entity_id=counter.entity_id, + ) + + +def check_customer_hierarchy_model_access( + hierarchy: CustomerHierarchyAuth, model: str, llm_router: "RegistryReader | None" +) -> None: + """Every level with an ``allowed_models`` list must permit ``model``; a level without one permits every model.""" + for scope in hierarchy.scopes: + if scope.allowed_models: + can_object_call_model( + model=model, llm_router=llm_router, models=list(scope.allowed_models), object_type="customer" + ) + + +def _untracked(message: str, header: str) -> None: + """An id the proxy does not know is logged and the request runs without that customer level, never refused: + the headers are attribution, and a customer whose rows are not set up yet must keep working.""" + verbose_proxy_logger.warning("Customer %s not tracked, %s", header, message) + + +OrganizationRegistrar: TypeAlias = Callable[[str, "PrismaClient"], Awaitable[Organization | Failure]] + + +def _organization_from_lookup( + customer_org_alias: str, organization: CustomerHierarchyAuth | AmbiguousOrganizationAlias | None +) -> CustomerHierarchyAuth | None: + if organization is None: + return _untracked(f"organization alias not found: {customer_org_alias}", CUSTOMER_ORG_ALIAS_HEADER) + if isinstance(organization, AmbiguousOrganizationAlias): + raise ProxyException( + message=f"Customer organization alias is not unique: {customer_org_alias}", + type=ProxyErrorTypes.bad_request_error, + param=CUSTOMER_ORG_ALIAS_HEADER, + code=status.HTTP_400_BAD_REQUEST, + ) + return organization + + +async def _register_organization( + customer_org_alias: str, + prisma_client: "PrismaClient", + user_api_key_cache: UserApiKeyCache, + register: OrganizationRegistrar, + key_metadata: Mapping[str, object], +) -> CustomerHierarchyAuth | None: + """An alias the proxy has never seen becomes an organization with that id and alias, so a customer's first + request is charged to it instead of running untracked. A key pinned to another organization is refused first.""" + if not _registrable(customer_org_alias): + verbose_proxy_logger.warning( + "Could not register customer organization: alias is not a valid customer identifier" + ) + return _untracked(f"organization alias not found: {customer_org_alias}", CUSTOMER_ORG_ALIAS_HEADER) + _check_key_scope(customer_org_alias, key_metadata) + registered: Final = await register(customer_org_alias, prisma_client) + if isinstance(registered, Failure): + if registered.code is ErrorCode.FREE_TIER_LIMIT: + verbose_proxy_logger.warning( + "CUSTOMER TRACKING OFF for organization %s because the free customer portal limit was reached. " + "Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise", + customer_org_alias, + ) + return None + verbose_proxy_logger.warning( + "Could not register customer organization %s: %s", customer_org_alias, registered.code.value + ) + return _untracked(f"organization alias not found: {customer_org_alias}", CUSTOMER_ORG_ALIAS_HEADER) + return _organization_from_lookup( + customer_org_alias, + await get_customer_organization_hierarchy(customer_org_alias, prisma_client, user_api_key_cache), + ) + + +async def _organization_named_by_alias( + customer_org_alias: str, + prisma_client: "PrismaClient | None", + user_api_key_cache: UserApiKeyCache, + register: OrganizationRegistrar, + key_metadata: Mapping[str, object], +) -> CustomerHierarchyAuth | None: + if prisma_client is None: + return _untracked(f"organization alias not found: {customer_org_alias}", CUSTOMER_ORG_ALIAS_HEADER) + found: Final = await get_customer_organization_hierarchy(customer_org_alias, prisma_client, user_api_key_cache) + if found is None: + return await _register_organization( + customer_org_alias, prisma_client, user_api_key_cache, register, key_metadata + ) + return _organization_from_lookup(customer_org_alias, found) + + +MemberRegistrar: TypeAlias = Callable[[str, str, "PrismaClient"], Awaitable[CustomerMember | Failure]] +_MEMBER_ID: Final[TypeAdapter[str]] = TypeAdapter(CustomerIdentifier) + + +def _registrable(customer_member_id: str) -> bool: + """A header value becomes a member row only within the identifier bound the customer tables share, so the spend + ledger can index every row the proxy creates.""" + try: + _MEMBER_ID.validate_python(customer_member_id) + except ValidationError: + return False + return True + + +async def _register_member( + customer_member_id: str, + organization: CustomerHierarchyAuth | None, + prisma_client: "PrismaClient", + user_api_key_cache: UserApiKeyCache, + register: MemberRegistrar, +) -> CustomerHierarchyAuth | None: + """A member the request named for the first time joins the aliased organization's default team. Without an + alias there is no organization to place it in, and the member stays unknown.""" + if organization is None: + return None + if not _registrable(customer_member_id): + verbose_proxy_logger.warning( + "Could not register customer member in organization %s: id is not a valid customer identifier", + organization.customer_org_id, + ) + return None + registered: Final = await register(organization.customer_org_id, customer_member_id, prisma_client) + if isinstance(registered, Failure): + if registered.code is ErrorCode.FREE_TIER_LIMIT: + verbose_proxy_logger.warning( + "CUSTOMER TRACKING OFF for member %s because the free customer portal limit was reached. " + "Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise", + customer_member_id, + ) + return None + verbose_proxy_logger.warning( + "Could not register customer member %s in organization %s: %s", + customer_member_id, + organization.customer_org_id, + registered.code.value, + ) + return None + return await get_customer_hierarchy(customer_member_id, prisma_client, user_api_key_cache) + + +async def _hierarchy_named_by_member( + customer_member_id: str, + organization: CustomerHierarchyAuth | None, + prisma_client: "PrismaClient | None", + user_api_key_cache: UserApiKeyCache, + register: MemberRegistrar, +) -> CustomerHierarchyAuth | None: + """The member's hierarchy, or the aliased organization alone when the member is unknown and cannot be + registered, or None when neither is known.""" + if prisma_client is None: + return _untracked(f"member not found: {customer_member_id}", CUSTOMER_MEMBER_ID_HEADER) + known: Final = await get_customer_hierarchy(customer_member_id, prisma_client, user_api_key_cache) + hierarchy: Final = ( + known + if known is not None + else await _register_member(customer_member_id, organization, prisma_client, user_api_key_cache, register) + ) + if hierarchy is None: + _untracked(f"member not found: {customer_member_id}", CUSTOMER_MEMBER_ID_HEADER) + return organization + return hierarchy + + +async def load_customer_hierarchy( + identity: CustomerRequestIdentity, + prisma_client: "PrismaClient | None", + user_api_key_cache: UserApiKeyCache, + register: MemberRegistrar = ensure_customer_member, + key_metadata: Mapping[str, object] = MappingProxyType({}), + register_organization: OrganizationRegistrar = ensure_customer_organization, +) -> CustomerHierarchyAuth | None: + """The hierarchy the headers name, or None when the proxy knows none of the ids, in which case the request runs + untracked. An alias the proxy has never seen is registered as a new organization first. A member header next to an alias header must name a member of that organization; a member the + organization has never seen is registered in its default team first, and when that fails the organization + alone is charged. A key pinned to another organization is refused before any row is created.""" + organization: Final = ( + await _organization_named_by_alias( + identity.customer_org_alias, prisma_client, user_api_key_cache, register_organization, key_metadata + ) + if identity.customer_org_alias is not None + else None + ) + if organization is not None: + check_customer_hierarchy_key_scope(organization, key_metadata) + if identity.customer_member_id is None: + if identity.customer_team_id is not None: + raise _forbidden("a customer team header needs a customer member header", CUSTOMER_TEAM_ID_HEADER) + return organization + hierarchy: Final = await _hierarchy_named_by_member( + identity.customer_member_id, organization, prisma_client, user_api_key_cache, register + ) + if hierarchy is None: + return None + if organization is not None and organization.customer_org_id != hierarchy.customer_org_id: + raise _forbidden( + f"Customer member {identity.customer_member_id} does not belong to the organization " + f"aliased {identity.customer_org_alias}", + CUSTOMER_ORG_ALIAS_HEADER, + ) + if identity.customer_team_id is None or hierarchy.customer_member_id is None: + return hierarchy + selected: Final = select_customer_team(hierarchy, identity.customer_team_id) + if selected is None: + raise _forbidden( + f"Customer member {identity.customer_member_id} is not in customer team {identity.customer_team_id}", + CUSTOMER_TEAM_ID_HEADER, + ) + return selected + + +def _forbidden(message: str, param: str) -> ProxyException: + return ProxyException(message=message, type=ProxyErrorTypes.auth_error, param=param, code=status.HTTP_403_FORBIDDEN) + + +async def resolve_customer_hierarchy( + identity: CustomerRequestIdentity, + prisma_client: "PrismaClient | None", + user_api_key_cache: UserApiKeyCache, + llm_router: "RegistryReader | None", + request_model: str | Sequence[str] | None, + route: str, + key_metadata: Mapping[str, object] | None = None, + register: MemberRegistrar = ensure_customer_member, + register_organization: OrganizationRegistrar = ensure_customer_organization, +) -> CustomerHierarchyAuth | None: + """Load the hierarchy the headers name and admit the request against it, or None when no id is known. + ``request_model`` is the model the route will actually serve, which may differ from the body.""" + from litellm.proxy.proxy_server import get_current_spend + + metadata: Final = key_metadata or {} + cached: Final = await load_customer_hierarchy( + identity, prisma_client, user_api_key_cache, register, metadata, register_organization + ) + if cached is None: + return None + hierarchy: Final = await current_customer_spend(cached, get_current_spend) + try: + await check_customer_hierarchy_access(hierarchy, llm_router, request_model, route, key_metadata) + except litellm.BudgetExceededError as error: + raise CustomerBudgetExceededError(hierarchy, error) from error + return hierarchy + + +async def check_customer_hierarchy_access( + hierarchy: CustomerHierarchyAuth, + llm_router: "RegistryReader | None", + request_model: str | Sequence[str] | None, + route: str, + key_metadata: Mapping[str, object] | None = None, +) -> None: + check_customer_hierarchy_key_scope(hierarchy, key_metadata or {}) + requested: Final = (request_model,) if isinstance(request_model, str) else tuple(request_model or ()) + for model in requested: + if model: + check_customer_hierarchy_model_access(hierarchy, model, llm_router) + if RouteChecks.is_info_route(route): + return + check_customer_hierarchy_budget(hierarchy) + if any(scope.budget_limits for scope in hierarchy.scopes): + await check_customer_hierarchy_windows(hierarchy) + + +def check_customer_hierarchy_key_scope(hierarchy: CustomerHierarchyAuth, key_metadata: Mapping[str, object]) -> None: + """A key whose metadata pins ``customer_org_id`` may only act for members of that organization.""" + _check_key_scope(hierarchy.customer_org_id, key_metadata) + + +def _check_key_scope(customer_org_id: str, key_metadata: Mapping[str, object]) -> None: + key_customer_org_id: Final = key_metadata.get("customer_org_id") + if key_customer_org_id is None or key_customer_org_id == customer_org_id: + return + raise ProxyException( + message="API key is not allowed to act for this customer member's organization", + type=ProxyErrorTypes.auth_error, + param=CUSTOMER_MEMBER_ID_HEADER, + code=status.HTTP_403_FORBIDDEN, + ) + + +@with_service_target(AUTH_OBJECTS_TARGET) +async def forget_customer_hierarchies(customer_member_ids: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None: + """Drop the cached hierarchies of members whose own, team, org or template budget changed so auth reloads them.""" + await evict_and_broadcast(tuple(customer_member_cache_key(m) for m in customer_member_ids), user_api_key_cache) + + +@with_service_target(AUTH_OBJECTS_TARGET) +async def forget_customer_organization_aliases( + aliases: Sequence[str | None], user_api_key_cache: UserApiKeyCache +) -> None: + """Drop the cached organization hierarchies behind aliases whose organization changed so request auth reloads them.""" + keys: Final = tuple(customer_org_alias_cache_key(alias) for alias in dict.fromkeys(aliases) if alias) + if keys: + await evict_and_broadcast(keys, user_api_key_cache) diff --git a/litellm/proxy/auth/customer_member_auto_create.py b/litellm/proxy/auth/customer_member_auto_create.py new file mode 100644 index 00000000000..ee176bb0cf7 --- /dev/null +++ b/litellm/proxy/auth/customer_member_auto_create.py @@ -0,0 +1,93 @@ +"""Registration of a customer member the request named but the proxy has never seen. + +Customers send a shared service key with ``x-litellm-customer-org-alias`` and ``x-litellm-customer-member-id`` +on every call, and their users appear for the first time on their first request. The member is created under +the organization's ``default`` team, so it inherits the organization's default budget. In an organization without +a default budget the member is registered with no budget: its spend is tracked and nothing caps it until an admin +assigns one. +""" + +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.management_helpers.customer_hierarchy import CustomerHierarchy, ErrorCode, Failure +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + CustomerMember, + MemberCreate, + Organization, + OrganizationCreate, + Team, + TeamCreate, +) + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +AUTO_CREATE_ACTOR: Final = "x-litellm-customer-member-id" +AUTO_CREATE_ORG_ACTOR: Final = "x-litellm-customer-org-alias" +DEFAULT_TEAM_ALIAS: Final = "default" + + +def default_team_id(customer_org_id: str) -> str: + return f"{customer_org_id}:{DEFAULT_TEAM_ALIAS}" + + +async def _default_team(hierarchy: CustomerHierarchy, customer_org_id: str) -> Team | Failure: + team_id: Final = default_team_id(customer_org_id) + existing: Final = await hierarchy.get_team(team_id) + if not isinstance(existing, Failure): + return existing + created: Final = await hierarchy.create_team( + TeamCreate(customer_team_id=team_id, alias=DEFAULT_TEAM_ALIAS, parent_org_id=customer_org_id), + AUTO_CREATE_ACTOR, + ) + if isinstance(created, Failure) and created.code is ErrorCode.ALREADY_EXISTS: + return await hierarchy.get_team(team_id) + return created + + +async def ensure_customer_member( + customer_org_id: str, customer_member_id: str, prisma_client: "PrismaClient" +) -> CustomerMember | Failure: + """The member row for ``customer_member_id`` in ``customer_org_id``'s default team, created when absent. + Two workers registering the same member at once both end up with the one row.""" + from litellm.proxy.proxy_server import premium_user + + hierarchy: Final = CustomerHierarchy(prisma_client, premium_user=premium_user) + team: Final = await _default_team(hierarchy, customer_org_id) + if isinstance(team, Failure): + return team + created: Final = await hierarchy.create_member( + MemberCreate(customer_member_id=customer_member_id, parent_team_ids=(team.customer_team_id,)), + AUTO_CREATE_ACTOR, + budget_required=False, + ) + if isinstance(created, Failure) and created.code is ErrorCode.ALREADY_EXISTS: + return await hierarchy.get_member(customer_member_id) + if not isinstance(created, Failure): + verbose_proxy_logger.info( + "Registered customer member %s in team %s of organization %s", + customer_member_id, + team.customer_team_id, + customer_org_id, + ) + return created + + +async def ensure_customer_organization( + customer_org_alias: str, prisma_client: "PrismaClient" +) -> Organization | Failure: + """The organization row carrying ``customer_org_alias``, created with the alias as its id when absent, so a + customer's first request is charged to it. Two workers registering the same alias at once both end up with + the one row.""" + from litellm.proxy.proxy_server import premium_user + + hierarchy: Final = CustomerHierarchy(prisma_client, premium_user=premium_user) + created: Final = await hierarchy.create_organization( + OrganizationCreate(customer_org_id=customer_org_alias, alias=customer_org_alias), AUTO_CREATE_ORG_ACTOR + ) + if isinstance(created, Failure) and created.code in (ErrorCode.ALREADY_EXISTS, ErrorCode.ALIAS_ALREADY_EXISTS): + return await hierarchy.get_organization(customer_org_alias) + if not isinstance(created, Failure): + verbose_proxy_logger.info("Registered customer organization %s", customer_org_alias) + return created diff --git a/litellm/proxy/auth/customer_spend.py b/litellm/proxy/auth/customer_spend.py new file mode 100644 index 00000000000..0c28a9412e5 --- /dev/null +++ b/litellm/proxy/auth/customer_spend.py @@ -0,0 +1,53 @@ +import asyncio +from collections.abc import Awaitable, Callable, Mapping +from typing import Final + +from pydantic import ValidationError + +from litellm.redis.stores.spend_counters import SpendCounterStore +from litellm.types.proxy.auth.customer_hierarchy import CustomerHierarchyAuth, CustomerScopeAuth + + +def customer_spend_counter(scope: CustomerScopeAuth) -> str: + period: Final = scope.budget_reset_at.isoformat() if scope.budget_reset_at is not None else "lifetime" + return f"spend:{scope.key}:{scope.entity_id}:{period}" + + +async def _current_scope( + scope: CustomerScopeAuth, current: float | None, store: SpendCounterStore +) -> CustomerScopeAuth: + spend: Final = ( + await store.seed(customer_spend_counter(scope), scope.spend, increment_if_lost=False) + if current is None + else current + ) + return scope.model_copy(update={"spend": max(scope.spend, spend)}) + + +async def current_customer_spend( + hierarchy: CustomerHierarchyAuth, + read_spend: Callable[[str, float, float | None], Awaitable[float]], +) -> CustomerHierarchyAuth: + readings: Final = await asyncio.gather( + *(read_spend(customer_spend_counter(scope), scope.spend, scope.max_budget) for scope in hierarchy.scopes) + ) + scopes: Final = tuple( + scope.model_copy(update={"spend": max(scope.spend, spend)}) + for scope, spend in zip(hierarchy.scopes, readings, strict=True) + ) + return hierarchy.model_copy(update={"scopes": tuple(scopes)}) + + +async def settle_customer_spend(metadata: Mapping[str, object], cost: float, store: SpendCounterStore) -> None: + snapshot: Final = metadata.get("user_api_key_customer_hierarchy") + if snapshot is None or cost == 0: + return + try: + hierarchy: Final = CustomerHierarchyAuth.model_validate(snapshot) + except ValidationError: + return + readings: Final = await asyncio.gather(*(store.read(customer_spend_counter(scope)) for scope in hierarchy.scopes)) + _ = await asyncio.gather( + *(_current_scope(scope, reading[0], store) for scope, reading in zip(hierarchy.scopes, readings, strict=True)) + ) + _ = await store.increment_many(tuple((customer_spend_counter(scope), cost) for scope in hierarchy.scopes)) diff --git a/litellm/proxy/auth/fallback_model_access.py b/litellm/proxy/auth/fallback_model_access.py index b550d4d6866..6be863b24a2 100644 --- a/litellm/proxy/auth/fallback_model_access.py +++ b/litellm/proxy/auth/fallback_model_access.py @@ -16,8 +16,14 @@ from typing import Final from pydantic import ValidationError from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import ProxyException, UserAPIKeyAuth -from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool +from litellm.proxy._types import UI_TEAM_ID, ModelAccessDeniedProxyException, ProxyException, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import ( + can_key_call_resolved_model, + can_token_call_search_tool, + check_customer_model_access, + dashboard_model_billing_auth, +) +from litellm.proxy.litellm_pre_call_utils import global_then_key_alias_target from litellm.router_api import RegistryReader, RoutingContext from litellm.search import asearch from litellm.types.llms.base import LiteLLMBaseModel @@ -88,6 +94,26 @@ def _enforced_by_general_settings() -> bool: return _FallbackAccessSettings.model_validate(general_settings).enforce_fallback_model_access +def _dashboard_model_group(model: str, auth: UserAPIKeyAuth, llm_router: RegistryReader) -> str: + aliased: Final = global_then_key_alias_target(model, auth) + return llm_router.registry().resolve(aliased, kind="alias") or aliased + + +async def dashboard_fallback_billing_allowed(model: str, auth: UserAPIKeyAuth, llm_router: RegistryReader) -> bool: + if auth.team_id != UI_TEAM_ID and not isinstance(auth.dashboard_session_origin, UserAPIKeyAuth): + return True + original: Final = auth.dashboard_session_model + if original is not None and _dashboard_model_group(original, auth, llm_router) == _dashboard_model_group( + model, auth, llm_router + ): + return True + try: + billed: Final = await dashboard_model_billing_auth(auth, model, llm_router) + except ModelAccessDeniedProxyException: + return False + return billed.team_id == auth.team_id + + @dataclass(frozen=True, slots=True) class RouterFallbackAccessCheck: """ @@ -104,9 +130,15 @@ class RouterFallbackAccessCheck: return True if _is_search_request(request_kwargs): return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token) - if not self.is_enforced(): - return True - return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=ctx) + enforced: Final = self.is_enforced() + if enforced and not await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=ctx): + return False + if not enforced: + try: + check_customer_model_access(model=model, valid_token=valid_token, llm_router=ctx) + except ProxyException: + return False + return await dashboard_fallback_billing_allowed(model, valid_token, ctx) router_fallback_access_check: Final = RouterFallbackAccessCheck(is_enforced=_enforced_by_general_settings) diff --git a/litellm/proxy/auth/fallback_rate_limit.py b/litellm/proxy/auth/fallback_rate_limit.py index 1d4a33b2994..d2f6cf9498a 100644 --- a/litellm/proxy/auth/fallback_rate_limit.py +++ b/litellm/proxy/auth/fallback_rate_limit.py @@ -13,10 +13,12 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Final +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.fallback_budget import ( _user_api_key_auth_from_request, # pyright: ignore[reportPrivateUsage] # shared auth lookup for fallback checks ) +from litellm.proxy.hooks.dynamic_rate_limiter_v3 import PROXY_DynamicRateLimitHandlerV3 from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage] # the proxy's limiter type ) @@ -42,7 +44,18 @@ class RouterFallbackRateLimitCheck: if valid_token is None: return True try: - return await limiter.async_fallback_target_allowed(valid_token, model, request_kwargs) + dynamic_limiter: Final = next( + (cb for cb in litellm.callbacks if isinstance(cb, PROXY_DynamicRateLimitHandlerV3)), None + ) + if dynamic_limiter is not None and not await dynamic_limiter.async_fallback_target_allowed( + valid_token, model, request_kwargs, commit=False + ): + return False + if not await limiter.async_fallback_target_allowed(valid_token, model, request_kwargs): + return False + return dynamic_limiter is None or await dynamic_limiter.async_fallback_target_allowed( + valid_token, model, request_kwargs + ) except Exception as e: # noqa: BLE001 # fail closed: a limiter failure must not hand out unchecked capacity verbose_proxy_logger.warning("Skipping fallback to model=%s: rate limit check failed: %s", model, e) return False diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 7d413121b56..16b59657d80 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -354,6 +354,7 @@ async def _sign_in( response = await generate_key_helper_fn( llm_router=None, request_type="key", + actor=None, **{ "user_role": LitellmUserRoles.PROXY_ADMIN, "duration": LITELLM_UI_SESSION_DURATION, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index ff6e7f84bed..a08dacae44b 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -18,11 +18,13 @@ from functools import partial from types import MappingProxyType from typing import Final, NamedTuple, Protocol, Union, cast +import anyio import fastapi from fastapi import HTTPException, Request, WebSocket, status from fastapi.security.api_key import APIKeyHeader -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from starlette.exceptions import WebSocketException +from starlette.requests import ClientDisconnect import litellm from litellm._internal_context import service_target, set_request_parent_otel_span @@ -43,6 +45,7 @@ from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias from litellm.models.end_user import LiteLLM_EndUserTable +from litellm.models.team import LiteLLM_TeamRebuiltFromToken from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers from litellm.proxy.auth.access_group_rules import access_group_ids_for_identity @@ -127,6 +130,11 @@ from litellm.proxy.auth.auth_utils import ( request_dispatched_to_provider_pass_through, route_in_additonal_public_routes, ) +from litellm.proxy.auth.customer_hierarchy_auth import ( + CustomerBudgetExceededError, + customer_identity_from_headers, + resolve_customer_hierarchy, +) from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler from litellm.proxy.auth.network import TrustedProxyConfig, resolve_network_context from litellm.proxy.auth.oauth2_check import Oauth2Handler @@ -243,6 +251,13 @@ def _route_requires_auth_despite_public(route: str, general_settings: dict | Non return False +def _is_unauthenticated_public_route(route: str) -> bool: + """A route the builder serves without credentials, so nothing on the request may create or charge anything.""" + return not _route_requires_auth_despite_public(route=route, general_settings=None) and ( + route in LiteLLMRoutes.public_routes.value or route_in_additonal_public_routes(current_route=route) + ) + + custom_litellm_key_header: Final = APIKeyHeader( name=SpecialHeaders.custom_litellm_api_key.value, auto_error=False, @@ -298,6 +313,7 @@ _CLAUDE_MODEL_ROUTES: Final = frozenset( ) _CLAUDE_MODEL_NORMALIZED: Final = "litellm.claude_model_normalized" _GENERAL_SETTINGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_KEY_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object]) async def _normalize_claude_model( @@ -2232,6 +2248,162 @@ async def _read_request_body_deferring_parse_failure( return populate_request_with_path_params(request_data=parsed_body, request=request), None +PREFLIGHT_FAILURE_JSON_BODY_MAX_BYTES: Final = 64 * 1024 +PREFLIGHT_FAILURE_JSON_BODY_TIMEOUT_SECONDS: Final = 1.0 + + +def _preflight_failure_model( + body: Mapping[str, object], route: str, request: Request, team_id: str | None +) -> str | Sequence[str] | None: + from litellm.proxy.proxy_server import llm_router + + try: + return _get_model_from_request_context(dict(body), route, request, llm_router, team_id) + except Exception: # noqa: BLE001 # logging enrichment must never replace the existing preflight rejection + return None + + +async def _preflight_failure_request_data( + request: Request, route: str, cached_key: UserAPIKeyAuth +) -> Mapping[str, object]: + from litellm.proxy.common_request_processing import resolve_litellm_call_id + from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields + from litellm.proxy.proxy_server import proxy_config + from litellm.proxy.utils import ( + _call_type_for_route, # pyright: ignore[reportPrivateUsage] # share proxy failure route classification + ) + + body: Final = await _small_preflight_failure_json(request) + model: Final = _preflight_failure_model(body, route, request, cached_key.team_id) + call_type: Final = _call_type_for_route(route, method=request.method) + _seed_request_destinations(cached_key, request) + seed_request_identity(cached_key, model=model, request_data=body) + return { + **{key: value for key, value in body.items() if key != "model"}, + **({"model": model} if model is not None else {}), + "litellm_call_id": resolve_litellm_call_id(request.headers.get("x-litellm-call-id")), + "call_type": call_type, + **key_and_team_logging_fields(user_api_key_dict=cached_key, proxy_config=proxy_config), + } + + +async def _small_preflight_failure_json(request: Request) -> Mapping[str, object]: + declared: Final = request.headers.get("content-length", "") + content_type: Final = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if ( + not declared.isdecimal() + or len(declared) > 10 + or not 0 < int(declared) <= PREFLIGHT_FAILURE_JSON_BODY_MAX_BYTES + or request.headers.get("transfer-encoding") is not None + or content_type != "application/json" + ): + return {} + buffered: Final = bytearray() + try: + with anyio.fail_after(PREFLIGHT_FAILURE_JSON_BODY_TIMEOUT_SECONDS): + async for chunk in request.stream(): + if len(buffered) + len(chunk) > PREFLIGHT_FAILURE_JSON_BODY_MAX_BYTES: + return {} + buffered.extend(chunk) + parsed: Final = _GENERAL_SETTINGS_ADAPTER.validate_json(buffered) + except (ValidationError, RuntimeError, ClientDisconnect, TimeoutError): + return {} + return { + **({"model": parsed["model"]} if isinstance(parsed.get("model"), str) else {}), + **{ + slot: { + name: value + for name, value in _GENERAL_SETTINGS_ADAPTER.validate_python(metadata).items() + if name in ("tags", "session_id", "trace_user_id", "trace_id", "trace_name") + } + for slot in ("metadata", "litellm_metadata") + if isinstance(metadata := parsed.get(slot), dict) + }, + } + + +async def _reject_from_headers_before_reading_body( + request: Request, api_key: str | None, custom_litellm_key_header: str | None, route: str +) -> None: + """Refuse a request its headers already condemn, a blocked key or an exhausted key limit, before reading its body. + + Only a key already in the cache is considered, so a miss changes nothing and the full auth path still runs. + A retry flood of large payloads against a limited key used to buffer every payload just to answer 429. + """ + if not RouteChecks.is_llm_api_route(route=route): + return + from litellm.proxy import proxy_server + from litellm.proxy.common_request_processing import proxy_exception_from_http_exception + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + configured_header: Final = proxy_server.general_settings_view().get("litellm_key_header_name") + bearer_token: Final = ( + get_api_key_from_custom_header(request=request, custom_litellm_key_header_name=configured_header) + if isinstance(configured_header, str) and configured_header in request.headers + else _get_bearer_token_or_received_api_key(custom_litellm_key_header) + if isinstance(custom_litellm_key_header, str) + else get_bearer_token(api_key=api_key) + if isinstance(api_key, str) + else "" + ) + if not bearer_token: + return + hashed_token: Final = hash_token(bearer_token) + try: + cached_key: Final = IdentityStore.key_from_principal( + await IdentityStore(prisma_client, user_api_key_cache, check_cache_only=True).resolve( + hashed_token=hashed_token + ) + ) + except Exception: # noqa: BLE001 # any cache miss defers to the full auth path below + return + parent_otel_span: Final[Span | None] = getattr(request.state, "parent_otel_span", None) + if cached_key.blocked is True: + request_data: Final = dict(await _preflight_failure_request_data(request, route, cached_key)) + await UserAPIKeyAuthExceptionHandler.handle_authentication_error( + e=ProxyException( + message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", + type=ProxyErrorTypes.auth_error, + param=None, + code=status.HTTP_401_UNAUTHORIZED, + ), + request=request, + request_data=request_data, + route=route, + parent_otel_span=parent_otel_span, + api_key=bearer_token, + resolved_identity=cached_key, + dispatched_call_type=str(request_data["call_type"]) if request_data["call_type"] is not None else None, + ) + return + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3): + return + rate_limited: Final = cached_key.model_copy( + update={ + "api_key": cached_key.api_key or hashed_token, + "parent_otel_span": parent_otel_span, + "request_route": normalize_request_route(route), + } + ) + try: + await limiter.raise_if_over_limit_before_body(rate_limited) + except HTTPException as rejection: + failure_data: Final = dict(await _preflight_failure_request_data(request, route, rate_limited)) + transformed: Final = await proxy_logging_obj.post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # bare dict in sig + request_data=failure_data, + original_exception=rejection, + user_api_key_dict=rate_limited, + route=route, + dispatched_call_type=str(failure_data["call_type"]) if failure_data["call_type"] is not None else None, + ) + failure: Final = transformed if transformed is not None else rejection + if isinstance(failure, ProxyException): + raise failure + raise proxy_exception_from_http_exception(failure, dict(failure.headers or {})) + + async def _record_unparsable_body_failure( user_api_key_dict: UserAPIKeyAuth, body_parse_exception: ProxyException, @@ -2448,9 +2620,7 @@ async def user_api_key_auth_builder( """ ######## Route Checks Before Reading DB / Cache for "token" ################ - if not _route_requires_auth_despite_public(route=route, general_settings=general_settings) and ( - route in LiteLLMRoutes.public_routes.value or route_in_additonal_public_routes(current_route=route) - ): + if _is_unauthenticated_public_route(route): # check if public endpoint return UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY) @@ -3536,13 +3706,13 @@ async def _safe_fetch(label: str, awaitable): return None -def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCachedObj: +def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamRebuiltFromToken: """Reconstruct a cached team object from the fields already on the UserAPIKeyAuth. Only called when valid_token.team_id is known to be non-None (the caller gates on it).""" assert valid_token.team_id is not None token_team_models: Final = _token_team_models(valid_token) - return LiteLLM_TeamTableCachedObj( + return LiteLLM_TeamRebuiltFromToken( team_id=valid_token.team_id, max_budget=valid_token.team_max_budget, soft_budget=valid_token.team_soft_budget, @@ -4008,7 +4178,11 @@ async def run_centralized_common_checks( user_api_key_auth_obj.project_metadata = project_object.metadata user_api_key_auth_obj.project_alias = project_object.project_alias - if end_user_id and key_end_user_budget_id is not None and prisma_client is not None: + if ( + end_user_id + and (key_end_user_budget_id is not None or user_api_key_auth_obj.jwt_claims is not None) + and prisma_client is not None + ): await _apply_key_end_user_default_budget_to_token( valid_token=user_api_key_auth_obj, end_user_object=end_user_object, @@ -4196,7 +4370,7 @@ async def _noop_none() -> None: async def _apply_key_end_user_default_budget_to_token( valid_token: UserAPIKeyAuth, end_user_object: LiteLLM_EndUserTable | None, - key_end_user_budget_id: str, + key_end_user_budget_id: str | None, prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None, @@ -4344,8 +4518,8 @@ async def _authorize_authenticated_request( ) -> UserAPIKeyAuth | None: """Authorize an already-authenticated request: disabled-route check, the single ``common_checks`` gate (which also reserves budget), and end-user fallback - resolution. Returns the auth object the exception handler recovered when a check - failed but the request may proceed anyway, else ``None``. + resolution. Returns a dashboard billing projection or recovered auth object, + otherwise ``None``. """ ## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ## RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request) @@ -4353,6 +4527,36 @@ async def _authorize_authenticated_request( await _resolve_router_settings_model_group_alias(request_data, user_api_key_auth_obj, request, route) _resolve_codex_native_chatgpt_model(request_data, user_api_key_auth_obj, request, route) + from litellm.proxy.auth.auth_checks import dashboard_model_billing_auth + from litellm.proxy.proxy_server import llm_router as billing_router + + billing_model: Final = _get_model_from_request_context( + request_data, route, request, llm_router=billing_router, team_id=user_api_key_auth_obj.team_id + ) + billing_auth: Final = ( + await dashboard_model_billing_auth( + user_api_key_auth_obj, + billing_model, + llm_router=billing_router, + ) + if RouteChecks.is_llm_api_route(route=route) + else user_api_key_auth_obj + ) + recovered: Final = await _authorize_authenticated_request_checks( + billing_auth, request, request_data, route, api_key, force_virtual_key_checks=force_virtual_key_checks + ) + return (recovered or billing_auth) if billing_auth is not user_api_key_auth_obj else recovered + + +async def _authorize_authenticated_request_checks( + user_api_key_auth_obj: UserAPIKeyAuth, + request: Request, + request_data: dict[str, object], # mutable-ok: common checks update the ASGI body cache + route: str, + api_key: str, + *, + force_virtual_key_checks: bool = False, +) -> UserAPIKeyAuth | None: # Single authorization point. Builder paths MUST NOT call common_checks. # Route through the same exception handler the builder uses so # authorization failures (ProxyException, or plain Exception from @@ -4372,9 +4576,35 @@ async def _authorize_authenticated_request( llm_model_list, # pyright: ignore[reportUnknownVariableType] # proxy global has untyped deployment entries llm_router, prisma_client, + user_api_key_cache, user_model, ) + customer_identity: Final = customer_identity_from_headers( + safe_get_request_headers(request) # pyright: ignore[reportUnknownArgumentType] # request headers are untyped + ) + user_api_key_auth_obj.customer_hierarchy = None # rebind-ok: request state, never inherited from a cached key + if customer_identity is not None and not _is_unauthenticated_public_route(route): + with tracer.trace("litellm.proxy.auth.resolve_customer_hierarchy"): # pyright: ignore[reportUnknownMemberType] # tracer is the untyped proxy tracer + try: + hierarchy: Final = await resolve_customer_hierarchy( + identity=customer_identity, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + llm_router=llm_router, + request_model=_get_model_from_request_context( + request_data, route, request, llm_router, team_id=user_api_key_auth_obj.team_id + ), + route=route, + key_metadata=_KEY_METADATA_ADAPTER.validate_python( + user_api_key_auth_obj.metadata or {} # pyright: ignore[reportUnknownMemberType] # untyped key metadata, validated here + ), + ) + except CustomerBudgetExceededError as error: + user_api_key_auth_obj.customer_hierarchy = error.customer_hierarchy # rebind-ok: request state + raise + user_api_key_auth_obj.customer_hierarchy = hierarchy # rebind-ok: request state + grants: Final = await access_group_ids_for_identity(user_api_key_auth_obj) user_api_key_auth_obj.apply_access_group_rule_grants(grants) key_check_required: Final = ( @@ -4588,9 +4818,12 @@ async def user_api_key_auth( # close, and the trace never reaches the backend. _ensure_parent_otel_span_on_request_state(request) + route: Final[str] = get_request_route(request=request) + await _reject_from_headers_before_reading_body( + request=request, api_key=api_key, custom_litellm_key_header=custom_litellm_key_header, route=route + ) request_data, body_parse_exception = await _read_request_body_deferring_parse_failure(request=request) phase_event("litellm.request.body_parsed") - route: Final[str] = get_request_route(request=request) ## CHECK IF ROUTE IS ALLOWED # Run the whole auth phase inside a live ``auth`` span so the DB lookups it @@ -4626,30 +4859,29 @@ async def user_api_key_auth( ) settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(general_settings) - user_api_key_auth_obj: Final = ( - authenticated.model_copy() if settings.get("access_group_rules") else authenticated - ) - user_api_key_auth_obj.budget_reservation = None - user_api_key_auth_obj.agent_caller = agent_caller_from_headers( - safe_get_request_headers(request), user_api_key_auth_obj - ) - _seed_request_destinations(user_api_key_auth_obj, request) + request_auth: Final = authenticated.model_copy() if settings.get("access_group_rules") else authenticated + request_auth.budget_reservation = None + request_auth.agent_caller = agent_caller_from_headers(safe_get_request_headers(request), request_auth) + _seed_request_destinations(request_auth, request) # A body that never parsed is authenticated (so the trace carries identity # and this ``auth`` span) but not authorized: there is no model to check it # against, and budget reservation would increment live spend counters that # only the endpoint's post-call path releases; the endpoint never runs, since # the parse failure is raised below. - if body_parse_exception is None: - recovered_auth_obj: Final = await _authorize_authenticated_request( - user_api_key_auth_obj=user_api_key_auth_obj, + authorized_auth: Final = ( + await _authorize_authenticated_request( + user_api_key_auth_obj=request_auth, request=request, request_data=request_data, route=route, api_key=api_key, ) - if recovered_auth_obj is not None: - return recovered_auth_obj + if body_parse_exception is None + else None + ) + + user_api_key_auth_obj: Final = authorized_auth or request_auth # Identity is now resolved. Seed it AFTER the auth span closes so the Baggage # persists on the request task (detaching the span's context token inside the diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f38676c9e5d..b0623132fb1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -14,6 +14,7 @@ from collections.abc import ( Iterable, Iterator, Mapping, + MutableMapping, Sequence, ) from dataclasses import dataclass @@ -66,6 +67,9 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException +from litellm.integrations.custom_logger import subclass_overrides_hook +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.legacy import hook_subscribers from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure @@ -170,7 +174,11 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression -from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash +from litellm.proxy.hooks.dynamic_rate_limiter_v3 import PROXY_DynamicRateLimitHandlerV3 +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROXY_MaxParallelRequestsHandler_v3, + get_or_create_request_stash, +) from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.openai_files_endpoints.common_utils import ( @@ -184,9 +192,13 @@ from litellm.proxy.route_llm_request import ( from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id from litellm.proxy.utils import ( # noqa: F401 # legacy module exports + HookRun, ProxyLogging, _check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_and_merge_model_level_guardrails, + hook_fields, + routes_through_apply_guardrail, + run_hook_subscribers, ) from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task @@ -338,6 +350,8 @@ from litellm.proxy.litellm_pre_call_utils import ( SCHEDULER_PRIORITY_HEADER, LiteLLMProxyRequestSetup, add_litellm_data_to_request, + check_customer_request_model_access, + customer_request_auth, get_metadata_variable_name, global_then_key_alias_target, pre_call_served_model, @@ -348,6 +362,7 @@ from litellm.proxy.litellm_pre_call_utils import ( from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.utils import ( + LLMResponseTypes, ModelResponse, ModelResponseStream, StandardLoggingPayloadErrorInformation, @@ -812,6 +827,23 @@ _REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(str _UNSEEN_DEPLOYMENT: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) +class _DashboardBillingRequest(BaseModel): + model_config = ConfigDict(frozen=True) + dashboard_fallback_model: str | None = None + + +async def _check_dashboard_request_billing(processing: object, auth: object, llm_router: Router | None) -> None: + if llm_router is None or not isinstance(auth, UserAPIKeyAuth): + return + if auth.dashboard_session_origin is None: + return + from litellm.proxy.auth.fallback_model_access import dashboard_fallback_billing_allowed + + model: Final = _DashboardBillingRequest.model_validate(processing, from_attributes=True).dashboard_fallback_model + if isinstance(model, str) and not await dashboard_fallback_billing_allowed(model, auth, RoutingContext(llm_router)): + raise HTTPException(403, "Dashboard fallbacks cannot change the billing team") + + def _is_auto_router(params: Mapping[str, object]) -> bool: model: Final = params.get("model") return isinstance(model, str) and model.startswith(AUTO_ROUTER_MODEL_PREFIX) @@ -1043,6 +1075,7 @@ async def hierarchical_router_settings_updates( _REQUEST_MODEL: Final[TypeAdapter[str | list[str] | None]] = TypeAdapter(str | list[str] | None) +_FALLBACK_LIST: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object], config=ConfigDict(strict=True)) _DECODED_JSON: Final = TypeAdapter(object) _ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -2364,6 +2397,271 @@ def _timing_values( return getattr(logging_obj, "response_timing_metrics", None) or {} +def _fallback_refusal_ends_the_walk( + exc: Exception, at_saturation: bool, general_settings: Mapping[str, object] +) -> bool: + """At saturation the requested model still has room, so a fallback refused for any reason is a miss. Otherwise a + fallback's own non-rate-limit failure, or a hard per-model 429, is the answer the caller gets.""" + from litellm.proxy.common_utils.proxy_rate_limit_error import ( + PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS, + ProxyRateLimitError, + per_model_rate_limits_disable_fallbacks, + ) + + if at_saturation: + return False + if not isinstance(exc, ProxyRateLimitError): + return True + return exc.descriptor_key in PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS and per_model_rate_limits_disable_fallbacks( + general_settings + ) + + +async def _fallback_access_groups( + fallback_model: str, user_api_key_dict: UserAPIKeyAuth, llm_router: Router, proxy_logging_obj: ProxyLogging +) -> tuple[str, ...]: + from litellm.proxy.auth.auth_checks import ( + collect_matched_model_access_groups, + get_project_object, + get_team_membership, + get_team_object, + ) + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + if prisma_client is None: + return () + team_id: Final = user_api_key_dict.team_id + user_id: Final = user_api_key_dict.user_id + project_id: Final = user_api_key_dict.project_id + team_object: Final = ( + await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if team_id is not None + else None + ) + team_membership: Final = ( + await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if user_id is not None and team_id is not None + else None + ) + project_object: Final = ( + await get_project_object( + project_id=project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + if project_id is not None + else None + ) + return await collect_matched_model_access_groups( + model=fallback_model, + valid_token=user_api_key_dict, + team_object=team_object, + project_object=project_object, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + team_membership=team_membership, + team_membership_loaded=True, + ) + + +_PER_TOKEN_PRICES: Final[TypeAdapter[tuple[float, float]]] = TypeAdapter(tuple[float, float]) +_RESERVATION_ENTRIES: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) +_MODEL_ACCESS_GROUP_ENTITY: Final = "Model access group" + + +def _per_token_prices(llm_router: Router, model: str) -> tuple[float, float] | None: + info: Final = llm_router.registry().group(model) + if info is None: + return None + try: + return _PER_TOKEN_PRICES.validate_python((info.input_cost_per_token, info.output_cost_per_token)) + except ValidationError: + return None + + +def _reserved_access_groups(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str] | None: + """The model access groups the request's ``budget_reservation`` already holds, None when its entries are + unreadable.""" + reservation: Final = user_api_key_dict.budget_reservation + if reservation is None: + return frozenset() + try: + entries: Final = _RESERVATION_ENTRIES.validate_python(reservation.get("entries") or ()) + except ValidationError: + return None + return frozenset( + str(entry.get("entity_id")) for entry in entries if entry.get("entity_type") == _MODEL_ACCESS_GROUP_ENTITY + ) + + +def _reservation_covers( + user_api_key_dict: UserAPIKeyAuth, + access_groups: Sequence[str], + held: frozenset[str] | None, + requested_model: str, + fallback_model: str, + llm_router: Router, +) -> bool: + """Whether the budget auth reserved for ``requested_model`` still fits ``fallback_model``: the fallback costs no + more per token than the model the estimate was made for, and every access group ``held`` by the reservation + grants the fallback, so settlement never charges a group that did not serve.""" + if user_api_key_dict.budget_reservation is None: + return True + if held is None: + return False + requested: Final = _per_token_prices(llm_router, requested_model) + fallback: Final = _per_token_prices(llm_router, fallback_model) + if requested is None or fallback is None or fallback[0] > requested[0] or fallback[1] > requested[1]: + return False + return held <= frozenset(access_groups) + + +def _fallback_pricing_admits(requested_model: str, fallback_model: str, llm_router: Router) -> bool: + """A zero-cost ``requested_model`` waived the aggregate budget checks at auth, so only another zero-cost model may + take its place, and an unpriced fallback is refused while ``block_requests_for_models_without_pricing`` is on.""" + from litellm.proxy.auth.auth_checks import ( + _unpriced_models_in_request, # pyright: ignore[reportPrivateUsage] # the pre-call walk shares the auth-time fallback contract + ) + + if is_model_cost_zero(model=requested_model, llm_router=llm_router) and not is_model_cost_zero( + model=fallback_model, llm_router=llm_router + ): + return False + return not ( + litellm.block_requests_for_models_without_pricing + and _unpriced_models_in_request(model=fallback_model, llm_router=llm_router) + ) + + +async def _release_pre_call_hold( + proxy_logging_obj: ProxyLogging, + request_data: Mapping[str, object], + refusal: Exception, + user_api_key_dict: UserAPIKeyAuth, +) -> None: + """Give back the slot and tokens the rate limiter holds for a model the walk is leaving: the requested model a + later hook rate limited after the limiter admitted it, or a fallback the proxy refused after admitting it. The + walk's next attempt overwrites the request's stash, so a hold not released here stays charged until the window + resets. The request has not failed, so only the limiters' releases run, not the failure logging and alerting of + ``post_call_failure_hook``. The dynamic rate limiter admits through its own inner limiter, so its team-share + reservation is released alongside the registered limiter's slot and tokens.""" + limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter") + dynamic_limiters: Final = litellm.logging_callback_manager.get_custom_loggers_for_type( + PROXY_DynamicRateLimitHandlerV3 + ) + for holder in (limiter, *dynamic_limiters): + if not isinstance(holder, (PROXY_MaxParallelRequestsHandler_v3, PROXY_DynamicRateLimitHandlerV3)): + continue + await holder.async_post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # the limiter hooks take the proxy's bare request dict + request_data=dict(request_data), original_exception=refusal, user_api_key_dict=user_api_key_dict + ) + + +async def _fallback_grant( + fallback_model: str, + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, + proxy_logging_obj: ProxyLogging, + requested_model: str, + listed_as: str | None = None, +) -> list[str] | None: + """The model access groups a pre-call fallback to ``fallback_model`` charges, or None when the caller may not use + it. The caller passes the group the fallback is served by, through every alias rewrite, and the name the fallback + list gave it as ``listed_as``, so an alias never skips the budgets of the model that answers nor those set on the + listed name: an agent-owned key never leaves the model it asked for, as at auth, then the key, team and project allowlists + while ``enforce_fallback_model_access`` is on, the end user's allowlist, the pricing rules, every per-model budget layer, the budgets of the access groups that grant the model, and the budget auth reserved for + ``requested_model``, the same checks a direct request passes at auth.""" + from litellm.proxy.auth.auth_checks import ( + _model_access_group_max_budget_check, # pyright: ignore[reportPrivateUsage] # the pre-call walk shares the auth-time fallback contract + can_customer_access_model, + get_end_user_object, + ) + from litellm.proxy.auth.fallback_model_access import ( + dashboard_fallback_billing_allowed, + is_model_authorized_for_token, + router_fallback_access_check, + ) + from litellm.proxy.auth.user_api_key_auth import ( + _is_within_every_model_budget, # pyright: ignore[reportPrivateUsage] # the pre-call walk shares the auth-time fallback contract + ) + from litellm.proxy.proxy_server import model_max_budget_limiter, prisma_client, user_api_key_cache + + if llm_router is None: + return [] + if user_api_key_dict.agent_id is not None: + return None + if router_fallback_access_check.is_enforced() and not await is_model_authorized_for_token( + model=fallback_model, valid_token=user_api_key_dict, llm_router=llm_router + ): + return None + if not _fallback_pricing_admits(requested_model, fallback_model, llm_router): + return None + try: + if not await dashboard_fallback_billing_allowed(fallback_model, user_api_key_dict, llm_router): + return None + end_user: Final = ( + await get_end_user_object( + end_user_id=user_api_key_dict.end_user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if user_api_key_dict.end_user_id is not None and prisma_client is not None + else None + ) + if end_user is not None and end_user.models: + can_customer_access_model( + model=fallback_model, end_user_object=end_user, llm_router=llm_router, valid_token=user_api_key_dict + ) + if not is_model_cost_zero(model=fallback_model, llm_router=llm_router): + budget_names: Final = tuple( + dict.fromkeys( + (listed_as or fallback_model, fallback_model, llm_router.registry().resolve(fallback_model)) + ) + ) + for budget_name in budget_names: + if not await _is_within_every_model_budget( + valid_token=user_api_key_dict, model_max_budget_limiter=model_max_budget_limiter, model=budget_name + ): + return None + access_groups: Final = await _fallback_access_groups( + fallback_model, user_api_key_dict, llm_router, proxy_logging_obj + ) + held: Final = _reserved_access_groups(user_api_key_dict) + await _model_access_group_max_budget_check( + matched_model_access_groups=tuple(group for group in access_groups if group not in (held or ())), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + if not _reservation_covers(user_api_key_dict, access_groups, held, requested_model, fallback_model, llm_router): + return None + except Exception as e: # noqa: BLE001 # fail closed: a grant the walk cannot settle must not admit the fallback + verbose_proxy_logger.warning( + "Skipping fallback to model=%s: %s", + fallback_model.replace("\r", "").replace("\n", ""), + e, + ) + return None + return list(access_groups) + + async def _team_member_within_fallback_model_budget( user_api_key_dict: UserAPIKeyAuth, model: str, @@ -2481,6 +2779,7 @@ class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): self.data = data self.served_model = served_model + self.dashboard_fallback_model: str | None = None self._tags_before_guardrails: frozenset[str] | None = None self.route_type: str | None = None @@ -2808,6 +3107,7 @@ class ProxyBaseLLMRequestProcessing: proxy_config=proxy_config, ) request_fields: Final = _REQUEST_FIELDS.validate_python(self.data) + customer_auth: Final = customer_request_auth(request_fields) metadata_variable_name: Final = get_metadata_variable_name(request, request_fields) if logging_auth is not None: request_metadata: Final = request_fields.get(metadata_variable_name) @@ -3001,6 +3301,12 @@ class ProxyBaseLLMRequestProcessing: aliased_model, kind="served", served_model=self.served_model ) + check_customer_request_model_access( + _REQUEST_FIELDS.validate_python(self.data), # pyright: ignore[reportUnknownMemberType] # validate the legacy request carrier at this boundary + customer_auth, + llm_router, + ) + return self.data, logging_obj async def _pre_call_with_fallbacks( @@ -3024,6 +3330,12 @@ class ProxyBaseLLMRequestProcessing: PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS, ProxyRateLimitError, RateLimitFallback, + arm_saturation_fallback_walk, + disarm_saturation_fallback_walk, + drop_client_saturation_opt_in, + fallback_on_saturation_enabled, + is_saturation_fallback, + mark_saturation_fallbacks_exhausted, per_model_rate_limits_disable_fallbacks, with_rate_limit_fallback, ) @@ -3038,12 +3350,26 @@ class ProxyBaseLLMRequestProcessing: and per_model_rate_limits_disable_fallbacks(general_settings_view) ) - configured_fallbacks: Final = ( - self._configured_fallbacks(llm_router=llm_router, user_api_key_dict=user_api_key_dict) + router_settings: Final = ( + await self._hierarchical_router_settings(user_api_key_dict, proxy_config, proxy_logging_obj) if llm_router is not None else None ) - pristine: Final = independent_snapshot(self.data) if configured_fallbacks else None + configured_fallbacks: Final = ( + self._configured_fallbacks(llm_router=llm_router, router_settings=router_settings) + if llm_router is not None + else None + ) + scrubbed: Final[dict] = self.data # mutable-ok: caller-defined request-payload shape + payload: Final = cast(MutableMapping[str, object], scrubbed) # cast-ok: untyped caller payload + drop_client_saturation_opt_in(payload) + walk: Final = arm_saturation_fallback_walk() + self.data = scrubbed + pristine: Final = ( + independent_snapshot(scrubbed) + if configured_fallbacks or fallback_on_saturation_enabled(router_settings) + else None + ) try: return await self.common_processing_pre_call_logic( @@ -3065,20 +3391,17 @@ class ProxyBaseLLMRequestProcessing: except ProxyRateLimitError as original_exc: rate_limited_data: Final = self.data original_model: Final = rate_limited_data.get("model") - if ( - pristine is None - or not configured_fallbacks - or rate_limited_data.get("disable_fallbacks") - or not isinstance(original_model, str) - or is_hard_per_model_limit(original_exc) - ): + # The limiter raises this one while the requested model still has room, so a walk that admits nothing + # ends on that model instead of a 429. + at_saturation: Final = is_saturation_fallback(original_exc) + if pristine is None or not isinstance(original_model, str) or is_hard_per_model_limit(original_exc): raise - - fallback_models: Final = self._resolve_fallback_models( - model=original_model, - fallbacks=configured_fallbacks, + fallback_models: Final = ( + self._resolve_fallback_models(model=original_model, fallbacks=configured_fallbacks) + if configured_fallbacks and not rate_limited_data.get("disable_fallbacks") + else None ) - if not fallback_models: + if not fallback_models and not at_saturation: raise verbose_proxy_logger.info( @@ -3094,11 +3417,26 @@ class ProxyBaseLLMRequestProcessing: if isinstance(raw_router_settings, Mapping) else None ) + await _release_pre_call_hold( + proxy_logging_obj, _REQUEST_FIELDS.validate_python(rate_limited_data), original_exc, user_api_key_dict + ) attempt_numbers: Final[Iterator[int]] = itertools.count(1) + requested_groups: Final = user_api_key_dict.matched_model_access_groups try: - for fallback_model in fallback_models: + for fallback_model in fallback_models or (): if fallback_model == original_model: continue + if ( + grant := await _fallback_grant( + pre_call_served_model(fallback_model, user_api_key_dict, group_alias), + user_api_key_dict, + llm_router, + proxy_logging_obj, + original_model, + listed_as=fallback_model, + ) + ) is None: + continue if not await _team_member_within_fallback_model_budget( user_api_key_dict=user_api_key_dict, model=fallback_model, @@ -3109,6 +3447,7 @@ class ProxyBaseLLMRequestProcessing: continue self.data = independent_snapshot(pristine) self.data["model"] = fallback_model + self.dashboard_fallback_model = fallback_model try: if endpoint_unsupported_by_model( route_type, @@ -3118,6 +3457,7 @@ class ProxyBaseLLMRequestProcessing: ): continue attempt = next(attempt_numbers) + user_api_key_dict.matched_model_access_groups = grant # rebind-ok: bill the fallback's groups admitted: tuple[ Mapping[str, object], LiteLLMLoggingObj ] = await self.common_processing_pre_call_logic( @@ -3137,9 +3477,16 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, rate_limited_model=original_model, ) - except ProxyRateLimitError as fallback_exc: - if is_hard_per_model_limit(fallback_exc): + except Exception as fallback_exc: # noqa: BLE001 # the helper decides which refusals end the walk + user_api_key_dict.matched_model_access_groups = requested_groups # rebind-ok: undo the fallback + if _fallback_refusal_ends_the_walk(fallback_exc, at_saturation, general_settings_view): raise + await _release_pre_call_hold( + proxy_logging_obj, + self.data, # pyright: ignore[reportUnknownMemberType] # caller-defined request-payload shape + fallback_exc, + user_api_key_dict, + ) continue else: admitted_data, admitted_logging_obj = admitted @@ -3156,19 +3503,65 @@ class ProxyBaseLLMRequestProcessing: ) self.data = stamped return stamped, admitted_logging_obj + if not at_saturation: + self.data = rate_limited_data + raise original_exc + verbose_proxy_logger.info( + "No fallback of model=%s is below the saturation threshold, serving the request on it", + original_model.replace("\r", "").replace("\n", ""), + ) + mark_saturation_fallbacks_exhausted() + self.data = independent_snapshot(dict(_REQUEST_FIELDS.validate_python(pristine))) + return await self.common_processing_pre_call_logic( + request=request, + general_settings=general_settings, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + model=model, # the requested name as the first attempt saw it, so its aliases apply once, not again + route_type=route_type, # pyright: ignore[reportArgumentType] # the proxy hands its route literal over as str, as the two calls above take it + llm_router=llm_router, + ) except BaseException: + user_api_key_dict.matched_model_access_groups = requested_groups # rebind-ok: undo the fallback self.data = rate_limited_data raise - - self.data = rate_limited_data - raise original_exc + finally: + disarm_saturation_fallback_walk(walk) @staticmethod - def _configured_fallbacks(llm_router: Router, user_api_key_dict: UserAPIKeyAuth) -> list | None: - key_router_settings: Final = user_api_key_dict.router_settings - key_fallbacks: Final = key_router_settings.get("fallbacks") if isinstance(key_router_settings, dict) else None - fallbacks: Final = key_fallbacks if key_fallbacks is not None else llm_router.settings().fallbacks - return fallbacks if isinstance(fallbacks, list) and fallbacks else None + async def _hierarchical_router_settings( + user_api_key_dict: UserAPIKeyAuth, proxy_config: ProxyConfig, proxy_logging_obj: ProxyLogging + ) -> Mapping[str, object] | None: + """The key's ``router_settings``, else its team's, read through the same cached lookup the request body + gets them from, so the pre-call fallback walk and the router agree on Key > Team > Global.""" + from litellm.proxy.proxy_server import prisma_client + + return _ROUTER_SETTINGS_ADAPTER.validate_python( + await proxy_config.get_hierarchical_router_settings( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + ) + ) + + @staticmethod + def _configured_fallbacks(llm_router: Router, router_settings: Mapping[str, object] | None) -> list[object] | None: + settings_fallbacks: Final = router_settings.get("fallbacks") if router_settings is not None else None + fallbacks: Final[object] = ( + settings_fallbacks if settings_fallbacks is not None else llm_router.settings().fallbacks + ) + try: + configured: Final = _FALLBACK_LIST.validate_python(fallbacks) + except ValidationError: + return None + return configured or None @staticmethod def _resolve_fallback_models(model: str, fallbacks: list) -> list[str] | None: @@ -3471,6 +3864,8 @@ class ProxyBaseLLMRequestProcessing: ) phase_event("litellm.request.pre_call_completed") + await _check_dashboard_request_billing(self, user_api_key_dict, llm_router) + _post_call_guardrails_active: Final = self._has_post_call_guardrails() if not self._is_streaming_request(data=self.data, is_streaming_request=is_streaming_request): @@ -4233,11 +4628,9 @@ class ProxyBaseLLMRequestProcessing: is not True ): continue - leaf_attrs = type(cb).__dict__ - holds_until_moderated = "async_post_call_streaming_iterator_hook" in leaf_attrs + holds_until_moderated = subclass_overrides_hook(type(cb), "async_post_call_streaming_iterator_hook") if ( - "apply_guardrail" not in leaf_attrs - or cb.use_native_lifecycle_hooks + not routes_through_apply_guardrail(cb) or cb.mask_response_content or cb.run_in_parallel or ProxyBaseLLMRequestProcessing._releases_held_stream_on_scan(cb, holds_until_moderated) @@ -4683,7 +5076,7 @@ class ProxyBaseLLMRequestProcessing: Extracted as a static method so tests can call the production implementation directly rather than reimplementing the closure. """ - _response = assembled_response + run: Final[HookRun[LLMResponseTypes]] = HookRun(response=assembled_response) try: from litellm.proxy.proxy_server import llm_router as _global_llm_router from litellm.proxy.utils import ( @@ -4695,47 +5088,18 @@ class ProxyBaseLLMRequestProcessing: data=captured_data, llm_router=_global_llm_router ) stream_gated: Final = stream_gated_guardrail_names(captured_data, captured_user_api_key_dict) - for cb in litellm.callbacks: - if not isinstance(cb, CustomGuardrail): - continue - if cb.guardrail_name in stream_gated: - continue - if not cb.should_run_guardrail( - data=guardrail_data, - event_type=GuardrailEventHooks.post_call, - ): - continue - try: - guardrail_result = None - if "apply_guardrail" in type(cb).__dict__ and not cb.use_native_lifecycle_hooks: - # Skip — unified-routed guardrails already ran via - # unified_guardrail's end-of-stream block in the - # streaming iterator pipeline. Running them again - # here would duplicate the guardrail API call - # (e.g. double OpenAI Moderation charges). - continue - if "async_post_call_streaming_iterator_hook" in type(cb).__dict__: - # Skip — the guardrail already scanned the assembled - # response via its own streaming iterator hook in the - # streaming pipeline. re running this function async_post_call_success_hook - # here would duplicate the scan and can spuriously block the guardrail that already passed / failed. - continue - else: - guardrail_result = await cb.async_post_call_success_hook( - user_api_key_dict=captured_user_api_key_dict, - data=guardrail_data, - response=_response, - ) - if guardrail_result is not None: - _response = guardrail_result - except Exception as e: - verbose_proxy_logger.exception( - "Error running post-call guardrail %s on streaming response: %s", - getattr(cb, "guardrail_name", type(cb).__name__), - e, - ) - if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "model_call_details"): - captured_logging_obj.model_call_details.setdefault("metadata", {})["guardrail_blocked"] = True + await run_hook_subscribers( + hook_subscribers(Hook.post_call_success, litellm.callbacks, parallel=False), + hook_fields(guardrail_data, run, Field.response), + partial( + ProxyBaseLLMRequestProcessing._deferred_stream_guardrail, + run, + guardrail_data, + stream_gated, + captured_user_api_key_dict, + captured_logging_obj, + ), + ) except Exception as e: verbose_proxy_logger.exception( "Error in deferred streaming guardrail initialization: %s", @@ -4750,7 +5114,7 @@ class ProxyBaseLLMRequestProcessing: # async markers stored in litellm_params). asyncio.create_task( _as_success_dispatcher(captured_logging_obj).dispatch_success_handlers( - _response, + run.response, cache_hit=cache_hit, start_time=None, end_time=None, @@ -4763,6 +5127,55 @@ class ProxyBaseLLMRequestProcessing: e, ) + @staticmethod + async def _deferred_stream_guardrail( + run: HookRun[LLMResponseTypes], + guardrail_data: dict[str, object], # mutable-ok: the captured request merged with model-level guardrails + stream_gated: frozenset[str], + captured_user_api_key_dict: "UserAPIKeyAuth", + captured_logging_obj: LiteLLMLoggingObj, + cb: object, + request: object, + ) -> None: + if not isinstance(cb, CustomGuardrail): + return + if cb.guardrail_name in stream_gated: + return + if not cb.should_run_guardrail( + data=guardrail_data, + event_type=GuardrailEventHooks.post_call, + ): + return + try: + if routes_through_apply_guardrail(cb): + # Skip — unified-routed guardrails already ran via + # unified_guardrail's end-of-stream block in the + # streaming iterator pipeline. Running them again + # here would duplicate the guardrail API call + # (e.g. double OpenAI Moderation charges). + return + if subclass_overrides_hook(type(cb), "async_post_call_streaming_iterator_hook"): + # Skip — the guardrail already scanned the assembled + # response via its own streaming iterator hook in the + # streaming pipeline. re running this function async_post_call_success_hook + # here would duplicate the scan and can spuriously block the guardrail that already passed / failed. + return + run.adopt( + await cb.async_post_call_success_hook( + user_api_key_dict=captured_user_api_key_dict, + data=guardrail_data, + response=run.response, + ) + ) + except Exception as e: + verbose_proxy_logger.exception( + "Error running post-call guardrail %s on streaming response: %s", + getattr(cb, "guardrail_name", type(cb).__name__), + e, + ) + if isinstance(e, HTTPException) and hasattr(captured_logging_obj, "model_call_details"): + captured_logging_obj.model_call_details.setdefault("metadata", {})["guardrail_blocked"] = True + def _apply_router_cooldown_retry_after(self, headers: dict, e: Exception) -> None: if isinstance(e, RouterRateLimitError) and e.cooldown_time > 0: headers["retry-after"] = str(math.ceil(e.cooldown_time)) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index dca410d9b28..c39f56b6a08 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -159,6 +159,18 @@ def initialize_callbacks_on_proxy( if isinstance(value, list): imported_list: Final[list[Any]] = [] for callback in value: # ["presidio", ] + if callback == "customer_budget_webhook": + from litellm.integrations.customer_budget_webhook import CustomerBudgetWebhook + + imported_list.append( + CustomerBudgetWebhook.from_config( + string_keyed(_LOADED_CALLBACK.validate_python(callback_specific_params)).get( + "customer_budget_webhook" + ) + ) + ) + continue + if isinstance(callback, str) and callback == "compression_interception": from litellm.integrations.compression_interception.handler import ( CompressionInterceptionLogger, diff --git a/litellm/proxy/common_utils/customer_usage.py b/litellm/proxy/common_utils/customer_usage.py new file mode 100644 index 00000000000..73157d986a6 --- /dev/null +++ b/litellm/proxy/common_utils/customer_usage.py @@ -0,0 +1,241 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from typing import Annotated, Final, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +from litellm.proxy.management_helpers.customer_hierarchy import effective_member_budget +from litellm.repositories.prisma_protocols import TableActions +from litellm.types.proxy.management_endpoints.customer_hierarchy import Budget + +CustomerIdentifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, max_length=256)] +_IDENTITY_FIELDS: Final = ("customer_org_id", "customer_team_id", "customer_member_id") +CUSTOMER_USAGE_METADATA_KEY: Final = "user_api_key_customer_usage" +_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +class CustomerUsageInput(BaseModel): + model_config = ConfigDict(frozen=True) + + customer_org_id: CustomerIdentifier | None = None + customer_team_id: CustomerIdentifier | None = None + customer_member_id: CustomerIdentifier | None = None + + +class CustomerUsageAttribution(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + customer_org_id: CustomerIdentifier + customer_team_id: CustomerIdentifier | None = None + customer_member_id: CustomerIdentifier | None = None + + +@dataclass(frozen=True, slots=True) +class ResolvedCustomerUsage: + attribution: CustomerUsageAttribution + model_scopes: tuple[tuple[str, ...], ...] + + +class _HierarchyBudget(Budget): + model_config = ConfigDict(frozen=True, from_attributes=True) + + +class _Organization(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + budget: _HierarchyBudget | None = None + + +class _Team(_Organization): + model_config = ConfigDict(frozen=True, from_attributes=True) + + parent_org_id: str + + +class _Member(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + parent_team_ids: tuple[str, ...] = () + budget: _HierarchyBudget | None = None + budget_override: _HierarchyBudget | None = None + budget_override_expires_at: datetime | None = None + + +def _budget_models(budget: Budget | None) -> tuple[str, ...]: + if budget is None: + return () + return budget.allowed_models or () + + +def _model_scopes( + organization: _Organization, team: _Team | None, member: _Member | None +) -> tuple[tuple[str, ...], ...]: + member_budget: Final = ( + effective_member_budget( + member.budget, member.budget_override, member.budget_override_expires_at, datetime.now(timezone.utc) + ) + if member is not None + else None + ) + scopes: Final = ( + _budget_models(organization.budget), + _budget_models(team.budget if team is not None else None), + _budget_models(member_budget), + ) + return tuple(scope for scope in scopes if scope) + + +class _KeyMetadata(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + metadata: object = None + + +class CustomerUsageDatabase(Protocol): + @property + def litellm_customerorganizationtable(self) -> TableActions[object]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[object]: ... + + @property + def litellm_customermembertable(self) -> TableActions[object]: ... + + +class CustomerUsageError(str, Enum): + INVALID = "Customer usage metadata contains invalid identifiers." + MISMATCH = "Customer usage identifiers must belong to the same organization and team." + FORBIDDEN = "Customer usage must match the customer organization assigned to this API key." + UNBOUND = "Assign a customer organization to the API key before tracking customer usage." + NOT_FOUND = "The customer usage organization, team, or member does not exist." + UNAVAILABLE = "Customer usage tracking requires a database and spend accounting ledger." + KEY_BINDING_FORBIDDEN = "Only proxy admins can change a key's customer organization, team, or member binding." + + +@dataclass(frozen=True, slots=True) +class CustomerUsageFailure: + error: CustomerUsageError + + +def normalize_customer_key_metadata( + metadata: Mapping[str, object] | None, + existing_metadata: Mapping[str, object] | None, + *, + administrator: bool, +) -> Mapping[str, object] | CustomerUsageFailure | None: + if administrator: + return metadata + stored: Final = existing_metadata or {} + requested: Final = metadata or {} + if any(name in requested and requested[name] != stored.get(name) for name in _IDENTITY_FIELDS): + return CustomerUsageFailure(CustomerUsageError.KEY_BINDING_FORBIDDEN) + bindings: Final = {name: stored[name] for name in _IDENTITY_FIELDS if stored.get(name) is not None} + if not bindings: + return metadata + return {**requested, **bindings} + + +def _input(metadata: Mapping[str, object], binding: Mapping[str, object]) -> CustomerUsageInput | CustomerUsageFailure: + if any( + name in metadata and name in binding and metadata[name] != binding[name] + for name in _IDENTITY_FIELDS + if binding.get(name) is not None + ): + return CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + try: + return CustomerUsageInput.model_validate( + {**metadata, **{name: binding[name] for name in _IDENTITY_FIELDS if binding.get(name) is not None}} + ) + except ValidationError: + return CustomerUsageFailure(CustomerUsageError.INVALID) + + +def _hierarchy_failure(error: CustomerUsageError, *, administrator: bool) -> CustomerUsageFailure: + if administrator: + return CustomerUsageFailure(error) + return CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + + +async def resolve_customer_usage( + *, + metadata: Mapping[str, object], + key_metadata: Mapping[str, object], + administrator: bool, + database: CustomerUsageDatabase | None, + ledger_enabled: bool, +) -> ResolvedCustomerUsage | CustomerUsageFailure | None: + if not customer_usage_requested(metadata, key_metadata): + return None + candidate: Final = _input(metadata, key_metadata) + if isinstance(candidate, CustomerUsageFailure): + return candidate + if not administrator and key_metadata.get("customer_org_id") is None: + return CustomerUsageFailure(CustomerUsageError.UNBOUND) + if database is None or not ledger_enabled: + return CustomerUsageFailure(CustomerUsageError.UNAVAILABLE) + member: Final = ( + await database.litellm_customermembertable.find_unique( + where={"customer_member_id": candidate.customer_member_id}, + include={"budget": True, "budget_override": True}, + ) + if candidate.customer_member_id is not None + else None + ) + if candidate.customer_member_id is not None and member is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + member_record: Final = _Member.model_validate(member) if member is not None else None + member_teams: Final = member_record.parent_team_ids if member_record is not None else () + if candidate.customer_team_id is not None and member_teams and candidate.customer_team_id not in member_teams: + return _hierarchy_failure(CustomerUsageError.MISMATCH, administrator=administrator) + team_id: Final = candidate.customer_team_id or (member_teams[0] if member_teams else None) + team: Final = ( + await database.litellm_customerteamtable.find_unique( + where={"customer_team_id": team_id}, include={"budget": True} + ) + if team_id is not None + else None + ) + if team_id is not None and team is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + team_record: Final = _Team.model_validate(team) if team is not None else None + team_org: Final = team_record.parent_org_id if team_record is not None else None + if candidate.customer_org_id is not None and team_org is not None and candidate.customer_org_id != team_org: + return _hierarchy_failure(CustomerUsageError.MISMATCH, administrator=administrator) + organization_id: Final = candidate.customer_org_id or team_org + if organization_id is None: + return CustomerUsageFailure(CustomerUsageError.INVALID) + organization: Final = await database.litellm_customerorganizationtable.find_unique( + where={"customer_org_id": organization_id}, include={"budget": True} + ) + if organization is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + return ResolvedCustomerUsage( + attribution=CustomerUsageAttribution( + customer_org_id=organization_id, + customer_team_id=team_id, + customer_member_id=candidate.customer_member_id, + ), + model_scopes=_model_scopes(_Organization.model_validate(organization), team_record, member_record), + ) + + +def customer_usage_requested(metadata: Mapping[str, object], key_metadata: Mapping[str, object]) -> bool: + return any(metadata.get(name) is not None or key_metadata.get(name) is not None for name in _IDENTITY_FIELDS) + + +def customer_usage_fields(value: object) -> Mapping[str, object]: + return _FIELDS.validate_python(value) if isinstance(value, Mapping) else {} + + +def customer_usage_key_fields(key: object) -> Mapping[str, object]: + return customer_usage_fields(_KeyMetadata.model_validate(key).metadata) + + +def customer_usage_http_status(error: CustomerUsageError) -> int: + if error == CustomerUsageError.UNAVAILABLE: + return 503 + if error in (CustomerUsageError.UNBOUND, CustomerUsageError.FORBIDDEN, CustomerUsageError.KEY_BINDING_FORBIDDEN): + return 403 + return 400 diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 6f3ef4e03d4..82d7f08d8ac 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -36,10 +36,11 @@ This module provides a single proxy-side error class that: """ import json -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping +from contextvars import ContextVar, Token from dataclasses import dataclass from types import MappingProxyType -from typing import Final +from typing import Final, Literal, TypeAlias from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError @@ -56,6 +57,9 @@ PER_MODEL_RATE_LIMIT_DESCRIPTOR_KEYS: Final = frozenset( "model_per_project", "model_per_project_itpm", "model_per_project_otpm", + "model_per_customer_org", + "model_per_customer_team", + "model_per_customer_member", } ) DISABLE_FALLBACKS_ON_PER_MODEL_RATE_LIMITS_SETTING: Final = "disable_fallbacks_on_per_model_rate_limits" @@ -75,6 +79,87 @@ def per_model_rate_limits_disable_fallbacks(general_settings: Mapping[str, objec return False +FALLBACK_ON_SATURATION_SETTING: Final = "fallback_on_saturation" +SATURATION_FALLBACK_DESCRIPTOR_KEY: Final = "model_saturation_fallback" +_FALLBACK_ON_SATURATION_FLAG: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None) + + +def fallback_on_saturation_enabled(router_settings: object) -> bool: + """Whether the key's or team's ``router_settings`` ask for fallbacks once the requested model reaches + ``saturation_threshold``, before the dynamic rate limiter admits or refuses the request on that model.""" + raw_value: Final = _metadata_bucket(router_settings).get(FALLBACK_ON_SATURATION_SETTING) + try: + return _FALLBACK_ON_SATURATION_FLAG.validate_python(raw_value, strict=True) is True + except ValidationError: + verbose_proxy_logger.warning( + "router_settings.%s=%r is not a boolean, treating it as disabled", FALLBACK_ON_SATURATION_SETTING, raw_value + ) + return False + + +def drop_client_saturation_opt_in( + request_data: MutableMapping[str, object], # mutable-ok: in-place edit keeps the caller's request object +) -> None: + """Drop ``fallback_on_saturation`` from the client's ``router_settings_override``. Only the key's or team's saved + ``router_settings`` turn the saturation walk on, and the proxy writes those over this field before the limiter + reads it.""" + override: Final = _metadata_bucket(request_data.get("router_settings_override")) + if FALLBACK_ON_SATURATION_SETTING not in override: + return + request_data["router_settings_override"] = { # rebind-ok: the request object must stay the caller's + key: value for key, value in override.items() if key != FALLBACK_ON_SATURATION_SETTING + } + + +_SaturationWalkState: TypeAlias = Literal["off", "armed", "exhausted"] +_SATURATION_FALLBACK_WALK: Final[ContextVar[_SaturationWalkState]] = ContextVar( + "litellm_saturation_fallback_walk", default="off" +) + + +def saturation_fallback_walk_armed() -> bool: + """Whether a caller that catches the handoff (``_pre_call_with_fallbacks``) is running this request's hooks and + has not walked the fallbacks yet.""" + return _SATURATION_FALLBACK_WALK.get() == "armed" + + +def saturation_fallbacks_exhausted() -> bool: + """Whether this request already tried every fallback at saturation and is back on the model it asked for.""" + return _SATURATION_FALLBACK_WALK.get() == "exhausted" + + +def arm_saturation_fallback_walk() -> Token[_SaturationWalkState]: + """Mark the current request as running under the fallback walk, so the limiter may hand a saturated model off to + it. The state lives in the request's context, never in its body, so nothing reaches the provider and no client + field can arm, skip or exhaust the walk. Routes that call the hooks directly never arm it and keep admitting or + refusing as before.""" + return _SATURATION_FALLBACK_WALK.set("armed") + + +def mark_saturation_fallbacks_exhausted() -> None: + _SATURATION_FALLBACK_WALK.set("exhausted") + + +def disarm_saturation_fallback_walk(token: Token[_SaturationWalkState]) -> None: + _SATURATION_FALLBACK_WALK.reset(token) + + +def fallback_on_saturation_requested(request_data: Mapping[str, object]) -> bool: + """Whether the dynamic rate limiter should hand a saturated model's request to the fallback walk instead of + admitting or refusing it: the key or team opted in through ``router_settings_override`` and the fallback walk + is the caller and has not already run for this request.""" + return ( + fallback_on_saturation_enabled(request_data.get("router_settings_override")) + and saturation_fallback_walk_armed() + ) + + +def is_saturation_fallback(exc: "ProxyRateLimitError") -> bool: + """Whether ``exc`` is the limiter asking for the fallback walk rather than refusing the request. The model it + names still has room, so when no fallback admits the request it goes back to that model instead of a 429.""" + return exc.descriptor_key == SATURATION_FALLBACK_DESCRIPTOR_KEY + + RATE_LIMIT_FALLBACK_METADATA_KEY: Final = "rate_limit_fallback" RATE_LIMIT_FALLBACK_REASON_HEADER: Final = "x-litellm-fallback-reason" ATTEMPTED_FALLBACKS_HEADER: Final = "x-litellm-attempted-fallbacks" @@ -201,7 +286,7 @@ def _coerce_message(detail: object) -> str: try: return json.dumps(detail) except (TypeError, ValueError): - return str(detail) + return f"{detail}" return str(detail) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index ae91cbb1f2a..27d9a0b7125 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -7,14 +7,16 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from enum import Enum from itertools import chain +from operator import attrgetter from types import MappingProxyType from typing import ( Final, Generic, Literal, Protocol, + TypeAlias, TypeVar, - cast, # noqa: TID251 # get_data returns validated rows selected by table_name + cast, # noqa: TID251 # get_data returns validated rows selected by table_name, ) from typing_extensions import assert_never @@ -30,7 +32,11 @@ from litellm.constants import ( RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN, RESET_BUDGET_JOB_NAME, ) -from litellm.litellm_core_utils.duration_parser import duration_in_seconds +from litellm.litellm_core_utils.duration_parser import ( + duration_in_seconds, + get_annual_budget_window_start, + is_annual_budget_duration, +) from litellm.proxy._types import ( DB_RETRY_SAFE_ERROR_TYPES, LiteLLM_BudgetTableFull, @@ -39,6 +45,8 @@ from litellm.proxy._types import ( LiteLLM_UserTable, LiteLLM_VerificationToken, ) +from litellm.proxy.auth.customer_hierarchy_auth import customer_member_cache_key, customer_org_alias_cache_key +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import broadcast_evictions from litellm.proxy.common_utils.timezone_utils import ( BudgetResetSettings, compute_budget_reset_at, @@ -76,6 +84,9 @@ from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.prisma_protocols import PrismaBatch, SpendLinkedTable from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ( + CustomerMemberRepository, + CustomerOrganizationRepository, + CustomerTeamRepository, EndUserRepository, ModelAccessGroupBudgetRepository, TagRepository, @@ -83,9 +94,11 @@ from litellm.repositories.table_repositories import ( ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.unit_of_work import ( + CustomerSpendTable, LinkedSpendResetWrites, SpendResetWrites, budget_cascade_unit_of_work, + customer_spend_reset_unit_of_work, enduser_spend_reset_unit_of_work, spend_reset_unit_of_work, ) @@ -94,7 +107,65 @@ from litellm.repositories.verification_token_repository import ( ) from litellm.types.services import ServiceTypes +CustomerSpendEntity: TypeAlias = Literal["member", "team", "org"] + + +class _CustomerOrgRow(Protocol): + @property + def customer_org_id(self) -> str: ... + + @property + def alias(self) -> str | None: ... + + +class _BudgetCapRow(Protocol): + @property + def max_budget(self) -> float | None: ... + + +class _CustomerScopedRow(Protocol): + """A customer team or member: spend governed by its template, or by an unexpired override.""" + + @property + def spend(self) -> float | None: ... + + @property + def budget_override_id(self) -> str | None: ... + + @property + def budget_override_expires_at(self) -> datetime | None: ... + + @property + def budget(self) -> _BudgetCapRow | None: ... + + @property + def budget_override(self) -> _BudgetCapRow | None: ... + + +class _CustomerTeamRow(_CustomerScopedRow, Protocol): + @property + def customer_team_id(self) -> str: ... + + +class _CustomerMemberRow(_CustomerScopedRow, Protocol): + @property + def customer_member_id(self) -> str: ... + + _RowT = TypeVar("_RowT") +_RowT_co: Final = TypeVar("_RowT_co", covariant=True) +_CustomerRowT: Final = TypeVar("_CustomerRowT", bound=_CustomerScopedRow) + + +class CustomerScopeTable(Protocol[_RowT_co]): + async def find_many( + self, + *, + where: Mapping[str, object], + include: Mapping[str, bool], + order: Mapping[str, str], + take: int, + ) -> Sequence[_RowT_co]: ... @dataclass(frozen=True, slots=True) @@ -238,8 +309,73 @@ def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]: def _budget_link_where( budget_ids: Sequence[str], extra: Mapping[str, object] = MappingProxyType({}), + link: str = "budget_id", ) -> dict[str, object]: - return {"budget_id": {"in": list(budget_ids)}, **extra} + # bounded-ok: budget_ids is one RESET_BUDGET_JOB_BATCH_SIZE batch of due budget rows + return {link: {"in": list(budget_ids)}, **extra} + + +def _active_override_where(now: datetime) -> Mapping[str, object]: + """Teams or members whose unexpired override carries its own window, reset on the override's schedule.""" + return { + **_SPENT_ROWS_WHERE, + "OR": [{"budget_override_expires_at": None}, {"budget_override_expires_at": {"gt": now}}], + } + + +def _customer_due_where(budget_ids: Sequence[str], now: datetime) -> Mapping[str, object]: + """Teams or members one of the expiring tiers resets: governed by it as their template, or as their override.""" + return { + "OR": [ + {**_budget_link_where(budget_ids), **_template_governed_where(now)}, + {**_budget_link_where(budget_ids, link="budget_override_id"), **_active_override_where(now)}, + ] + } + + +def _customer_effective_cap(row: _CustomerScopedRow, now: datetime) -> float | None: + """The limit the row is admitted against: an unexpired override's ``max_budget`` when it sets one, else the + template's. Rollover carries spend past this cap, not past whichever row happens to own the reset clock.""" + expires_at: Final = row.budget_override_expires_at + override_active: Final = row.budget_override_id is not None and (expires_at is None or expires_at > now) + override: Final = row.budget_override if override_active else None + if override is not None and override.max_budget is not None: + return _rollover_cap(override.max_budget) + return _rollover_cap(row.budget.max_budget) if row.budget is not None else None + + +def _customer_ids_by_cap( + rows: Sequence[_CustomerScopedRow], ids: Sequence[str], now: datetime +) -> Mapping[float | None, tuple[str, ...]]: + caps: Final = tuple(_customer_effective_cap(row, now) if _rollover_enabled() else None for row in rows) + return MappingProxyType( + {cap: tuple(row_id for row_id, row_cap in zip(ids, caps) if row_cap == cap) for cap in dict.fromkeys(caps)} + ) + + +def _queue_customer_page_resets( + writes: LinkedSpendResetWrites, id_field: str, ids_by_cap: Mapping[float | None, tuple[str, ...]] +) -> None: + """``_queue_enduser_tier_resets`` for one page of customer rows, grouped by the cap each row carries over.""" + for cap, ids in ids_by_cap.items(): + page = {id_field: {"in": list(ids)}} # bounded-ok: one page of RESET_BUDGET_JOB_BATCH_SIZE ids + if cap is None: + writes.queue_spend_zero(where={**page, **_SPENT_ROWS_WHERE}) + continue + writes.queue_spend_zero(where={**page, "spend": {"gt": 0, "lte": cap}}) + writes.queue_spend_decrement(where={**page, "spend": {"gt": cap}}, amount=cap) + + +def _template_governed_where(now: datetime) -> Mapping[str, object]: + """Teams or members whose window comes from their budget: no override, an expired one, or one without a duration.""" + return { + **_SPENT_ROWS_WHERE, + "OR": [ + {"budget_override_id": None}, + {"budget_override_expires_at": {"lte": now}}, + {"budget_override": {"is": {"budget_duration": None}}}, + ], + } def _enduser_invalidation_where(budget_ids: Sequence[str]) -> dict[str, object]: @@ -259,6 +395,7 @@ def _queue_budget_linked_resets( writes: SpendResetWrites, cascade: "_BudgetCascade", extra: Mapping[str, object] = MappingProxyType({}), + link: str = "budget_id", ) -> None: """Reset one linked table's spend for every expiring tier: tiers with a rollover cap keep spend beyond the cap (decrement preserves writes racing @@ -267,11 +404,11 @@ def _queue_budget_linked_resets( one transaction, so the reverse order lets the zero re-match a row the decrement just moved into the (0, cap] range and erase its carried spend.""" for budget_id, cap in cascade.rollover_caps.items(): - writes.queue_spend_zero(where={"budget_id": budget_id, **extra, "spend": {"gt": 0, "lte": cap}}) - writes.queue_spend_decrement(where={"budget_id": budget_id, **extra, "spend": {"gt": cap}}, amount=cap) + writes.queue_spend_zero(where={link: budget_id, **extra, "spend": {"gt": 0, "lte": cap}}) + writes.queue_spend_decrement(where={link: budget_id, **extra, "spend": {"gt": cap}}, amount=cap) plain_ids: Final = tuple(bid for bid in cascade.budget_ids if bid not in cascade.rollover_caps) if plain_ids: - writes.queue_spend_zero(where=_budget_link_where(plain_ids, extra)) + writes.queue_spend_zero(where=_budget_link_where(plain_ids, extra, link)) def _enduser_reset_where(cascade: "_BudgetCascade") -> Mapping[str, object]: @@ -324,6 +461,8 @@ class _BudgetCascade: counter_resets: tuple[tuple[str, float], ...] = () cache_keys: tuple[str, ...] = () rollover_caps: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) + customer_ids: Mapping[CustomerSpendEntity, tuple[str, ...]] = field(default_factory=lambda: MappingProxyType({})) + customer_org_alias_keys: tuple[str, ...] = () @dataclass(frozen=True, slots=True) @@ -599,6 +738,40 @@ class ResetBudgetJob: if lease is _Lease.LEADER and self.pod_lock_manager is not None: await self.pod_lock_manager.release_lock(cronjob_id=RESET_BUDGET_JOB_NAME) + async def _evict_on_every_worker(self, keys: Sequence[str]) -> None: + if not keys: + return + await self._invalidate_cache("user_api_key_cache", keys) + await broadcast_evictions(keys) + + async def _invalidate_customer_budget_caches(self, entity: CustomerSpendEntity, identities: Sequence[str]) -> None: + if entity == "member": + await self._evict_on_every_worker(tuple(customer_member_cache_key(identity) for identity in identities)) + return + table: Final = CustomerMemberRepository(self.prisma_client, use_writer=True).table + parents: Final = ( + {"parent_team_ids": {"has_some": list(identities)}} # bounded-ok: one RESET_BUDGET_JOB_BATCH_SIZE page + if entity == "team" + else {"parent_org_id": {"in": list(identities)}} # bounded-ok: one RESET_BUDGET_JOB_BATCH_SIZE page + ) + cursor: str | None = None # rebind-ok: keyset walk must invalidate every descendant after commit + while True: + records = await self._with_db_retry( + lambda: table.find_many( + where={**parents, **({"customer_member_id": {"gt": cursor}} if cursor is not None else {})}, + order={"customer_member_id": "asc"}, + take=RESET_BUDGET_JOB_BATCH_SIZE, + ), + reason="reset_budget_read_customer_cache_failure", + ) + if not records: + return + rows: tuple[_CustomerMemberRow, ...] = tuple(records) + await self._invalidate_customer_budget_caches("member", tuple(row.customer_member_id for row in rows)) + if len(rows) < RESET_BUDGET_JOB_BATCH_SIZE: + return + cursor = rows[-1].customer_member_id + async def _with_db_retry(self, operation: Callable[[], Awaitable[_RowT]], *, reason: str) -> _RowT: """Reconnect and retry once on a transport error, so a dropped connection costs one retry instead of the whole tick. @@ -814,6 +987,11 @@ class ResetBudgetJob: where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE), log_subject="projects", ) + customer_orgs: Final[tuple[_CustomerOrgRow, ...]] = await self._fetch_linked_rows( + table=CustomerOrganizationRepository(self.prisma_client).table, + where=_budget_link_where(budget_ids, _SPENT_ROWS_WHERE), + log_subject="customer organizations", + ) rollover_caps: Final[Mapping[str, float]] = MappingProxyType( { b.budget_id: cap @@ -849,6 +1027,10 @@ class ResetBudgetJob: *((_project_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in projects), ), rollover_caps=rollover_caps, + customer_ids=MappingProxyType({"org": tuple(row.customer_org_id for row in customer_orgs)}), + customer_org_alias_keys=tuple( + customer_org_alias_cache_key(row.alias) for row in customer_orgs if row.alias + ), cache_keys=( *(key for row in team_memberships for key in _team_membership_cache_keys(row)), *(key for row in keys for key in _key_cache_keys(row)), @@ -895,6 +1077,85 @@ class ResetBudgetJob: return _EndUserWalk(cursor=None, invalidated=walked) return _EndUserWalk(cursor=rows[-1].user_id, invalidated=walked) + async def _reset_customer_team_spend(self, cascade: _BudgetCascade) -> None: + table: Final[CustomerScopeTable[_CustomerTeamRow]] = CustomerTeamRepository(self.prisma_client).table + await self._reset_customer_scope_spend( + cascade, + entity="team", + table=table, + batch_table="litellm_customerteamtable", + id_field="customer_team_id", + row_id=attrgetter("customer_team_id"), + ) + + async def _reset_customer_member_spend(self, cascade: _BudgetCascade) -> None: + table: Final[CustomerScopeTable[_CustomerMemberRow]] = CustomerMemberRepository(self.prisma_client).table + await self._reset_customer_scope_spend( + cascade, + entity="member", + table=table, + batch_table="litellm_customermembertable", + id_field="customer_member_id", + row_id=attrgetter("customer_member_id"), + ) + + async def _reset_customer_scope_spend( + self, + cascade: _BudgetCascade, + entity: CustomerSpendEntity, + table: CustomerScopeTable[_CustomerRowT], + batch_table: CustomerSpendTable, + id_field: str, + row_id: Callable[[_CustomerRowT], str], + ) -> None: + """Reset the customer teams or members the expiring tiers govern, one committed page at a time. + + Paged like ``_reset_enduser_spend``: an aliased request can register a member, so the rows on one + template are unbounded. Each row carries its own rollover cap, read with its budget rows, because an + override may set the limit while the template owns the reset clock, or the other way round.""" + if not cascade.budget_ids: + return + now: Final = datetime.now(timezone.utc) + where: Final = _customer_due_where(cascade.budget_ids, now) + cursor: str | None = "" # rebind-ok: keyset cursor advances one committed page at a time + while cursor is not None: + rows: tuple[_CustomerRowT, ...] = tuple( + await self._with_db_retry( + lambda: table.find_many( + where={**where, id_field: {"gt": cursor}}, + include={"budget": True, "budget_override": True}, + order={id_field: "asc"}, + take=RESET_BUDGET_JOB_BATCH_SIZE, + ), + reason=f"reset_budget_read_customer_{entity}s_failure", + ) + ) + if not rows: + return + ids = tuple(row_id(row) for row in rows) + ids_by_cap = _customer_ids_by_cap(rows, ids, now) + await self._with_db_write_retry( + lambda: self._commit_customer_page(batch_table, id_field, ids_by_cap), + reason=f"reset_budget_write_customer_{entity}s_failure", + ) + await self._evict_customer_page(entity, ids) + cursor = ids[-1] if len(rows) == RESET_BUDGET_JOB_BATCH_SIZE else None + + async def _evict_customer_page(self, entity: CustomerSpendEntity, ids: Sequence[str]) -> None: + try: + await self._invalidate_customer_budget_caches(entity, ids) + except Exception as e: # noqa: BLE001 # the page is committed; a failed eviction must not stop the walk + verbose_proxy_logger.warning("Failed to invalidate customer %s hierarchy caches: %s", entity, e) + + async def _commit_customer_page( + self, batch_table: CustomerSpendTable, id_field: str, ids_by_cap: Mapping[float | None, tuple[str, ...]] + ) -> None: + async with ( + db_span("reset_budget_customers", batch_table), + customer_spend_reset_unit_of_work(self._new_batch, batch_table) as writes, + ): + _queue_customer_page_resets(writes, id_field, ids_by_cap) + async def _commit_enduser_page(self, rows: Sequence[_EndUserRow], cascade: _BudgetCascade) -> None: async with ( db_span("reset_budget_endusers", "LiteLLM_EndUserTable"), @@ -931,6 +1192,7 @@ class ResetBudgetJob: _queue_budget_linked_resets(uow.projects, cascade, extra=_SPENT_ROWS_WHERE) _queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE) _queue_budget_linked_resets(uow.model_access_groups, cascade, extra=_SPENT_ROWS_WHERE) + _queue_budget_linked_resets(uow.customer_organizations, cascade, extra=_SPENT_ROWS_WHERE) for budget_id, budget_reset_at in cascade.budget_resets: uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at) @@ -939,6 +1201,14 @@ class ResetBudgetJob: counter_keys=tuple(counter_key for counter_key, _ in cascade.counter_resets), cache_keys=cascade.cache_keys, ) + await self._evict_on_every_worker(cascade.customer_org_alias_keys) + for entity, identities in cascade.customer_ids.items(): + if not identities: + continue + try: + await self._invalidate_customer_budget_caches(entity, identities) + except Exception as e: # noqa: BLE001 # the reset is committed; a failed eviction must not fail the phase + verbose_proxy_logger.warning("Failed to invalidate customer %s hierarchy caches: %s", entity, e) async def _reset_expired_budget_cascade(self) -> _BudgetCascadeCommitted | _BudgetCascadeFailed: now: Final = datetime.now(timezone.utc) @@ -958,6 +1228,8 @@ class ResetBudgetJob: try: await self._reset_enduser_spend(cascade) + await self._reset_customer_team_spend(cascade) + await self._reset_customer_member_spend(cascade) await self._commit_budget_cascade(cascade) except Exception as e: return _BudgetCascadeFailed(cascade=cascade, error=e) @@ -1514,16 +1786,21 @@ class ResetBudgetJob: reset_at_str: Final = window.get("reset_at") if not reset_at_str: return False - reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None) - if reset_at > now: + budget_duration: Final = window["budget_duration"] + annual: Final = isinstance(budget_duration, str) and is_annual_budget_duration(budget_duration) + saved_reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")) + reset_at: Final = _as_utc(saved_reset_at) if annual else saved_reset_at.replace(tzinfo=None) + cutoff: Final = _as_utc(now) if annual else now + if reset_at > cutoff: return False new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_store) try: await spend_counter_store.set(counter_key, new_value) except Exception as redis_err: verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err) - budget_duration: Final = window["budget_duration"] - next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings) + next_reset_at: Final = compute_budget_reset_at( + budget_duration=budget_duration, settings=reset_settings, now=now if annual else None + ) window["reset_at"] = next_reset_at.isoformat() await ResetBudgetJob._roll_window_spend_row( prisma_client=prisma_client, @@ -1531,6 +1808,7 @@ class ResetBudgetJob: entity_id=entity_id, budget_duration=budget_duration, next_reset_at=next_reset_at, + reset_settings=reset_settings, ) return True @@ -1541,6 +1819,7 @@ class ResetBudgetJob: entity_id: str, budget_duration: str, next_reset_at: datetime, + reset_settings: BudgetResetSettings | None = None, ) -> None: """Move this window's LiteLLM_BudgetWindowSpend row onto the window that just started, so the maintained total the read path uses starts @@ -1551,7 +1830,11 @@ class ResetBudgetJob: windows from having their counters reset. """ try: - window_start: Final = next_reset_at - timedelta(seconds=duration_in_seconds(budget_duration)) + window_start: Final = ( + get_annual_budget_window_start(next_reset_at, (reset_settings or get_budget_reset_settings()).timezone) + if is_annual_budget_duration(budget_duration) + else next_reset_at - timedelta(seconds=duration_in_seconds(budget_duration)) + ) except Exception as e: # noqa: BLE001 # duration_in_seconds raises bare exceptions on bad input verbose_proxy_logger.warning("Unparseable budget_duration %s: %s", budget_duration, e) return @@ -1708,7 +1991,9 @@ class ResetBudgetJob: item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0 if hasattr(item, "budget_duration") and item.budget_duration is not None: item.budget_reset_at = compute_budget_reset_at( - budget_duration=item.budget_duration, settings=reset_settings + budget_duration=item.budget_duration, + settings=reset_settings, + now=current_time if is_annual_budget_duration(item.budget_duration) else None, ) return item except Exception as e: diff --git a/litellm/proxy/common_utils/timezone_utils.py b/litellm/proxy/common_utils/timezone_utils.py index 7a40931f50d..1a44585f75e 100644 --- a/litellm/proxy/common_utils/timezone_utils.py +++ b/litellm/proxy/common_utils/timezone_utils.py @@ -63,11 +63,13 @@ def get_budget_reset_settings() -> BudgetResetSettings: ) -def compute_budget_reset_at(budget_duration: str, settings: BudgetResetSettings) -> datetime: +def compute_budget_reset_at( + budget_duration: str, settings: BudgetResetSettings, now: datetime | None = None +) -> datetime: """Compute the next reset time for a budget duration using injected settings.""" return get_next_standardized_reset_time( duration=budget_duration, - current_time=datetime.now(timezone.utc), + current_time=now if now is not None else datetime.now(timezone.utc), timezone_str=settings.timezone, reset_time_of_day=settings.reset_time_of_day, ) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index 91529ee36fc..cabeb866864 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -4,6 +4,7 @@ import asyncio import re from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload +from weakref import WeakValueDictionary from pydantic import BaseModel, TypeAdapter @@ -47,6 +48,14 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None: return _TTL.validate_python(kwargs.get("ttl")) +class CacheLease: + __slots__ = ("__weakref__", "key", "shared") + + def __init__(self, key: str, shared: str | None) -> None: + self.key: Final = key + self.shared: Final = shared + + class UserApiKeyCache(DualCache): """ DualCache for UserAPIKeyAuth-like payloads. @@ -104,6 +113,7 @@ class UserApiKeyCache(DualCache): self._auth_cache: Final = auth_cache self._redis_connection: RedisCache | None = None self._shared: AuthCacheStore | None = None + self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary() self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl) @property @@ -174,8 +184,42 @@ class UserApiKeyCache(DualCache): return self._python_memory(key).set_cache(key=key, value=value, ttl=effective_ttl) + async def lease(self, key: str) -> CacheLease: + """Taken on a cache miss, before the database read that will fill ``key``, and handed to + ``async_set_leased``. Evicting the key voids it: the rows read under it may predate the write behind + that eviction. With ``enable_redis_auth_cache`` an eviction on any worker voids it. Loads of one key + on this worker share a lease until an eviction, and it is dropped once none holds it.""" + held: Final = self._leases.get(key) + if held is not None: + return held + shared: Final = None if self._shared is None else await self._shared.lease(key) + return self._leases.setdefault(key, CacheLease(key, shared)) + + async def async_set_leased(self, lease: CacheLease, value: CacheValue, ttl: float | None) -> bool: + """Caches ``value`` unless the lease's key was evicted since it was taken; False when refused.""" + if self._leases.get(lease.key) is not lease: + return False + shared: Final = self._shared + if shared is None: + await self.async_set_cache(key=lease.key, value=value, ttl=ttl) + return True + payload: Final[object] = CacheCodec.serialize(value, model_type=None) + written: Final = await shared.set_leased(lease.key, lease.shared, payload, ttl) + if self._leases.get(lease.key) is not lease: + return False + if not written: + self._void_leases((lease.key,)) + return False + shared.set_local(lease.key, payload, ttl) + return True + + def _void_leases(self, keys: Sequence[str]) -> None: + for key in keys: + _ = self._leases.pop(key, None) + def delete_local(self, key: str) -> None: """Drops this worker's memory copy alone, as a cross-worker eviction broadcast does.""" + self._void_leases((key,)) if key.startswith(_BUDGET_WINDOWS_ABSENT_PREFIX): self.budget_windows_absent_cache.delete_cache(key) if self._shared is not None: @@ -327,11 +371,7 @@ class UserApiKeyCache(DualCache): await self._set_shared(shared, key, payload, local_only=local_only, pipelined=pipelined, kwargs=kwargs) return None if key is not None and is_user_key_cache_key(key): - if pipelined and await self.key_object_cache.async_set_cache_pre_call(key, payload, ttl) is not None: - return None return await self.key_object_cache.async_set_cache(key=key, value=payload, local_only=local_only, **kwargs) - if pipelined and await super().async_set_cache_pre_call(key, payload, ttl) is not None: - return None return await super().async_set_cache(key=key, value=payload, local_only=local_only, **kwargs) @staticmethod @@ -354,6 +394,7 @@ class UserApiKeyCache(DualCache): verbose_proxy_logger.error("UserApiKeyCache.async_set_cache failed for key=%r: %s", key, e) def delete_cache(self, key: str) -> None: + self._void_leases((key,)) if self._shared is not None: self._shared.delete_sync((key,)) return @@ -363,6 +404,7 @@ class UserApiKeyCache(DualCache): super().delete_cache(key) async def async_delete_cache(self, key: str) -> None: + self._void_leases((key,)) if self._shared is not None: await self._shared.delete((key,)) return @@ -371,12 +413,10 @@ class UserApiKeyCache(DualCache): return await super().async_delete_cache(key) - async def async_delete_cache_pre_call(self, key: str) -> Awaitable[None] | None: - if self._shared is not None: - return self._shared.declare_delete(key) - if is_user_key_cache_key(key): - return await self.key_object_cache.async_delete_cache_pre_call(key) - return await super().async_delete_cache_pre_call(key) + def declare_delete(self, key: str) -> Awaitable[None] | None: + """The DEL on the request's pre-call batch; None without Redis or outside a request, so the caller deletes.""" + self._void_leases((key,)) + return None if self._shared is None else self._shared.declare_delete(key) async def async_delete_cache_keys(self, keys: Sequence[str]) -> None: """Batch twin of ``async_delete_cache``, partitioned like @@ -385,6 +425,7 @@ class UserApiKeyCache(DualCache): Both partitions are cleared even when one raises, because a caller batching these has already committed the rows they cache. """ + self._void_leases(keys) if self._shared is not None: if keys: await self._shared.delete(keys) @@ -401,6 +442,7 @@ class UserApiKeyCache(DualCache): raise failed[0] def flush_cache(self) -> None: + self._leases.clear() self.budget_windows_absent_cache.flush_cache() if self._shared is not None: self._shared.flush_local() diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 4ddcf7e600b..cc1806e7f77 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -21,7 +21,7 @@ import random from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone -from itertools import groupby +from itertools import chain, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Final, NamedTuple @@ -30,6 +30,7 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES from litellm.proxy.db.create_views import SupportsExecuteRaw from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.exception_handler import db_write_never_sent if TYPE_CHECKING: from litellm.proxy._types import SpendLogsPayload @@ -536,77 +537,110 @@ def _session_partition(transaction: AutoRouterTurnTransaction) -> tuple[str, str return (*identity, transaction.session_id, transaction.router_name) +class UnwrittenTurns(NamedTuple): + """Turns the database provably never received, and the error that proved it.""" + + turns: tuple[AutoRouterTurnTransaction, ...] + error: Exception + + +class _StoppedPartition(NamedTuple): + turns: tuple[AutoRouterTurnTransaction, ...] + error: Exception + + async def _drain_session_partition( prisma_client: PrismaClient, transactions: tuple[AutoRouterTurnTransaction, ...], n_retry_times: int, statement: str, -) -> tuple[AutoRouterTurnTransaction, ...]: +) -> _StoppedPartition | None: for position, transaction in enumerate(transactions): try: await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times, statement) - except Exception as flush_err: # noqa: BLE001 # stop dependent turns without retrying an ambiguous write - verbose_proxy_logger.error( - "Spend tracking - auto-router session rollup flush failed for router %s; " - "%s of %s turn writes stopped in this partition: %s", - transaction.router_name, - len(transactions) - position, - len(transactions), - flush_err, - ) - return transactions[position:] - return () + except Exception as flush_err: # noqa: BLE001 # the caller keeps or drops the stopped turns + return _StoppedPartition(transactions[position:], flush_err) + return None + + +def _log_dropped_turns(stopped: _StoppedPartition, partition_size: int) -> None: + verbose_proxy_logger.error( + "Spend tracking - auto-router session rollup flush failed for router %s; " + "%s of %s turn writes stopped in this partition: %s", + stopped.turns[0].router_name, + len(stopped.turns), + partition_size, + stopped.error, + ) async def _flush_session_partition( prisma_client: PrismaClient, transactions: tuple[AutoRouterTurnTransaction, ...], n_retry_times: int, -) -> None: - failed_suffix: Final = await _drain_session_partition( +) -> UnwrittenTurns | None: + stopped: Final = await _drain_session_partition( prisma_client, transactions, n_retry_times, UPSERT_AUTOROUTER_SESSION_SQL ) - if not failed_suffix or not failed_suffix[0].api_key: - return - failed_user: Final = failed_suffix[0].user_id + if stopped is None: + return None + if db_write_never_sent(stopped.error): + return UnwrittenTurns(stopped.turns, stopped.error) + _log_dropped_turns(stopped, len(transactions)) + if not stopped.turns[0].api_key: + return None + failed_user: Final = stopped.turns[0].user_id other_users: Final = sorted( ( transaction - for transaction in failed_suffix[1:] + for transaction in stopped.turns[1:] if transaction.user_id and transaction.user_id != failed_user ), key=lambda transaction: transaction.user_id, ) - for _, user_turns in groupby(other_users, key=lambda transaction: transaction.user_id): - await _drain_session_partition( - prisma_client, tuple(user_turns), n_retry_times, UPSERT_AUTOROUTER_USER_SESSION_SQL - ) + user_partitions: Final = tuple( + tuple(user_turns) for _, user_turns in groupby(other_users, key=lambda transaction: transaction.user_id) + ) + for user_partition in user_partitions: + if ( + projection_stopped := await _drain_session_partition( + prisma_client, user_partition, n_retry_times, UPSERT_AUTOROUTER_USER_SESSION_SQL + ) + ) is not None: + _log_dropped_turns(projection_stopped, len(user_partition)) + return None async def flush_autorouter_turn_transactions( prisma_client: PrismaClient, transactions: Sequence[AutoRouterTurnTransaction], n_retry_times: int = 3, -) -> None: +) -> UnwrittenTurns | None: """Drain a queue batch into the rollup, one upsert per turn. Statements run sequentially in per-session event order: a turn's classification depends on the turns before it, and Postgres rejects one multi-row INSERT touching the same key twice. Only ConnectError is retried, per statement, because it proves - that statement never reached the database. A failed write stops its key and user - histories for this batch. Other users sharing that key can still advance their - independent user histories, with the key projection disabled and the real key - identity preserved. The failed turn is never replayed. Callers must not add their - own retry around this function. + that statement never reached the database. When a failure proves the database could + not take the write at all, the flush stops and returns that turn, the rest of its + session and every session not yet attempted, so the caller can queue them again; + turns already written are never returned. Any other failed write stops its key and + user histories for this batch and is never replayed. Other users sharing that key can + still advance their independent user histories, with the key projection disabled and + the real key identity preserved. Callers must not add their own retry around this + function. """ if not transactions: - return + return None ordered: Final = sorted( transactions, key=lambda transaction: (*_session_partition(transaction), transaction.turn_at), ) - for _, session_group in groupby( - ordered, - key=_session_partition, - ): - await _flush_session_partition(prisma_client, tuple(session_group), n_retry_times) + partitions: Final = tuple(tuple(group) for _, group in groupby(ordered, key=_session_partition)) + for position, partition in enumerate(partitions): + if (unwritten := await _flush_session_partition(prisma_client, partition, n_retry_times)) is not None: + return UnwrittenTurns( + (*unwritten.turns, *chain.from_iterable(partitions[position + 1 :])), + unwritten.error, + ) + return None diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index 38b132e7e10..784eec03dff 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -10,13 +10,15 @@ the window in which those locks are held. import uuid from collections.abc import Mapping, Sequence from dataclasses import dataclass +from hashlib import sha256 from itertools import groupby +from json import dumps from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from pydantic import TypeAdapter -DailySpendEntity = Literal["user", "team", "org", "tag", "end_user", "agent"] +DailySpendEntity: TypeAlias = Literal["user", "team", "org", "tag", "end_user", "agent", "customer_member"] SqlValue = str | int | float | None @@ -32,10 +34,21 @@ class DailySpendTable: name: str entity_id_column: str carries_request_id: bool = False + parent_id_columns: tuple[str, ...] = () + # The customer table arbitrates on a bounded hash of its identity (model group + # included) stored in ``id``, so three long customer ids never exceed an index entry + # and rows already written under that identity keep accumulating. + hashed_identity: bool = False DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingProxyType( { + "customer_member": DailySpendTable( + name="LiteLLM_DailyCustomerMemberSpend", + entity_id_column="customer_member_id", + parent_id_columns=("customer_team_id", "customer_org_id"), + hashed_identity=True, + ), "user": DailySpendTable(name="LiteLLM_DailyUserSpend", entity_id_column="user_id"), "team": DailySpendTable(name="LiteLLM_DailyTeamSpend", entity_id_column="team_id"), "org": DailySpendTable(name="LiteLLM_DailyOrganizationSpend", entity_id_column="organization_id"), @@ -47,6 +60,7 @@ DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingP _ENTITY_INPUT_KEYS: Final[Mapping[DailySpendEntity, str]] = MappingProxyType( { + "customer_member": "customer_member_id", "user": "user", "team": "team_id", "org": "organization_id", @@ -59,6 +73,12 @@ _TAGS: Final = TypeAdapter(tuple[object, ...]) def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEntity) -> tuple[str | None, ...]: + if entity == "customer_member" and not ( + isinstance(payload.get("customer_org_id"), str) + and payload.get("customer_org_id") + and all(isinstance(payload.get(column), str) for column in ("customer_member_id", "customer_team_id")) + ): + return () key: Final = _ENTITY_INPUT_KEYS[entity] if key not in payload: return () @@ -125,9 +145,24 @@ def _as_float(value: object) -> float: return float(value) if isinstance(value, (int, float)) else 0.0 +def _identity_columns(table: DailySpendTable) -> tuple[str, ...]: + return ( + table.entity_id_column, + *table.parent_id_columns, + *_KEY_COLUMNS, + *(("model_group",) if table.hashed_identity else ()), + ) + + def conflict_key(table: DailySpendTable, transaction: SpendRow) -> tuple[str, ...]: """The tuple the database arbitrates the upsert on, normalized free of NULLs.""" - return tuple(_as_text(transaction.get(column)) for column in (table.entity_id_column, *_KEY_COLUMNS)) + return tuple(_as_text(transaction.get(column)) for column in _identity_columns(table)) + + +def hashed_row_id(table: DailySpendTable, key: tuple[str, ...]) -> str: + """The ``id`` of a hashed-identity row, identical for every writer of the same identity.""" + pairs: Final = sorted(zip(_identity_columns(table), key, strict=True)) + return sha256(dumps(pairs, separators=(",", ":")).encode("utf-8")).hexdigest() def _merge(group: Sequence[SpendRow]) -> SpendRow: @@ -162,10 +197,11 @@ def _row_params( transaction: SpendRow, ) -> tuple[SqlValue, ...]: request_id: Final = transaction.get("request_id") + model_group: Final = transaction.get("model_group") return ( - str(uuid.uuid4()), - *key, - None if transaction.get("model_group") is None else _as_text(transaction.get("model_group")), + hashed_row_id(table, key) if table.hashed_identity else str(uuid.uuid4()), + *key[: len(key) - 1 if table.hashed_identity else len(key)], + _as_text(model_group) if table.hashed_identity or model_group is not None else None, *(_as_int(transaction.get(column)) for column in _COUNTER_COLUMNS), *(_as_float(transaction.get(column)) for column in _SPEND_COLUMNS), *((None if request_id is None else _as_text(request_id),) if table.carries_request_id else ()), @@ -176,6 +212,7 @@ def _insert_columns(table: DailySpendTable) -> tuple[str, ...]: return ( "id", table.entity_id_column, + *table.parent_id_columns, *_KEY_COLUMNS, "model_group", *_COUNTER_COLUMNS, @@ -214,7 +251,7 @@ def build_bulk_upsert( sql: Final = ( f'INSERT INTO {quoted_table} ({_quoted(columns)}, "updated_at")\n' f"VALUES {rows}\n" - f"ON CONFLICT ({_quoted((table.entity_id_column, *_KEY_COLUMNS))}) DO UPDATE SET\n" + f"ON CONFLICT ({_quoted(('id',) if table.hashed_identity else _identity_columns(table))}) DO UPDATE SET\n" f" {increments}{request_id_update},\n" f" \"updated_at\" = (NOW() AT TIME ZONE 'UTC')" ) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5a9a52447b7..8122c4de589 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -45,6 +45,7 @@ from litellm.proxy._types import ( DB_RETRY_SAFE_ERROR_TYPES, BaseDailySpendTransaction, DailyAgentSpendTransaction, + DailyCustomerMemberSpendTransaction, DailyEndUserSpendTransaction, DailyOrganizationSpendTransaction, DailyTagSpendTransaction, @@ -61,6 +62,7 @@ from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, p from litellm.proxy.db.daily_spend_bulk_upsert import ( DAILY_SPEND_TABLES, build_bulk_upsert, + conflict_key, daily_spend_entity_ids, merge_by_conflict_key, ) @@ -79,13 +81,14 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendUpdateQueue, ) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_failure_is_requeue_safe -from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction +from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction, utc_date_from_start_time from litellm.proxy.db.request_error_tracking import request_error_accumulator from litellm.proxy.db.rollup_lock_timeout import apply_rollup_lock_timeout from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, ) +from litellm.proxy.spend_tracking.customer_budget_alerts import CustomerBudgetAlerting from litellm.proxy.spend_tracking.savings import ( compute_savings_spend, extract_cache_creation_tokens, @@ -96,6 +99,11 @@ from litellm.proxy.spend_tracking.session_spend_total import SessionSpendTotals from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable from litellm.responses.streaming_iterator import spawn_logging_task +from litellm.types.proxy.auth.customer_hierarchy import ( + CustomerHierarchyIds, + customer_budget_scopes_from_request, + customer_hierarchy_ids_from_request, +) from litellm.types.utils import CallTypes from litellm.types.videos.utils import VIDEO_COST_REQUEST_ID_SUFFIX, VIDEO_STATUS_CALL_TYPES @@ -103,6 +111,7 @@ if TYPE_CHECKING: from litellm.accounting.projection import Contribution from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution + from litellm.proxy.spend_tracking.budget_reservation import BudgetCounter from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router else: @@ -363,6 +372,9 @@ _SpendTableName = Literal[ "tag_list_transactions", "model_access_group_list_transactions", "agent_list_transactions", + "customer_org_list_transactions", + "customer_team_list_transactions", + "customer_member_list_transactions", ] _SPEND_TABLE_COMMIT_ORDER: Final[tuple[_SpendTableName, ...]] = ( "org_list_transactions", @@ -376,6 +388,36 @@ _SPEND_TABLE_COMMIT_ORDER: Final[tuple[_SpendTableName, ...]] = ( "tag_list_transactions", "model_access_group_list_transactions", "agent_list_transactions", + "customer_org_list_transactions", + "customer_team_list_transactions", + "customer_member_list_transactions", +) + + +@dataclasses.dataclass(frozen=True, slots=True) +class _CustomerHierarchyLevel: + transactions: _SpendTableName + table: str + prisma_table: str + id_column: str + + +_CUSTOMER_HIERARCHY_LEVELS: Final = ( + _CustomerHierarchyLevel( + "customer_org_list_transactions", + "LiteLLM_CustomerOrganizationTable", + "litellm_customerorganizationtable", + "customer_org_id", + ), + _CustomerHierarchyLevel( + "customer_team_list_transactions", "LiteLLM_CustomerTeamTable", "litellm_customerteamtable", "customer_team_id" + ), + _CustomerHierarchyLevel( + "customer_member_list_transactions", + "LiteLLM_CustomerMemberTable", + "litellm_customermembertable", + "customer_member_id", + ), ) @@ -412,6 +454,7 @@ class _SpendRestore(TypedDict, total=False): daily_org_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] daily_end_user_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] daily_agent_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_customer_member_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] window_spend_update_transactions: ReadOnly[Sequence[WindowSpendTransaction]] @@ -734,6 +777,7 @@ class DBSpendUpdateWriter: self.daily_team_spend_update_queue = DailySpendUpdateQueue() self.daily_end_user_spend_update_queue = DailySpendUpdateQueue() self.daily_agent_spend_update_queue = DailySpendUpdateQueue() + self.daily_customer_member_spend_update_queue = DailySpendUpdateQueue() self.daily_org_spend_update_queue = DailySpendUpdateQueue() self.daily_tag_spend_update_queue = DailySpendUpdateQueue() self.window_spend_update_queue = WindowSpendUpdateQueue() @@ -754,6 +798,7 @@ class DBSpendUpdateWriter: end_time: datetime, response_cost: float | None, project_id: str | None = None, + customer_counters: "Sequence[BudgetCounter] | None" = None, ) -> bool: """Record the request's spend, answering whether its cost still needs charging. @@ -762,6 +807,7 @@ class DBSpendUpdateWriter: """ from litellm.proxy.proxy_server import ( accounting_window_keys, + customer_accounting_counters, disable_spend_logs, general_settings, # pyright: ignore[reportUnknownVariableType] # legacy proxy configuration is narrowed below litellm_proxy_budget_name, @@ -774,6 +820,7 @@ class DBSpendUpdateWriter: ) use_ledger: Final = proxy_settings.get("use_spend_logs_ledger", True) is not False request_fields: Final = _SPEND_METADATA_ADAPTER.validate_python(kwargs or {}) + customer_hierarchy: Final = customer_hierarchy_ids_from_request(request_fields) if kwargs is not None: kwargs[ACCOUNTING_LEDGER_KWARG] = use_ledger # rebind-ok: spend counters leave ledger windows to the worker try: @@ -792,13 +839,13 @@ class DBSpendUpdateWriter: ## CREATE SPEND LOG PAYLOAD ## from litellm.proxy.spend_tracking.spend_tracking_utils import ( - get_logging_payload, + get_logging_payload_async, get_request_model_access_groups, ) llm_router: Final = get_llm_router() - payload: Final = get_logging_payload( - kwargs=kwargs, + payload: Final = await get_logging_payload_async( + kwargs=_SPEND_METADATA_ADAPTER.validate_python(kwargs) if kwargs is not None else None, response_obj=completion_response, start_time=start_time, end_time=end_time, @@ -834,13 +881,24 @@ class DBSpendUpdateWriter: "team_id": team_id, "organization_id": org_id, "project_id": project_id, + **(customer_hierarchy.entities() if customer_hierarchy is not None else {}), "agent_id": payload["agent_id"], "proxy_budget_name": litellm_proxy_budget_name if litellm.max_budget > 0 else None, }, model_access_groups=debitable_model_access_groups( get_request_model_access_groups(request_fields), payload["model_id"], llm_router ), - windows=await accounting_window_keys(hashed_token, team_id) if billable_cost else (), + windows=await accounting_window_keys( + hashed_token, + team_id, + customer_counters + if customer_counters is not None + else await customer_accounting_counters( + customer_hierarchy, customer_budget_scopes_from_request(request_fields) + ), + ) + if billable_cost + else (), router=get_llm_router, ) keep_spend_logs: Final = disable_spend_logs is False and not ( @@ -891,6 +949,7 @@ class DBSpendUpdateWriter: litellm_proxy_budget_name=litellm_proxy_budget_name, payload=payload, request_model_access_groups=get_request_model_access_groups(request_fields), + customer_hierarchy=customer_hierarchy, ), task_name="spend rollups", ) @@ -1357,6 +1416,7 @@ class DBSpendUpdateWriter: payload: SpendLogsPayload, request_model_access_groups: Sequence[str] = (), project_id: str | None = None, + customer_hierarchy: CustomerHierarchyIds | None = None, ): """ Runs every spend-update helper for one request. Each helper goes through @@ -1400,6 +1460,13 @@ class DBSpendUpdateWriter: self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client), payload=payload_copy, ) + await self._run_spend_update_step( + "_update_customer_hierarchy_db", + self._update_customer_hierarchy_db( + response_cost=response_cost, customer_hierarchy=customer_hierarchy, prisma_client=prisma_client + ), + payload=payload_copy, + ) await self._run_spend_update_step( "_update_tag_db", self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client), @@ -1412,6 +1479,13 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, router=get_llm_router(), ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_customer_member_transaction", + self.add_spend_log_transaction_to_daily_customer_member_transaction( + payload_copy, customer_hierarchy, prisma_client + ), + payload=payload_copy, + ) await self._run_spend_update_step( "_update_agent_db", self._update_agent_db( @@ -1589,6 +1663,19 @@ class DBSpendUpdateWriter: ) raise e + async def _update_customer_hierarchy_db( + self, + response_cost: float | None, + customer_hierarchy: CustomerHierarchyIds | None, + prisma_client: PrismaClient | None, + ) -> None: + if customer_hierarchy is None or prisma_client is None: + return + for entity_type, entity_id in customer_hierarchy.spend_entities(): + await self.spend_update_queue.add_update( + update=SpendUpdateQueueItem(entity_type=entity_type, entity_id=entity_id, response_cost=response_cost) + ) + async def _update_org_db( self, response_cost: float | None, @@ -1860,6 +1947,7 @@ class DBSpendUpdateWriter: daily_org_spend_update_queue=self.daily_org_spend_update_queue, daily_end_user_spend_update_queue=self.daily_end_user_spend_update_queue, daily_agent_spend_update_queue=self.daily_agent_spend_update_queue, + daily_customer_member_spend_update_queue=self.daily_customer_member_spend_update_queue, window_spend_update_queue=self.window_spend_update_queue, ) @@ -1881,6 +1969,7 @@ class DBSpendUpdateWriter: daily_end_user_spend_update_transactions, daily_agent_spend_update_transactions, window_spend_update_transactions, + daily_customer_member_spend_update_transactions, ) = await self.redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline() uncommitted = { @@ -1914,6 +2003,13 @@ class DBSpendUpdateWriter: if daily_agent_spend_update_transactions is None else {"daily_agent_spend_update_transactions": daily_agent_spend_update_transactions} ), + "daily_customer_member_spend_update_transactions": ( + {} + if daily_customer_member_spend_update_transactions is None + else { + "daily_customer_member_spend_update_transactions": daily_customer_member_spend_update_transactions + } + ), "window_spend_update_transactions": ( {} if window_spend_update_transactions is None @@ -1925,7 +2021,7 @@ class DBSpendUpdateWriter: verbose_proxy_logger.info( "Spend tracking - committing spend updates from Redis to DB: " "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, org_members=%d, " - "projects=%d, tags=%d, agents=%d, model_access_groups=%d", + "projects=%d, tags=%d, agents=%d, model_access_groups=%d, customer_members=%d", len(db_spend_update_transactions.get("key_list_transactions") or ()), len(db_spend_update_transactions.get("user_list_transactions") or ()), len(db_spend_update_transactions.get("team_list_transactions") or ()), @@ -1937,6 +2033,7 @@ class DBSpendUpdateWriter: len(db_spend_update_transactions.get("tag_list_transactions") or ()), len(db_spend_update_transactions.get("agent_list_transactions") or ()), len(db_spend_update_transactions.get("model_access_group_list_transactions") or ()), + len(db_spend_update_transactions.get("customer_member_list_transactions") or ()), ) try: await self._commit_spend_updates_to_db( @@ -2000,6 +2097,15 @@ class DBSpendUpdateWriter: daily_spend_transactions=daily_agent_spend_update_transactions, ) uncommitted.pop("daily_agent_spend_update_transactions", None) + + if daily_customer_member_spend_update_transactions is not None: + await DBSpendUpdateWriter.update_daily_customer_member_spend( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + daily_spend_transactions=daily_customer_member_spend_update_transactions, + ) + uncommitted.pop("daily_customer_member_spend_update_transactions", None) if window_spend_update_transactions is not None: try: await DBSpendUpdateWriter._commit_window_spend_updates( @@ -2036,7 +2142,7 @@ class DBSpendUpdateWriter: async def _flush_daily_spend_queue( self, queue: DailySpendUpdateQueue, - entity_type: Literal["user", "team", "org", "tag", "end_user", "agent"], + entity_type: Literal["user", "team", "org", "tag", "end_user", "agent", "customer_member"], commit: _DailySpendCommit[_DailySpendTransactionT], n_retry_times: int, prisma_client: PrismaClient, @@ -2159,6 +2265,15 @@ class DBSpendUpdateWriter: proxy_logging_obj=proxy_logging_obj, ) + await self._flush_daily_spend_queue( + queue=self.daily_customer_member_spend_update_queue, + entity_type="customer_member", + commit=DBSpendUpdateWriter.update_daily_customer_member_spend, + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + ) + ################## Budget Window Spend Update Transactions ################## # Aggregate all in memory budget window spend transactions and commit to db window_spend_update_transactions: Final = ( @@ -2699,6 +2814,105 @@ class DBSpendUpdateWriter: if on_table_committed is not None: on_table_committed("agent_list_transactions") + await self._commit_customer_hierarchy_spend( + db_spend_update_transactions, + prisma_client=prisma_client, + n_retry_times=n_retry_times, + proxy_logging_obj=proxy_logging_obj, + outer_transaction=outer_transaction, + on_table_committed=on_table_committed, + ) + + @staticmethod + async def _commit_customer_hierarchy_spend( + db_spend_update_transactions: DBSpendUpdateTransactions, + *, + prisma_client: PrismaClient, + n_retry_times: int, + proxy_logging_obj: ProxyLogging, + outer_transaction: _SpendTransaction | None, + on_table_committed: Callable[[_SpendTableName], None] | None, + ) -> None: + """Increment the customer org, team and member rows a request ran under, one table per commit so a + partial failure re-sends only the tables still owed.""" + for level in _CUSTOMER_HIERARCHY_LEVELS: + await DBSpendUpdateWriter._commit_customer_hierarchy_level( + level, + db_spend_update_transactions.get(level.transactions), + prisma_client=prisma_client, + n_retry_times=n_retry_times, + proxy_logging_obj=proxy_logging_obj, + outer_transaction=outer_transaction, + ) + if on_table_committed is not None: + on_table_committed(level.transactions) + if outer_transaction is None and db_spend_update_transactions.get(level.transactions): + await CustomerBudgetAlerting.detect_committed_batch( + {level.transactions: db_spend_update_transactions.get(level.transactions)}, + prisma_client, + proxy_logging_obj.internal_usage_cache.dual_cache, + ) + + @staticmethod + async def _commit_customer_hierarchy_level( + level: _CustomerHierarchyLevel, + transactions: Mapping[str, float] | None, + *, + prisma_client: PrismaClient, + n_retry_times: int, + proxy_logging_obj: ProxyLogging, + outer_transaction: _SpendTransaction | None, + ) -> None: + if not transactions: + return + for attempt in range(n_retry_times + 1): + if await DBSpendUpdateWriter._commit_customer_hierarchy_level_once( + level, + transactions, + attempt=attempt, + prisma_client=prisma_client, + n_retry_times=n_retry_times, + proxy_logging_obj=proxy_logging_obj, + outer_transaction=outer_transaction, + ): + return + + @staticmethod + async def _commit_customer_hierarchy_level_once( + level: _CustomerHierarchyLevel, + transactions: Mapping[str, float], + *, + attempt: int, + prisma_client: PrismaClient, + n_retry_times: int, + proxy_logging_obj: ProxyLogging, + outer_transaction: _SpendTransaction | None, + ) -> bool: + """One attempt at the level's increments; False means the attempt failed and was reported for retry.""" + start_time: Final = time.time() + try: + async with ( + _spend_update_tx(prisma_client, level.table, outer_transaction=outer_transaction) as tx, + tx.batch_() as batcher, + ): + for entity_id, response_cost in sorted(transactions.items()): + getattr(batcher, level.prisma_table).update_many( + where={level.id_column: entity_id}, + data={"spend": {"increment": response_cost}}, + ) + except Exception as e: + if outer_transaction is not None: + raise + await DBSpendUpdateWriter.handle_spend_update_failure( + e=e, + attempt=attempt, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, + ) + return False + return True + @staticmethod @with_service_target(AUTH_OBJECTS_TARGET) async def _invalidate_project_caches(project_ids: Sequence[str], proxy_logging_obj: ProxyLogging | None) -> None: @@ -2830,6 +3044,18 @@ class DBSpendUpdateWriter: ) -> None: ... + @overload + @staticmethod + async def _update_daily_spend( + n_retry_times: int, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + daily_spend_transactions: dict[str, DailyCustomerMemberSpendTransaction], # mutable-ok: shared writer removes committed batches for safe requeue + entity_type: Literal["customer_member"], + entity_id_field: str, + ) -> None: + ... + @overload @staticmethod async def _update_daily_spend( @@ -2853,8 +3079,9 @@ class DBSpendUpdateWriter: | dict[str, DailyTagSpendTransaction] | dict[str, DailyOrganizationSpendTransaction] | dict[str, DailyEndUserSpendTransaction] - | dict[str, DailyAgentSpendTransaction], - entity_type: Literal["user", "team", "org", "tag", "end_user", "agent"], + | dict[str, DailyAgentSpendTransaction] + | dict[str, DailyCustomerMemberSpendTransaction], + entity_type: Literal["user", "team", "org", "tag", "end_user", "agent", "customer_member"], entity_id_field: str, ) -> None: """ @@ -3074,6 +3301,24 @@ class DBSpendUpdateWriter: entity_id_field="agent_id", ) + @staticmethod + async def update_daily_customer_member_spend( + n_retry_times: int, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + daily_spend_transactions: dict[ # mutable-ok: shared writer removes committed batches for safe requeue + str, DailyCustomerMemberSpendTransaction + ], + ) -> None: + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + daily_spend_transactions=daily_spend_transactions, + entity_type="customer_member", + entity_id_field="customer_member_id", + ) + @staticmethod async def update_daily_tag_spend( n_retry_times: int, @@ -3120,7 +3365,7 @@ class DBSpendUpdateWriter: self, payload: dict | SpendLogsPayload, prisma_client: PrismaClient, - type: Literal["user", "team", "org", "request_tags", "end_user", "agent"] = "user", + type: Literal["user", "team", "org", "request_tags", "end_user", "agent", "customer_member"] = "user", ) -> BaseDailySpendTransaction | None: entity: Final = "tag" if type == "request_tags" else type identity_payload: Final = ( @@ -3350,6 +3595,38 @@ class DBSpendUpdateWriter: daily_transaction: Final = DailyEndUserSpendTransaction(end_user_id=end_user_id, **base_daily_transaction) await self.daily_end_user_spend_update_queue.add_update(update={daily_transaction_key: daily_transaction}) + async def add_spend_log_transaction_to_daily_customer_member_transaction( + self, + payload: SpendLogsPayload, + customer_hierarchy: CustomerHierarchyIds | None, + prisma_client: PrismaClient | None = None, + ) -> None: + if prisma_client is None or customer_hierarchy is None: + return + member_id: Final = customer_hierarchy.customer_member_id or "" + team_id: Final = customer_hierarchy.customer_team_id or "" + org_id: Final = customer_hierarchy.customer_org_id + base: Final = await self._common_add_spend_log_transaction_to_daily_transaction( # pyright: ignore[reportUnknownMemberType] # existing helper accepts legacy untyped payloads + {**payload, "customer_member_id": member_id, "customer_team_id": team_id, "customer_org_id": org_id}, + prisma_client, + "customer_member", + ) + if base is None: + return + date: Final = utc_date_from_start_time(payload["startTime"]) + if date is None: + return + transaction: Final[DailyCustomerMemberSpendTransaction] = { + **base, + "date": date, + "customer_member_id": member_id, + "customer_team_id": team_id, + "customer_org_id": org_id, + } + table: Final = DAILY_SPEND_TABLES["customer_member"] + key: Final = json.dumps(conflict_key(table, transaction)) + await self.daily_customer_member_spend_update_queue.add_update({key: transaction}) + async def add_spend_log_transaction_to_daily_agent_transaction( self, payload: SpendLogsPayload, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 459f95972ce..cb3e1371e42 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -13,7 +13,6 @@ from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import TypeAdapter -from redis.exceptions import RedisError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -21,6 +20,7 @@ from litellm.caching import RedisCache from litellm.constants import ( MAX_REDIS_BUFFER_DEQUEUE_COUNT, REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY, + REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY, REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY, REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY, REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY, @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ( BaseDailySpendTransaction, DailyAgentSpendTransaction, + DailyCustomerMemberSpendTransaction, DailyEndUserSpendTransaction, DailyOrganizationSpendTransaction, DailyTagSpendTransaction, @@ -89,6 +90,9 @@ _SpendTransactionField: TypeAlias = Literal[ "tag_list_transactions", "agent_list_transactions", "model_access_group_list_transactions", + "customer_org_list_transactions", + "customer_team_list_transactions", + "customer_member_list_transactions", ] _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = ( @@ -103,6 +107,9 @@ _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = ( "tag_list_transactions", "agent_list_transactions", "model_access_group_list_transactions", + "customer_org_list_transactions", + "customer_team_list_transactions", + "customer_member_list_transactions", ) _ValueT = TypeVar("_ValueT") @@ -244,6 +251,7 @@ class RedisUpdateBuffer: daily_org_spend_update_queue: DailySpendUpdateQueue, daily_end_user_spend_update_queue: DailySpendUpdateQueue, daily_agent_spend_update_queue: DailySpendUpdateQueue, + daily_customer_member_spend_update_queue: DailySpendUpdateQueue, window_spend_update_queue: WindowSpendUpdateQueue | None = None, ): """ @@ -313,6 +321,9 @@ class RedisUpdateBuffer: daily_agent_spend_update_transactions: Final = ( await daily_agent_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() ) + daily_customer_member_spend_update_transactions: Final = ( + await daily_customer_member_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) window_spend_update_transactions: Final = ( await window_spend_update_queue.flush_and_get_aggregated_window_spend_transactions() if window_spend_update_queue is not None @@ -354,6 +365,11 @@ class RedisUpdateBuffer: REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY, ServiceTypes.REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE, ), + ( + daily_customer_member_spend_update_transactions, + REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY, + ServiceTypes.REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_QUEUE, + ), ( tuple(map(to_wire_payload, window_spend_update_transactions)), REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY, @@ -399,6 +415,7 @@ class RedisUpdateBuffer: daily_org_spend_update_transactions=daily_org_spend_update_transactions, daily_end_user_spend_update_transactions=daily_end_user_spend_update_transactions, daily_agent_spend_update_transactions=daily_agent_spend_update_transactions, + daily_customer_member_spend_update_transactions=daily_customer_member_spend_update_transactions, window_spend_update_transactions=window_spend_update_transactions, spend_update_queue=spend_update_queue, daily_spend_update_queue=daily_spend_update_queue, @@ -406,6 +423,7 @@ class RedisUpdateBuffer: daily_org_spend_update_queue=daily_org_spend_update_queue, daily_end_user_spend_update_queue=daily_end_user_spend_update_queue, daily_agent_spend_update_queue=daily_agent_spend_update_queue, + daily_customer_member_spend_update_queue=daily_customer_member_spend_update_queue, window_spend_update_queue=window_spend_update_queue, ) return @@ -426,6 +444,7 @@ class RedisUpdateBuffer: daily_org_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None, daily_end_user_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None, daily_agent_spend_update_transactions: dict[str, BaseDailySpendTransaction] | None, + daily_customer_member_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None, window_spend_update_transactions: tuple[WindowSpendTransaction, ...] | None, spend_update_queue: SpendUpdateQueue, daily_spend_update_queue: DailySpendUpdateQueue, @@ -433,6 +452,7 @@ class RedisUpdateBuffer: daily_org_spend_update_queue: DailySpendUpdateQueue, daily_end_user_spend_update_queue: DailySpendUpdateQueue, daily_agent_spend_update_queue: DailySpendUpdateQueue, + daily_customer_member_spend_update_queue: DailySpendUpdateQueue, window_spend_update_queue: WindowSpendUpdateQueue | None, ) -> None: """ @@ -443,7 +463,7 @@ class RedisUpdateBuffer: because the source queues were already drained before the rpush. """ if db_spend_update_transactions is not None: - entity_entries: Final[list[tuple[Litellm_EntityType, dict[str, float] | None]]] = [ + entity_entries: Final[list[tuple[Litellm_EntityType, Mapping[str, float] | None]]] = [ ( Litellm_EntityType.USER, db_spend_update_transactions.get("user_list_transactions"), @@ -488,6 +508,18 @@ class RedisUpdateBuffer: Litellm_EntityType.MODEL_ACCESS_GROUP, db_spend_update_transactions.get("model_access_group_list_transactions"), ), + ( + Litellm_EntityType.CUSTOMER_ORG, + db_spend_update_transactions.get("customer_org_list_transactions"), + ), + ( + Litellm_EntityType.CUSTOMER_TEAM, + db_spend_update_transactions.get("customer_team_list_transactions"), + ), + ( + Litellm_EntityType.CUSTOMER_MEMBER, + db_spend_update_transactions.get("customer_member_list_transactions"), + ), ] for entity_type, entities in entity_entries: if not entities: @@ -501,7 +533,7 @@ class RedisUpdateBuffer: ) ) - daily_pairs: Final[list[tuple[dict[str, BaseDailySpendTransaction] | None, DailySpendUpdateQueue]]] = [ + daily_pairs: Final[list[tuple[Mapping[str, BaseDailySpendTransaction] | None, DailySpendUpdateQueue]]] = [ (daily_spend_update_transactions, daily_spend_update_queue), (daily_team_spend_update_transactions, daily_team_spend_update_queue), (daily_org_spend_update_transactions, daily_org_spend_update_queue), @@ -510,10 +542,11 @@ class RedisUpdateBuffer: daily_end_user_spend_update_queue, ), (daily_agent_spend_update_transactions, daily_agent_spend_update_queue), + (daily_customer_member_spend_update_transactions, daily_customer_member_spend_update_queue), ] for daily_txns, daily_queue in daily_pairs: if daily_txns: - await daily_queue.update_queue.put(daily_txns) + await daily_queue.update_queue.put(dict(daily_txns)) if window_spend_update_transactions and window_spend_update_queue is not None: await window_spend_update_queue.update_queue.put(window_spend_update_transactions) @@ -527,6 +560,7 @@ class RedisUpdateBuffer: daily_org_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, daily_end_user_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, daily_agent_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, + daily_customer_member_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, daily_tag_spend_update_transactions: Mapping[str, BaseDailySpendTransaction] | None = None, window_spend_update_transactions: Sequence[WindowSpendTransaction] | None = None, ) -> None: @@ -549,6 +583,7 @@ class RedisUpdateBuffer: (daily_org_spend_update_transactions, REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY), (daily_end_user_spend_update_transactions, REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY), (daily_agent_spend_update_transactions, REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY), + (daily_customer_member_spend_update_transactions, REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY), (daily_tag_spend_update_transactions, REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY), ( None @@ -565,6 +600,7 @@ class RedisUpdateBuffer: ) if len(rpush_list) == 0: return + from redis.exceptions import RedisError try: await self.redis_cache.async_rpush_pipeline(rpush_list=rpush_list) @@ -584,6 +620,8 @@ class RedisUpdateBuffer: """Move this worker's pending tool rows to Redis so only the leader writes LiteLLM_ToolTable.""" if self.redis_cache is None: return + from redis.exceptions import RedisError + items: Final = tool_discovery_queue.flush() if not items: return @@ -771,11 +809,12 @@ class RedisUpdateBuffer: dict[str, DailyEndUserSpendTransaction] | None, dict[str, DailyAgentSpendTransaction] | None, tuple[WindowSpendTransaction, ...] | None, + dict[str, DailyCustomerMemberSpendTransaction] | None, ]: """ - Drains the main 7 Redis buffer queues in a single pipeline round-trip. + Drains the main 8 Redis buffer queues in a single pipeline round-trip. - Returns a 7-tuple of parsed results in this order: + Returns an 8-tuple of parsed results in this order: 0: DBSpendUpdateTransactions 1: daily user spend 2: daily team spend @@ -783,9 +822,10 @@ class RedisUpdateBuffer: 4: daily end-user spend 5: daily agent spend 6: budget window spend + 7: daily customer member spend """ if self.redis_cache is None: - return None, None, None, None, None, None, None + return None, None, None, None, None, None, None, None lpop_list: Final[list[RedisPipelineLpopOperation]] = [ RedisPipelineLpopOperation(key=REDIS_UPDATE_BUFFER_KEY, count=MAX_REDIS_BUFFER_DEQUEUE_COUNT), @@ -813,12 +853,16 @@ class RedisUpdateBuffer: key=REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY, count=MAX_REDIS_BUFFER_DEQUEUE_COUNT, ), + RedisPipelineLpopOperation( + key=REDIS_DAILY_CUSTOMER_MEMBER_SPEND_UPDATE_BUFFER_KEY, + count=MAX_REDIS_BUFFER_DEQUEUE_COUNT, + ), ] raw_results: Final = await self.redis_cache.async_lpop_pipeline(lpop_list=lpop_list) # Pad with None if pipeline returned fewer results than expected - while len(raw_results) < 7: + while len(raw_results) < 8: raw_results.append(None) # Slot 0: DBSpendUpdateTransactions @@ -830,7 +874,7 @@ class RedisUpdateBuffer: # Slots 1-5: daily spend categories daily_results: Final[list[dict[str, BaseDailySpendTransaction] | None]] = [] - for slot in range(1, 6): + for slot in (1, 2, 3, 4, 5, 7): slot_result = raw_results[slot] if slot_result is None: daily_results.append(None) @@ -855,6 +899,9 @@ class RedisUpdateBuffer: cast(dict[str, DailyEndUserSpendTransaction] | None, daily_results[3]), cast(dict[str, DailyAgentSpendTransaction] | None, daily_results[4]), window_spend, + cast( # cast-ok: dedicated customer queue contains the three validated identity fields + dict[str, DailyCustomerMemberSpendTransaction] | None, daily_results[5] + ), ) async def store_in_memory_daily_tag_spend_updates_in_redis( @@ -1049,6 +1096,15 @@ class RedisUpdateBuffer: model_access_group_list_transactions=_merged_entity_transactions( list_of_transactions, "model_access_group_list_transactions" ), + customer_org_list_transactions=_merged_entity_transactions( + list_of_transactions, "customer_org_list_transactions" + ), + customer_team_list_transactions=_merged_entity_transactions( + list_of_transactions, "customer_team_list_transactions" + ), + customer_member_list_transactions=_merged_entity_transactions( + list_of_transactions, "customer_member_list_transactions" + ), ) async def _emit_new_item_added_to_redis_buffer_event( diff --git a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py index e88b9fc2bfb..cef57f5a92c 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py @@ -145,6 +145,9 @@ class SpendUpdateQueue(BaseUpdateQueue): tag_list_transactions={}, agent_list_transactions={}, model_access_group_list_transactions={}, + customer_org_list_transactions={}, + customer_team_list_transactions={}, + customer_member_list_transactions={}, ) # Map entity types to their corresponding transaction dictionary keys @@ -160,6 +163,9 @@ class SpendUpdateQueue(BaseUpdateQueue): Litellm_EntityType.TAG: "tag_list_transactions", Litellm_EntityType.AGENT: "agent_list_transactions", Litellm_EntityType.MODEL_ACCESS_GROUP: "model_access_group_list_transactions", + Litellm_EntityType.CUSTOMER_ORG: "customer_org_list_transactions", + Litellm_EntityType.CUSTOMER_TEAM: "customer_team_list_transactions", + Litellm_EntityType.CUSTOMER_MEMBER: "customer_member_list_transactions", } for update in updates: @@ -205,6 +211,12 @@ class SpendUpdateQueue(BaseUpdateQueue): transactions_dict = db_spend_update_transactions["agent_list_transactions"] elif dict_key == "model_access_group_list_transactions": transactions_dict = db_spend_update_transactions["model_access_group_list_transactions"] + elif dict_key == "customer_org_list_transactions": + transactions_dict = db_spend_update_transactions["customer_org_list_transactions"] + elif dict_key == "customer_team_list_transactions": + transactions_dict = db_spend_update_transactions["customer_team_list_transactions"] + elif dict_key == "customer_member_list_transactions": + transactions_dict = db_spend_update_transactions["customer_member_list_transactions"] else: continue diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 33d973cccfb..d9d0ece4468 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -776,8 +776,17 @@ def db_write_failure_is_requeue_safe(e: Exception) -> bool: def db_write_never_sent(e: Exception) -> bool: - """Whether the database could not take the write at all, because it was unreachable (P1001), - refusing connections (57P03) or full (53300, P2024), so even a non-idempotent write can be sent again.""" + """Whether the database could not take the write at all, because the client had no query engine to send it + to (ClientNotConnectedError, NotConnectedError, ConnectError), it was unreachable (P1001), refusing connections + (57P03) or full (53300, P2024), so even a non-idempotent write can be sent again.""" + import prisma + import prisma.engine.errors + + unsent_types: Final = _exception_types( + prisma.errors.ClientNotConnectedError, prisma.engine.errors.NotConnectedError + ) + if isinstance(e, (*DB_RETRY_SAFE_ERROR_TYPES, *unsent_types)): + return True if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e): return True if PrismaDBExceptionHandler.is_database_capacity_error(e): diff --git a/litellm/proxy/db/model_usage_rollup.py b/litellm/proxy/db/model_usage_rollup.py index 6b8f5bfa70e..a17232394a8 100644 --- a/litellm/proxy/db/model_usage_rollup.py +++ b/litellm/proxy/db/model_usage_rollup.py @@ -4,7 +4,7 @@ import asyncio import random from collections.abc import Mapping, Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from itertools import groupby from types import TracebackType from typing import TYPE_CHECKING, Final, Protocol @@ -97,6 +97,17 @@ def _internal_call_origin(metadata: str) -> object: return decoded.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) +def utc_date_from_start_time(start_time: datetime | str) -> str | None: + try: + parsed: Final = ( + datetime.fromisoformat(start_time.replace("Z", "+00:00")) if isinstance(start_time, str) else start_time + ) + except ValueError: + return None + aware: Final = parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed + return aware.astimezone(timezone.utc).date().isoformat() + + def date_from_start_time(start_time: datetime | str) -> str | None: if isinstance(start_time, datetime): return start_time.date().isoformat() diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index e0be2e1e5ef..f90a9f72bf4 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -74,6 +74,15 @@ if TYPE_CHECKING: def delete_many(self, where: Mapping[str, object] | None = None) -> None: ... class PrismaBatchSession(Protocol): + @property + def litellm_customermembertable(self) -> PrismaBatchActions: ... + + @property + def litellm_customerteamtable(self) -> PrismaBatchActions: ... + + @property + def litellm_customerorganizationtable(self) -> PrismaBatchActions: ... + @property def litellm_agentstable(self) -> PrismaBatchActions: ... @@ -1281,6 +1290,7 @@ class PrismaManager: from litellm_proxy_extras.utils import ProxyExtrasDBManager ProxyExtrasDBManager.raise_if_lens_rename_pending() + ProxyExtrasDBManager.raise_if_customer_member_budget_rename_pending() # pyright: ignore[reportUnknownMemberType] # litellm-proxy-extras is an optional install PrismaManager._raise_if_partitioned_spend_logs() run_prisma( [ diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index ae45b43f15f..00cb35866f8 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -13,6 +13,7 @@ memory in long-lived deployments. """ import asyncio +import re from collections import OrderedDict from collections.abc import Mapping from datetime import datetime, timezone @@ -25,15 +26,20 @@ from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import Litellm_EntityType from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate +from litellm.proxy.management_helpers.customer_hierarchy import customer_row_reset_at from litellm.proxy.spend_tracking.spend_counter_batch import ( read_batched_spend_counter, record_spend_counter_value, ) from litellm.redis.stores.spend_counters import END_USER_COUNTER_PREFIX, SPEND_COUNTERS_TARGET, SpendCounterStore +from litellm.redis.stores.spend_counters.types import parse_customer_window_counter from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ( BudgetWindowSpendRepository, + CustomerMemberRepository, + CustomerOrganizationRepository, + CustomerTeamRepository, EndUserRepository, SpendLogsRepository, TeamMembershipRepository, @@ -45,6 +51,7 @@ from litellm.repositories.verification_token_repository import ( ) if TYPE_CHECKING: + from prisma.models import LiteLLM_BudgetWindowSpend from prisma.types import LiteLLM_EndUserTableWhereUniqueInput from litellm.proxy.utils import PrismaClient @@ -54,6 +61,9 @@ _WINDOW_SPEND_ENTITY_TYPES: Final[Mapping[str, str]] = MappingProxyType( { "Key": Litellm_EntityType.KEY.value, "Team": Litellm_EntityType.TEAM.value, + "customer_org": Litellm_EntityType.CUSTOMER_ORG.value, + "customer_team": Litellm_EntityType.CUSTOMER_TEAM.value, + "customer_member": Litellm_EntityType.CUSTOMER_MEMBER.value, } ) @@ -69,6 +79,34 @@ def _as_utc(value: datetime) -> datetime: return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc) +def recorded_window_spend( + row: "LiteLLM_BudgetWindowSpend | None", entity_type: str, expected_window_start: datetime | None +) -> float | None: + if expected_window_start is None: + return None + if row is None or _as_utc(row.window_start) < _as_utc(expected_window_start): + return 0.0 if entity_type.startswith("customer_") else None + return float(row.spend or 0.0) + + +_CUSTOMER_OVERRIDE_INCLUDE: Final = MappingProxyType({"budget": True, "budget_override": True}) +_CUSTOMER_COUNTER: Final = re.compile( + r"spend:(customer_org|customer_team|customer_member):(.+):" + r"(lifetime|\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})?)" +) + + +def _customer_counter_period_matches(row: object, period: str) -> bool: + try: + expected: Final = None if period == "lifetime" else datetime.fromisoformat(period) + except ValueError: + return False + actual: Final = customer_row_reset_at(row, datetime.now(timezone.utc)) + if expected is None or actual is None: + return expected is actual + return _as_utc(actual) == _as_utc(expected) + + class SpendCounterReseed: """ Reseeds spend counters from the authoritative DB and warms the cache, @@ -133,6 +171,10 @@ class SpendCounterReseed: """ if prisma_client is None: return None + customer_window: Final = parse_customer_window_counter(counter_key) + if customer_window is not None: + scope, entity_id, duration, start = customer_window + return await SpendCounterReseed.window_from_db(prisma_client, scope, entity_id, duration, start) # Per-window key/team counters share prefixes with primary counters # but don't correspond to a DB row. Do not reject arbitrary entity IDs # or tag names that merely contain ":window:". @@ -147,11 +189,28 @@ class SpendCounterReseed: return None if row is None: return None + customer: Final = _CUSTOMER_COUNTER.fullmatch(counter_key) + if customer is not None and not _customer_counter_period_matches(row, customer.group(3)): + return None return float(getattr(row, "spend", 0.0) or 0.0) @staticmethod async def _counter_row(prisma_client: "PrismaClient", counter_key: str) -> object | None: async with db_lookup_gate.current(): + customer: Final = _CUSTOMER_COUNTER.fullmatch(counter_key) + if customer is not None: + family, entity_id, _ = customer.groups() + if family == "customer_org": + return await CustomerOrganizationRepository(prisma_client).table.find_unique( + where={"customer_org_id": entity_id}, include={"budget": True} + ) + if family == "customer_team": + return await CustomerTeamRepository(prisma_client).table.find_unique( + where={"customer_team_id": entity_id}, include=_CUSTOMER_OVERRIDE_INCLUDE + ) + return await CustomerMemberRepository(prisma_client).table.find_unique( + where={"customer_member_id": entity_id}, include=_CUSTOMER_OVERRIDE_INCLUDE + ) if counter_key.startswith("spend:key:"): token: Final = counter_key[len("spend:key:") :] return await VerificationTokenRepository(prisma_client).table.find_unique(where={"token": token}) @@ -266,12 +325,14 @@ class SpendCounterReseed: async def _read_cached(spend_counter_store: SpendCounterStore, counter_key: str) -> float | None: """Re-check after acquiring the lock. A clean Redis miss, in the request's MGET or a GET, skips memory, which only holds this pod's writes.""" - batched: Final = await SpendCounterReseed._read_active_batch(counter_key) - if batched is not None: - return batched[0] - value, _ = await spend_counter_store.read(counter_key) + value, _ = await SpendCounterReseed._cached_reading(spend_counter_store, counter_key) return None if value is None else float(value) + @staticmethod + async def _cached_reading(spend_counter_store: SpendCounterStore, counter_key: str) -> tuple[float | None, bool]: + batched: Final = await SpendCounterReseed._read_active_batch(counter_key) + return batched if batched is not None else await spend_counter_store.read(counter_key) + @staticmethod async def window_from_table( prisma_client: Optional["PrismaClient"], @@ -319,11 +380,7 @@ class SpendCounterReseed: ) return None - if row is None: - return None - if _as_utc(row.window_start) < _as_utc(expected_window_start): - return None - return float(row.spend or 0.0) + return recorded_window_spend(row, entity_type, expected_window_start) @staticmethod async def window_from_db( @@ -407,10 +464,16 @@ class SpendCounterReseed: entity_id: str, window_duration: str | None, window_start: datetime, + *, + require_authoritative: bool = False, ) -> float | None: + strict: Final = require_authoritative or entity_type.startswith("customer_") lock: Final = await SpendCounterReseed._get_lock(counter_key) async with lock: - cached: Final = await SpendCounterReseed._read_cached(spend_counter_store, counter_key) + cached, authoritative = await SpendCounterReseed._cached_reading(spend_counter_store, counter_key) + if strict and spend_counter_store.has_redis and not authoritative: + # A pod-local value or durable floor omits other workers' holds and unflushed charges. + return None if cached is not None: return cached @@ -430,6 +493,8 @@ class SpendCounterReseed: "SpendCounterReseed.coalesced_window: failed to warm counter %s", counter_key, ) + if strict: + return None raise if spend_counter_store.has_redis: record_spend_counter_value(counter_key, float(seeded)) diff --git a/litellm/proxy/discovery_endpoints/skill_hub_mutations.py b/litellm/proxy/discovery_endpoints/skill_hub_mutations.py index 63c519923c6..fff81b8a90c 100644 --- a/litellm/proxy/discovery_endpoints/skill_hub_mutations.py +++ b/litellm/proxy/discovery_endpoints/skill_hub_mutations.py @@ -58,6 +58,15 @@ def submission_owner(caller: UserAPIKeyAuth) -> str | None | SkillMutationError: return caller.user_id +def submission_editor(caller: UserAPIKeyAuth) -> str | SkillMutationError: + error: Final = writer_error(caller) + if error is not None: + return error + if not caller.user_id or caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID: + return SkillMutationError(403, "Editing submissions requires the original submitting user's identity.") + return caller.user_id + + async def archive_writer(caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]) -> UserAPIKeyAuth: from litellm.proxy.proxy_server import proxy_logging_obj @@ -149,13 +158,19 @@ async def review_skill( approve: bool, reason: str | None, publication_base_url: str | None = None, + expected_revision: int | None = None, ) -> SubmissionReviewed | SkillMutationError: if not is_proxy_admin(caller): return SkillMutationError(403, "Only proxy admins may review skill submissions.") if submissions is None: return SkillMutationError(503, "Skills Hub requires a connected database") result: Final = await submissions.review( - submission_id, caller.user_id or "proxy-admin", approve, reason, publication_base_url=publication_base_url + submission_id, + caller.user_id or "proxy-admin", + approve, + reason, + publication_base_url=publication_base_url, + expected_revision=expected_revision, ) match result: case SubmissionMissing(): diff --git a/litellm/proxy/discovery_endpoints/skill_hub_submissions.py b/litellm/proxy/discovery_endpoints/skill_hub_submissions.py index c1bb422b2a0..884e15e72a0 100644 --- a/litellm/proxy/discovery_endpoints/skill_hub_submissions.py +++ b/litellm/proxy/discovery_endpoints/skill_hub_submissions.py @@ -1,27 +1,49 @@ import asyncio +from dataclasses import replace from typing import Annotated, Final -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response, UploadFile -from litellm.models.plugin_upload import PluginPublication -from litellm.models.skill_submission import SkillSubmission, SkillSubmissionStore, SubmissionStatus +from litellm.models.plugin_upload import PluginPublication, PluginPublicationError, PluginUploadStore +from litellm.models.skill_submission import ( + SkillSubmission, + SkillSubmissionStore, + SubmissionConflict, + SubmissionLimitExceeded, + SubmissionMissing, + SubmissionStatus, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import resolve_skill_visibility from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.deps.repositories import PluginUploadRepositoryDep, SkillSubmissionRepositoryDep from litellm.proxy.discovery_endpoints.agent_skills_archive import build_agent_skill_archive from litellm.proxy.discovery_endpoints.skill_hub_endpoints import ( + prepare_skill_publication, skill_metadata_response, skill_proposal_response, skill_publication_response, ) -from litellm.proxy.discovery_endpoints.skill_hub_mutations import SkillMutationError, review_skill, submission_owner +from litellm.proxy.discovery_endpoints.skill_hub_mutations import ( + SkillMutationError, + archive_writer, + publication_error, + review_skill, + submission_editor, + submission_owner, + write_target, +) +from litellm.proxy.discovery_endpoints.skill_import import MAX_IMPORT_BYTES +from litellm.repositories.plugin_publication import resolve_plugin_version from litellm.types.proxy.skill_hub import ( + SkillHubMetadata, SkillHubRejection, SkillHubReviewResponse, SkillHubSubmission, SkillHubSubmissionDetail, + SkillHubSubmissionEdit, SkillHubSubmissionList, + SkillHubSubmissionRevision, ) router: Final = APIRouter(prefix="/skill-hub/submissions", tags=["Skills Hub"]) @@ -40,7 +62,7 @@ def _owner(caller: UserAPIKeyAuth) -> str | None: return owner -def _response(request: Request, submission: SkillSubmission) -> SkillHubSubmission: +def _response(request: Request, submission: SkillSubmission, caller: UserAPIKeyAuth) -> SkillHubSubmission: return SkillHubSubmission( id=submission.id, skill_name=submission.plugin_name, @@ -55,6 +77,8 @@ def _response(request: Request, submission: SkillSubmission) -> SkillHubSubmissi reviewed_by=submission.reviewed_by, reviewed_at=submission.reviewed_at, rejection_reason=submission.rejection_reason, + revision=submission.revision, + can_edit=submission.status == "pending_review" and submission_editor(caller) == submission.submitted_by, ) @@ -71,7 +95,7 @@ async def list_submissions( owner: Final = _owner(caller) page: Final = await _store(submissions).list(owner, status, limit, offset) return SkillHubSubmissionList( - submissions=tuple(_response(request, row) for row in page.submissions), + submissions=tuple(_response(request, row, caller) for row in page.submissions), count=len(page.submissions), has_more=page.has_more, ) @@ -85,7 +109,7 @@ async def get_submission( uploads: PluginUploadRepositoryDep, caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> SkillHubSubmissionDetail: - """Read an immutable proposal and its currently visible published skill.""" + """Read a proposal revision and its currently visible published skill.""" owner: Final = _owner(caller) submission: Final = await _store(submissions).get(submission_id, owner) if submission is None: @@ -95,7 +119,7 @@ async def get_submission( current: Final = await uploads.get_metadata(submission.plugin_name) visible: Final = current is not None and (owner is None or (await resolve_skill_visibility(caller)).allows(current)) return SkillHubSubmissionDetail( - submission=_response(request, submission), + submission=_response(request, submission, caller), current_skill=skill_metadata_response(request, current) if current is not None and visible else None, ) @@ -143,11 +167,12 @@ async def _review( caller: UserAPIKeyAuth, approve: bool, reason: str | None, + expected_revision: int | None, ) -> SkillHubReviewResponse: from litellm.proxy.utils import get_custom_url result: Final = await review_skill( - submission_id, submissions, caller, approve, reason, get_custom_url(str(request.base_url)) + submission_id, submissions, caller, approve, reason, get_custom_url(str(request.base_url)), expected_revision ) if isinstance(result, SkillMutationError): raise HTTPException(result.status, result.message) @@ -164,7 +189,7 @@ async def _review( ) return SkillHubReviewResponse( action="approved" if approve else "rejected", - submission=_response(request, result.submission), + submission=_response(request, result.submission, caller), skill=None if result.plugin is None else skill_publication_response(request, publication, result.plugin), ) @@ -175,9 +200,12 @@ async def approve_submission( submission_id: str, submissions: SkillSubmissionRepositoryDep, caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + body: SkillHubSubmissionRevision | None = None, ) -> SkillHubReviewResponse: """Publish a pending proposal and record its approval atomically. Only proxy admins may review.""" - return await _review(request, submission_id, submissions, caller, True, None) + return await _review( + request, submission_id, submissions, caller, True, None, None if body is None else body.expected_revision + ) @router.put("/{submission_id}/reject", response_model=SkillHubReviewResponse) @@ -189,4 +217,133 @@ async def reject_submission( body: SkillHubRejection | None = None, ) -> SkillHubReviewResponse: """Reject a pending proposal without changing the published skill. Only proxy admins may review.""" - return await _review(request, submission_id, submissions, caller, False, None if body is None else body.reason) + return await _review( + request, + submission_id, + submissions, + caller, + False, + None if body is None else body.reason, + None if body is None else body.expected_revision, + ) + + +def _editor(caller: UserAPIKeyAuth) -> str: + owner: Final = submission_editor(caller) + if isinstance(owner, SkillMutationError): + raise HTTPException(owner.status, owner.message) + return owner + + +def _changed( + result: SkillSubmission | SubmissionMissing | SubmissionConflict | SubmissionLimitExceeded, +) -> SkillSubmission: + if isinstance(result, SubmissionMissing): + raise HTTPException(404, "Skill submission not found") + if isinstance(result, SubmissionConflict): + raise HTTPException(409, result.message) + if isinstance(result, SubmissionLimitExceeded): + raise HTTPException(429, result.message) + return result + + +async def _editable( + submission_id: str, + owner: str, + store: SkillSubmissionStore, + revision: int, + version: str | None, + uploads: PluginUploadStore | None, + caller: UserAPIKeyAuth, +) -> SkillSubmission: + submission: Final = await store.get(submission_id, owner) + if submission is None: + raise HTTPException(404, "Skill submission not found") + if submission.status != "pending_review" or submission.revision != revision: + raise HTTPException(409, "This submission changed or is no longer pending. Reload it before trying again.") + existing: Final = await write_target( + submission.plugin_name, submission.operation == "update", uploads, caller, submission.base_updated_at + ) + if isinstance(existing, SkillMutationError): + raise HTTPException(existing.status, existing.message) + validation: Final = resolve_plugin_version( + version, None if existing is None else existing.version, updating=existing is not None + ) + if isinstance(validation, PluginPublicationError): + error: Final = publication_error(validation) + raise HTTPException(error.status, error.message) + return submission + + +@router.put("/{submission_id}", response_model=SkillHubSubmission) +async def edit_submission( + request: Request, + submission_id: str, + body: SkillHubSubmissionEdit, + submissions: SkillSubmissionRepositoryDep, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubSubmission: + """Replace a pending proposal's metadata, preserving files. Only its original submitting user may edit.""" + owner: Final = _editor(caller) + store: Final = _store(submissions) + submission: Final = await _editable( + submission_id, owner, store, body.expected_revision, body.version, uploads, caller + ) + publication: Final = PluginPublication( + name=submission.plugin_name, + version=body.version, + description=body.description, + manifest={ + **{ + key: value + for key, value in submission.proposal.manifest.items() + if key not in SkillHubMetadata.model_fields + }, + **body.model_dump(exclude={"expected_revision"}, exclude_none=True), + }, + created_by=owner, + ) + result: Final = await store.edit(submission_id, owner, body.expected_revision, publication) + return _response(request, _changed(result), caller) + + +@router.put("/{submission_id}/upload", response_model=SkillHubSubmission) +async def edit_submission_archive( + request: Request, + submission_id: str, + submissions: SkillSubmissionRepositoryDep, + uploads: PluginUploadRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(archive_writer)], + file: Annotated[UploadFile, File(description="Replacement Agent Skills ZIP with the same declared skill name")], + expected_revision: Annotated[int, Form(ge=0)], + version: Annotated[str | None, Form()] = None, +) -> SkillHubSubmission: + """Replace a pending proposal's files using the upload limits. Only its original submitting user may edit.""" + owner: Final = _editor(caller) + store: Final = _store(submissions) + submission: Final = await _editable(submission_id, owner, store, expected_revision, version, uploads, caller) + content: Final = await file.read(MAX_IMPORT_BYTES + 1) + if len(content) > MAX_IMPORT_BYTES: + raise HTTPException(413, "Skill ZIP must be 10 MiB or smaller") + archive: Final = await asyncio.to_thread(build_agent_skill_archive, content) + if archive is None or archive.declared_name != submission.plugin_name: + raise HTTPException(422, "Replacement SKILL.md must declare the existing skill name and a description") + prepared: Final = await prepare_skill_publication(request, archive, version, None, owner) + publication: Final = replace(prepared, manifest={**submission.proposal.manifest, **prepared.manifest}) + result: Final = await store.edit(submission_id, owner, expected_revision, publication) + return _response(request, _changed(result), caller) + + +@router.put("/{submission_id}/withdraw", response_model=SkillHubSubmission) +async def withdraw_submission( + request: Request, + submission_id: str, + body: SkillHubSubmissionRevision, + submissions: SkillSubmissionRepositoryDep, + caller: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> SkillHubSubmission: + """Withdraw a pending proposal without publishing it. Only its original submitting user may withdraw.""" + owner: Final = _editor(caller) + result: Final = await _store(submissions).withdraw(submission_id, owner, body.expected_revision) + return _response(request, _changed(result), caller) diff --git a/litellm/proxy/google_endpoints/endpoints.py b/litellm/proxy/google_endpoints/endpoints.py index 31b88e8110c..df0c91dfde2 100644 --- a/litellm/proxy/google_endpoints/endpoints.py +++ b/litellm/proxy/google_endpoints/endpoints.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, JsonValue, T from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types from litellm.proxy._types import * from litellm.proxy._types import ProxyException, TokenCountRequest +from litellm.proxy.auth.auth_utils import MODEL_ROUTING_HEADER_NAME from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -487,6 +488,10 @@ async def create_interaction( ) +def _interaction_model_group(request: Request) -> str | None: + return request.query_params.get("model") or request.headers.get(MODEL_ROUTING_HEADER_NAME) + + @router.get( "/v1beta/interactions/{interaction_id}", dependencies=[Depends(user_api_key_auth)], @@ -524,7 +529,8 @@ async def get_interaction( version, ) - data: Final = {"interaction_id": interaction_id, "custom_llm_provider": "gemini"} + model: Final = _interaction_model_group(request) + data: Final = {"interaction_id": interaction_id, "custom_llm_provider": "gemini", "model": model} processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: @@ -538,7 +544,7 @@ async def get_interaction( general_settings=general_settings, proxy_config=proxy_config, select_data_generator=select_data_generator, - model=None, + model=model, user_model=user_model, user_temperature=user_temperature, user_request_timeout=user_request_timeout, @@ -592,7 +598,8 @@ async def delete_interaction( version, ) - data: Final = {"interaction_id": interaction_id, "custom_llm_provider": "gemini"} + model: Final = _interaction_model_group(request) + data: Final = {"interaction_id": interaction_id, "custom_llm_provider": "gemini", "model": model} processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: @@ -606,7 +613,7 @@ async def delete_interaction( general_settings=general_settings, proxy_config=proxy_config, select_data_generator=select_data_generator, - model=None, + model=model, user_model=user_model, user_temperature=user_temperature, user_request_timeout=user_request_timeout, @@ -660,7 +667,8 @@ async def cancel_interaction( version, ) - data: Final = {"interaction_id": interaction_id, "custom_llm_provider": "gemini"} + model: Final = _interaction_model_group(request) + data: Final = {"interaction_id": interaction_id, "custom_llm_provider": "gemini", "model": model} processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: @@ -674,7 +682,7 @@ async def cancel_interaction( general_settings=general_settings, proxy_config=proxy_config, select_data_generator=select_data_generator, - model=None, + model=model, user_model=user_model, user_temperature=user_temperature, user_request_timeout=user_request_timeout, diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 5a6d6fc4cdf..e201ccbe5c1 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -3,45 +3,73 @@ Dynamic rate limiter v3 - Saturation-aware priority-based rate limiting """ import os -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal from fastapi import HTTPException -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm import Router from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.cost_calculator import get_usage_object +from litellm.exceptions import ServiceUnavailableError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) +from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ( + SATURATION_FALLBACK_DESCRIPTOR_KEY, ProxyRateLimitError, + fallback_on_saturation_requested, map_v3_rate_limit_type, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import ( # noqa: F401 # legacy module exports + RATE_LIMIT_UNVERIFIABLE_MESSAGE, + BorrowingTeamModelHold, + BorrowingTeamModelLimits, PROXY_MaxParallelRequestsHandler_v3, RateLimitDescriptor, RateLimitDescriptorRateLimitObject, RateLimitedModel, RateLimitResponse, RateLimitStatus, + RateLimitUnverifiableError, + RequestRateLimiterStash, + ReservationOperation, + ReservationWindowIdentity, + WindowGuardedIncrement, _PROXY_MaxParallelRequestsHandler_v3, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _socket_turn_tpm_windows, # pyright: ignore[reportPrivateUsage] # same per-turn TPM window the regular limiter checks borrowing_team_model_limits, + callback_kwargs_mapping, claim_request_stash_for_data, get_or_create_request_stash, + get_request_stash, + request_stash_for_callback, +) +from litellm.proxy.hooks.priority_class_share import ( + DEFAULT_CLASS, + ActiveTeamRegistry, + TeamShare, + counter_value, + pool_member, + team_member, + team_share, ) from litellm.proxy.hooks.rate_limiter_utils import ( convert_priority_to_percent, resolve_llm_provider_for_rate_limit, + socket_session_usage, ) from litellm.proxy.utils import InternalUsageCache from litellm.redis.stores.rate_limits import RATE_LIMITS_TARGET, CounterIncrement, RateLimitCounter, RateLimitScope @@ -49,6 +77,7 @@ from litellm.types.router import ModelGroupInfo from litellm.types.utils import CallTypesLiteral, LLMResponseTypes, Usage if TYPE_CHECKING: + from litellm.integrations.prometheus import PrometheusLogger from litellm.types.utils import PriorityReservationSettings @@ -59,6 +88,7 @@ class _SettlementMetadata(BaseModel): user_api_key_auth_metadata: Mapping[str, object] | None = None user_api_key_team_id: str | None = None + user_api_key_team_alias: str | None = None @property def priority(self) -> str | None: @@ -66,12 +96,91 @@ class _SettlementMetadata(BaseModel): return value if isinstance(value, str) else None -def _usage_of(response_obj: object) -> Usage | None: +class _SettlementLoggingObject(BaseModel): + """The slice of a request's standard logging object that token settlement reads.""" + + model_config = ConfigDict(frozen=True) + + metadata: _SettlementMetadata | None = None + + +def _prometheus_loggers() -> "Sequence[PrometheusLogger]": + from litellm.integrations.prometheus import PrometheusLogger + + return tuple( + logger + for logger in litellm.logging_callback_manager.get_custom_loggers_for_type(PrometheusLogger) + if isinstance(logger, PrometheusLogger) + ) + + +def _tightest_team_share(descriptors: Iterable[RateLimitDescriptor | None]) -> tuple[RateLimitDescriptor, int] | None: + """The team share descriptor with the smallest TPM share, the one that binds first at saturation.""" + shares: Final = tuple( + (descriptor, tokens) + for descriptor in descriptors + if descriptor is not None and (tokens := (descriptor["rate_limit"] or {}).get("tokens_per_unit")) is not None + ) + return min(shares, key=lambda share: share[1], default=None) + + +def _tightest_of(*shares: tuple[RateLimitDescriptor, int] | None) -> tuple[RateLimitDescriptor, int] | None: + return min((share for share in shares if share is not None), key=lambda share: share[1], default=None) + + +def _share_on_model(share: RateLimitDescriptor | None, team: str, model_group: str) -> RateLimitDescriptor | None: + return share if share is not None and share["value"] == f"{team}:{model_group}" else None + + +def _capped_idle_share( + share: RateLimitDescriptor | None, cap: RateLimitDescriptor | None +) -> tuple[RateLimitDescriptor, int] | None: + """A capped team's usage while the model is idle sits on its cap counter, read against its share.""" + tightest: Final = _tightest_team_share((share,)) + if tightest is None or cap is None: + return None + return cap, tightest[1] + + +def _usage_of(kwargs: Mapping[str, object], response_obj: object) -> Usage | None: + """Usage of a modeled response, a Responses WebSocket turn's combined usage, or its raw event list.""" usage: Final = getattr(response_obj, "usage", None) - return usage if isinstance(usage, Usage) else get_usage_object(response_obj) + if isinstance(usage, Usage): + return usage + combined_usage: Final = kwargs.get("combined_usage_object") + if isinstance(combined_usage, Usage): + return combined_usage + socket_usage: Final = socket_session_usage(kwargs, response_obj) + return socket_usage if socket_usage is not None else get_usage_object(response_obj) + + +def _settled_tokens(usage: Usage | None, rate_limit_type: str) -> int: + if usage is None: + return 0 + match rate_limit_type: + case "output": + return usage.completion_tokens + case "input": + return usage.prompt_tokens + case _: + return usage.total_tokens TEAM_MODEL_LIMIT_AT_SATURATION: Final = "model_per_team_saturated" +PRIORITY_CLASS_SHARE_AT_SATURATION: Final = "priority_class_share_saturated" +TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "model_per_team_max_borrow_multiple" + + +@dataclass(frozen=True, slots=True) +class _BorrowingTeamShare: + """The team's share and, when the team caps how far it borrows, its ceiling, both on the share's counter.""" + + limits: BorrowingTeamModelLimits + share: RateLimitDescriptor + ceiling: RateLimitDescriptor | None + + def enforced(self, saturated: bool) -> RateLimitDescriptor | None: + return self.share if saturated else None def _team_limit_name(rate_limit_type: str) -> str: @@ -79,10 +188,10 @@ def _team_limit_name(rate_limit_type: str) -> str: def _team_model_token_increment( - descriptor: RateLimitDescriptor | None, total_tokens: int, ttl: int + descriptor: RateLimitDescriptor | None, amount: int, ttl: int ) -> tuple[CounterIncrement, ...]: - """Charge the team counter the pre-call hook tracked, and only when that team limit has a TPM share.""" - if descriptor is None: + """Move the team counter the pre-call hook tracked by ``amount``, and only when that team limit has a TPM share.""" + if descriptor is None or amount == 0: return () rate_limit: Final = descriptor["rate_limit"] if rate_limit is None or rate_limit.get("tokens_per_unit") is None: @@ -90,12 +199,70 @@ def _team_model_token_increment( return ( CounterIncrement( counter=RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens"), - amount=total_tokens, + amount=amount, ttl=ttl, ), ) +def _window_guarded( + operations: tuple[CounterIncrement, ...], window: ReservationWindowIdentity | None +) -> tuple[ReservationOperation, ...]: + """Apply a reservation's settlement only while the window it was held in is still open, never to a newer window.""" + if window is None: + return operations + return tuple( + WindowGuardedIncrement(increment=op, expected_window_start=window[1], reservation_backend=window[2]) + if op.counter == window[0] + else op + for op in operations + ) + + +def _team_model_settlement( + stash: RequestRateLimiterStash | None, model_group: str | None, total_tokens: int, ttl: int +) -> tuple[ReservationOperation, ...]: + """Settle the counter that served the call to actual usage and return the admitting reservation it did not use. + + The admitting team counter serves unless a fallback group did, in which case that fallback's hold serves and + the admitting reservation is returned. A hold on a fallback target that did not serve stays for that attempt's + own failure callback, which may still be reporting the usage it recorded. + """ + if stash is None: + return () + served: Final = stash.borrowing_team_model_fallback_limits.get(model_group) if model_group is not None else None + admitted: Final = stash.borrowing_team_model_limit + cap: Final = stash.borrowing_team_model_cap + reserved_tokens: Final = stash.borrowing_team_model_reserved_tokens + cap_reserved_tokens: Final = stash.borrowing_team_model_cap_reserved_tokens + window: Final = stash.borrowing_team_model_reserved_window + cap_window: Final = stash.borrowing_team_model_cap_reserved_window + if served is None: + return ( + *_window_guarded(_team_model_token_increment(admitted, total_tokens - reserved_tokens, ttl), window), + *_window_guarded(_team_model_token_increment(cap, total_tokens - cap_reserved_tokens, ttl), cap_window), + ) + return ( + *_window_guarded(_team_model_token_increment(admitted, -reserved_tokens, ttl), window), + *_window_guarded(_team_model_token_increment(cap, -cap_reserved_tokens, ttl), cap_window), + *_window_guarded( + _team_model_token_increment(served.descriptor, total_tokens - served.reserved_tokens, ttl), served.window + ), + *_window_guarded( + _team_model_token_increment(served.cap, total_tokens - served.cap_reserved_tokens, ttl), served.cap_window + ), + ) + + +def _reserved_window( + response: RateLimitResponse, descriptor: RateLimitDescriptor | None +) -> ReservationWindowIdentity | None: + if descriptor is None: + return None + counter: Final = RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens") + return next((w for w in response.get("reservation_windows", frozenset()) if w[0] == counter), None) + + def _get_priority_settings() -> "PriorityReservationSettings": """ Get the priority reservation settings, guaranteed to be non-None. @@ -143,6 +310,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ): self.internal_usage_cache = InternalUsageCache(dual_cache=internal_usage_cache) self.v3_limiter = PROXY_MaxParallelRequestsHandler_v3(self.internal_usage_cache, time_provider=time_provider) + self.active_teams = ActiveTeamRegistry(internal_usage_cache) def update_variables(self, llm_router: Router) -> None: self.llm_router = llm_router @@ -155,21 +323,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): self, counter: RateLimitCounter, ) -> int | float | str | None: - """ - Get saturation value with configurable local cache TTL. - - Uses DualCache with configurable TTL for local cache storage. - TTL is configurable via litellm.priority_reservation_settings.saturation_check_cache_ttl - - Args: - counter: The saturation counter - - Returns: - Counter value, or None if not found - """ - value: Final = await self.v3_limiter.rate_limits.get( - counter, backfill_ttl=self._get_saturation_check_cache_ttl() - ) + """The model's counter as Redis holds it, so every worker agrees on saturation. A worker's own in-memory copy + carries only the usage it served itself, and steady traffic keeps that copy alive past any TTL. The read is + written back to memory for ``saturation_check_cache_ttl`` seconds; without Redis, or when the read fails, memory is + all there is.""" + rate_limits: Final = self.v3_limiter.rate_limits + try: + value: Final = await rate_limits.get_from_redis( + counter, backfill_ttl=self._get_saturation_check_cache_ttl() + ) + except Exception as e: # noqa: BLE001 # any Redis failure degrades to the local mirror, as the limiter's counter reads do + verbose_proxy_logger.warning("Saturation read failed in Redis, using this worker's copy: %s", e) + local: Final = rate_limits.local((counter,))[0] + return None if isinstance(local, dict) else local return None if isinstance(value, dict) else value def _get_priority_weight(self, priority: str | None, model_info: ModelGroupInfo | None = None) -> float: @@ -335,11 +501,56 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # Fail open: assume not saturated on error return 0.0 + def _priority_class(self, priority: str | None) -> str: + """The class a team is counted under: its explicit priority, or one shared default class.""" + if ( + priority is not None + and litellm.priority_reservation is not None + and priority in litellm.priority_reservation + ): + return priority + return DEFAULT_CLASS + + async def _team_class_share( + self, + model: str, + user_api_key_dict: UserAPIKeyAuth, + priority: str | None, + normalized_weights: Mapping[str, float], + ) -> TeamShare | None: + """The caller's slice of its class share, or None when the split is off or the key has no team. A key with no + team keeps the class pool but still marks its class active, so the pool's capacity is not lent away.""" + if ( + litellm.priority_reservation is None + or not _get_priority_settings().priority_class_share_splits_equally_among_active_teams + ): + return None + priority_class: Final = self._priority_class(priority) + member: Final = ( + pool_member(priority_class) if user_api_key_dict.team_id is None else team_member(user_api_key_dict.team_id) + ) + from redis.exceptions import RedisError + + try: + members: Final = await self.active_teams.touch(model, member, priority_class, self.v3_limiter.window_size) + except (RedisError, RedisCircuitBreakerOpenError, ValidationError) as e: + verbose_proxy_logger.warning( + "Active team registry unavailable for %s, holding team %s to the class pool: %s", + model, + user_api_key_dict.team_id, + e, + ) + return None + if user_api_key_dict.team_id is None: + return None + return team_share(priority_class, members, normalized_weights, _get_priority_settings().default_priority) + def _create_priority_based_descriptors( self, model: str, user_api_key_dict: UserAPIKeyAuth, priority: str | None, + share: TeamShare | None = None, ) -> list[RateLimitDescriptor]: """ Create rate limit descriptors with normalized priority weights. @@ -369,14 +580,16 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) rate_limit_config: Final[RateLimitDescriptorRateLimitObject] = {} + weight: Final = priority_weight if share is None else share.fraction + floor: Final = 0 if share is None else 1 # Apply priority weight to model limits if model_group_info.tpm is not None: - reserved_tpm: Final = int(model_group_info.tpm * priority_weight) + reserved_tpm: Final = max(floor, int(model_group_info.tpm * weight)) rate_limit_config["tokens_per_unit"] = reserved_tpm if model_group_info.rpm is not None: - reserved_rpm: Final = int(model_group_info.rpm * priority_weight) + reserved_rpm: Final = max(floor, int(model_group_info.rpm * weight)) rate_limit_config["requests_per_unit"] = reserved_rpm if rate_limit_config: @@ -384,17 +597,15 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): descriptors.append( RateLimitDescriptor( - key="priority_model", - value=priority_key, + key="priority_model" if share is None else PRIORITY_CLASS_SHARE_AT_SATURATION, + value=priority_key if share is None else counter_value(model, str(user_api_key_dict.team_id)), rate_limit=rate_limit_config, ) ) return descriptors - def _create_borrowing_team_model_limit_descriptor( - self, model: str, user_api_key_dict: UserAPIKeyAuth - ) -> RateLimitDescriptor | None: + def _borrowing_team_share(self, model: str, user_api_key_dict: UserAPIKeyAuth) -> _BorrowingTeamShare | None: """One counter per team and resolved model group, so an alias and its target share the team's share.""" rate_limited: Final = self.v3_limiter.rate_limited_model(model) or RateLimitedModel( requested=model, group=model @@ -402,9 +613,22 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): limits: Final = borrowing_team_model_limits(user_api_key_dict, rate_limited) if limits.tpm is None and limits.rpm is None: return None + ceiling: Final = limits.borrow_ceiling() + value: Final = f"{user_api_key_dict.team_id}:{rate_limited.group}" + return _BorrowingTeamShare( + limits=limits, + share=self._team_share_descriptor(value, limits), + ceiling=None + if ceiling is None + else self._team_share_descriptor(value, ceiling, TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE), + ) + + def _team_share_descriptor( + self, value: str, limits: BorrowingTeamModelLimits, key: str = TEAM_MODEL_LIMIT_AT_SATURATION + ) -> RateLimitDescriptor: return RateLimitDescriptor( - key=TEAM_MODEL_LIMIT_AT_SATURATION, - value=f"{user_api_key_dict.team_id}:{rate_limited.group}", + key=key, + value=value, rate_limit={ "tokens_per_unit": limits.tpm, "requests_per_unit": limits.rpm, @@ -412,6 +636,130 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): }, ) + def _team_share_error( + self, + model: str, + status: RateLimitStatus, + saturation: float, + priority: str | None, + user_api_key_dict: UserAPIKeyAuth, + team_share: _BorrowingTeamShare, + ) -> ProxyRateLimitError: + if status["descriptor_key"] != TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: + return self._team_limit_error(model, status, saturation, priority, user_api_key_dict) + return self._team_borrow_ceiling_error( + model, status, saturation, priority, user_api_key_dict, team_share.limits + ) + + def _throttled( + self, refusal: ProxyRateLimitError, model: str, user_api_key_dict: UserAPIKeyAuth + ) -> ProxyRateLimitError: + """Count a refusal that reaches the caller under the reason its ``rate_limited_by`` header carries.""" + headers: Final = refusal.headers or {} + reason: Final = headers.get("rate_limited_by") + if user_api_key_dict.team_id is None or reason is None: + return refusal + for logger in _prometheus_loggers(): + try: + logger.increment_dynamic_limiter_throttled( + model_group=model, + team=user_api_key_dict.team_id, + team_alias=user_api_key_dict.team_alias, + priority=headers.get("x-litellm-priority") or "default", + rate_limit_type=headers.get("rate_limit_type") or "tokens", + reason=reason, + ) + except Exception as metrics_error: # noqa: BLE001 # a failing metrics sink must not change the 429 + verbose_proxy_logger.warning("Dynamic rate limiter throttle metric failed: %s", metrics_error) + return refusal + + async def _record_team_share( + self, + stash: RequestRateLimiterStash | None, + model_group: str | None, + settlement: _SettlementMetadata, + hold: BorrowingTeamModelHold | None, + ) -> None: + """Publish the team's share on the model that served the request, the fallback target's own hold when a + router fallback served it, and the tokens settled above that share so far this window.""" + try: + loggers: Final = _prometheus_loggers() + if not loggers or stash is None or model_group is None or settlement.user_api_key_team_id is None: + return + share: Final = ( + _tightest_of(_tightest_team_share((hold.descriptor,)), _capped_idle_share(hold.share, hold.cap)) + if hold is not None + else _tightest_of( + _tightest_team_share((stash.borrowing_team_model_limit, stash.priority_class_share)), + _capped_idle_share(stash.borrowing_team_model_share, stash.borrowing_team_model_cap), + ) + ) or _tightest_team_share( + ( + hold.share + if hold is not None + else _share_on_model( + stash.borrowing_team_model_share, + settlement.user_api_key_team_id, + ( + self.v3_limiter.rate_limited_model(model_group) + or RateLimitedModel(requested=model_group, group=model_group) + ).group, + ), + ) + ) + if share is None: + return + descriptor, share_tokens = share + used: Final = await self._get_saturation_value_from_cache( + RateLimitScope(descriptor["key"], descriptor["value"]).counter("tokens") + ) + used_tokens: Final = 0 if used is None else int(float(used)) + for logger in loggers: + logger.set_dynamic_limiter_share( + model_group=model_group, + team=settlement.user_api_key_team_id, + team_alias=settlement.user_api_key_team_alias, + priority=stash.dynamic_limiter_priority or "default", + share_tokens=share_tokens, + borrowed_tokens=max(0, used_tokens - share_tokens), + ) + except Exception as metrics_error: # noqa: BLE001 # a failing metrics sink must not change settlement + verbose_proxy_logger.warning("Dynamic rate limiter share metrics failed: %s", metrics_error) + + def _team_borrow_ceiling_error( + self, + model: str, + status: RateLimitStatus, + saturation: float, + priority: str | None, + user_api_key_dict: UserAPIKeyAuth, + limits: BorrowingTeamModelLimits, + ) -> ProxyRateLimitError: + resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + limit_name: Final = _team_limit_name(status["rate_limit_type"]) + return ProxyRateLimitError( + detail={ + "error": f"Team {limit_name} borrow cap reached for {model}: the team may use at most " + f"{limits.max_borrow_multiple}x its {limit_name} while borrowing idle capacity. " + f"Team: {user_api_key_dict.team_id}, " + f"Rate limit type: {status['rate_limit_type']}, " + f"Team {limit_name}: {limits.share(status['rate_limit_type'])}, " + f"Borrow cap: {status['current_limit']}, " + f"Remaining: {status['limit_remaining']}, " + f"Model saturation: {saturation:.1%}" + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "rate_limit_type": str(status["rate_limit_type"]), + "rate_limited_by": TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE, + "x-litellm-priority": priority or "default", + "x-litellm-saturation": f"{saturation:.2%}", + }, + rate_limit_type=map_v3_rate_limit_type(status["rate_limit_type"]), + model=resolved_model, + llm_provider=llm_provider, + ) + def _team_limit_error( self, model: str, @@ -443,35 +791,119 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) + def _class_share_error( + self, + model: str, + status: RateLimitStatus, + saturation: float, + priority: str | None, + user_api_key_dict: UserAPIKeyAuth, + share: TeamShare, + ) -> ProxyRateLimitError: + resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + error: Final = ProxyRateLimitError( + detail={ + "error": f"Priority class share reached for {model} and the model is saturated. " + f"Team: {user_api_key_dict.team_id}, " + f"Priority: {priority or 'default'}, " + f"Active teams in class: {share.active_teams_in_class}, " + f"Active classes: {share.active_classes}, " + f"Rate limit type: {status['rate_limit_type']}, " + f"Team share: {status['current_limit']}, " + f"Remaining: {status['limit_remaining']}, " + f"Model saturation: {saturation:.1%}" + }, + headers={ + "retry-after": str(self.v3_limiter.window_size), + "rate_limit_type": str(status["rate_limit_type"]), + "rate_limited_by": PRIORITY_CLASS_SHARE_AT_SATURATION, + "x-litellm-priority": priority or "default", + "x-litellm-saturation": f"{saturation:.2%}", + }, + rate_limit_type=map_v3_rate_limit_type(status["rate_limit_type"]), + model=resolved_model, + llm_provider=llm_provider, + ) + return error + async def enforce_token_limits_for_frame( self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None ) -> None: - """Hold a Responses WebSocket turn to the team's share once the model saturates, like the regular limiter does.""" + """Hold a Responses WebSocket turn to the team's shares once the model saturates, like the regular limiter does.""" if requested_model is None: return - team_limit_descriptor: Final = self._create_borrowing_team_model_limit_descriptor( - model=requested_model, user_api_key_dict=user_api_key_dict - ) model_group_info: Final = self.llm_router.registry().group(requested_model) - if team_limit_descriptor is None or model_group_info is None: + if model_group_info is None: + return + priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict) + share: Final = await self._team_class_share( + requested_model, user_api_key_dict, priority, self._normalize_priority_weights(model_group_info) + ) + share_descriptors: Final = ( + () + if share is None + else tuple( + self._create_priority_based_descriptors( + model=requested_model, user_api_key_dict=user_api_key_dict, priority=priority, share=share + ) + ) + ) + frame_share: Final = next( + (d for d in share_descriptors if d["key"] == PRIORITY_CLASS_SHARE_AT_SATURATION), None + ) + connection_stash: Final = get_request_stash() + if connection_stash is not None and frame_share is not None: + connection_stash.priority_class_share = frame_share + borrowing: Final = self._borrowing_team_share(model=requested_model, user_api_key_dict=user_api_key_dict) + if not share_descriptors and borrowing is None: return saturation: Final = await self._check_model_saturation(requested_model, model_group_info) - if saturation < _get_priority_settings().saturation_threshold: - return - tpm_windows: Final = _socket_turn_tpm_windows((team_limit_descriptor,)) - response: Final = await self.v3_limiter.should_rate_limit( - tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True + saturated: Final = saturation >= _get_priority_settings().saturation_threshold + enforced_team: Final = None if borrowing is None else borrowing.enforced(saturated) + team_cap: Final = None if borrowing is None else borrowing.ceiling + if connection_stash is not None and borrowing is not None: + connection_stash.borrowing_team_model_limit = ( + connection_stash.borrowing_team_model_limit + if enforced_team is None and connection_stash.borrowing_team_model_reserved_tokens + else enforced_team + ) + connection_stash.borrowing_team_model_share = borrowing.share + connection_stash.borrowing_team_model_cap = team_cap + descriptors: Final = ( + (share_descriptors if saturated else ()) + + (() if enforced_team is None else (enforced_team,)) + + (() if team_cap is None else (team_cap,)) ) + if not descriptors: + return + tpm_windows: Final = _socket_turn_tpm_windows(descriptors) + try: + response: Final = await self.v3_limiter.should_rate_limit( + tpm_windows, + parent_otel_span=user_api_key_dict.parent_otel_span, + read_only=True, + skip_parallel_check=True, + ) + except RateLimitUnverifiableError as unverifiable: + raise ServiceUnavailableError( + message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model + ) from unverifiable over_limit: Final = next((s for s in response["statuses"] if s["code"] == "OVER_LIMIT"), None) if over_limit is None: return - raise self._team_limit_error( - requested_model, - over_limit, - saturation, - self._get_priority_from_user_api_key_dict(user_api_key_dict), - user_api_key_dict, - ) + over_class_share: Final = over_limit["descriptor_key"] == PRIORITY_CLASS_SHARE_AT_SATURATION + if share is not None and (over_class_share or borrowing is None): + raise self._throttled( + self._class_share_error(requested_model, over_limit, saturation, priority, user_api_key_dict, share), + requested_model, + user_api_key_dict, + ) + if borrowing is not None: + raise self._throttled( + self._team_share_error(requested_model, over_limit, saturation, priority, user_api_key_dict, borrowing), + requested_model, + user_api_key_dict, + ) def _create_model_tracking_descriptor( self, @@ -507,6 +939,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: UserAPIKeyAuth, priority: str | None, saturation: float, + data: Mapping[str, object] = MappingProxyType({}), + call_type: str | None = None, ) -> None: """ Check rate limits using THREE-PHASE approach to prevent partial increments. @@ -533,6 +967,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): user_api_key_dict: User authentication info priority: User's priority level saturation: Current saturation level + data: Request body, used to size the token reservation a saturated team limit holds + call_type: The endpoint's call type, used to read the body's prompt fields Raises: HTTPException: If any limit is exceeded @@ -553,11 +989,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): descriptors_to_check.append(model_wide_descriptor) # Priority descriptors (always track, conditionally enforce) + share: Final = await self._team_class_share( + model, user_api_key_dict, priority, self._normalize_priority_weights(model_group_info) + ) priority_descriptors: Final = self._create_priority_based_descriptors( model=model, user_api_key_dict=user_api_key_dict, priority=priority, + share=share, ) + get_or_create_request_stash().priority_class_share = next( + (d for d in priority_descriptors if d["key"] == PRIORITY_CLASS_SHARE_AT_SATURATION), None + ) + get_or_create_request_stash().dynamic_limiter_priority = priority if priority_descriptors: descriptors_to_check.extend(priority_descriptors) @@ -569,26 +1013,41 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # asyncio.Lock + in-memory fallback for single-process deployments. # All-or-nothing: if any enforced descriptor would exceed its limit, # no counter is modified and the response carries "OVER_LIMIT". - team_limit_descriptor: Final = self._create_borrowing_team_model_limit_descriptor( - model=model, user_api_key_dict=user_api_key_dict - ) - get_or_create_request_stash().borrowing_team_model_limit = team_limit_descriptor - if team_limit_descriptor is not None: - descriptors_to_check.append(team_limit_descriptor) + team_share: Final = self._borrowing_team_share(model=model, user_api_key_dict=user_api_key_dict) + if team_share is not None: + descriptors_to_check.append(team_share.share) + enforced_team_share: Final = None if team_share is None else team_share.enforced(should_enforce_priority) + team_cap: Final = None if team_share is None else team_share.ceiling + held_team_share: Final = enforced_team_share if enforced_team_share is not None else team_cap + get_or_create_request_stash().borrowing_team_model_limit = enforced_team_share + get_or_create_request_stash().borrowing_team_model_share = None if team_share is None else team_share.share + get_or_create_request_stash().borrowing_team_model_cap = team_cap enforced_descriptors: Final[list[RateLimitDescriptor]] = [model_wide_descriptor] if priority_descriptors and should_enforce_priority: enforced_descriptors.extend(priority_descriptors) - if team_limit_descriptor is not None and should_enforce_priority: - enforced_descriptors.append(team_limit_descriptor) + if enforced_team_share is not None: + enforced_descriptors.append(enforced_team_share) + if team_cap is not None: + enforced_descriptors.append(team_cap) per_request_increment: Final[dict[Literal["requests", "tokens"], int]] = { "requests": 1, "tokens": 0, } + team_reservation: Final = ( + await self._team_token_reservation(team_share.share, data, model, call_type, user_api_key_dict) + if team_share is not None and held_team_share is not None + else 0 + ) atomic_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( descriptors=enforced_descriptors, - increments=[per_request_increment for _ in enforced_descriptors], + increments=[ + {"requests": 1, "tokens": team_reservation} + if descriptor is enforced_team_share or descriptor is team_cap + else per_request_increment + for descriptor in enforced_descriptors + ], parent_otel_span=user_api_key_dict.parent_otel_span, ) @@ -645,8 +1104,13 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) - if descriptor_key == TEAM_MODEL_LIMIT_AT_SATURATION: - raise self._team_limit_error(model, status, saturation, priority, user_api_key_dict) + if ( + descriptor_key in (TEAM_MODEL_LIMIT_AT_SATURATION, TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE) + and team_share is not None + ): + raise self._team_share_error(model, status, saturation, priority, user_api_key_dict, team_share) + if descriptor_key == PRIORITY_CLASS_SHARE_AT_SATURATION and share is not None: + raise self._class_share_error(model, status, saturation, priority, user_api_key_dict, share) # Fail-closed guard: overall_code says OVER_LIMIT but no status # matched a descriptor key we know how to translate into a 429. @@ -687,13 +1151,19 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # priority allocation has accurate usage when it later becomes # enforced. The increment-then-check semantics of should_rate_limit # are fine because we ignore the OVER_LIMIT response. - if team_limit_descriptor is not None and not should_enforce_priority: - await self.v3_limiter.atomic_check_and_increment_by_n( - descriptors=[team_limit_descriptor], - increments=[per_request_increment], - parent_otel_span=user_api_key_dict.parent_otel_span, - ) - if priority_descriptors and not should_enforce_priority: + stash: Final = get_or_create_request_stash() + stash.borrowing_team_model_reserved_tokens = team_reservation if enforced_team_share is not None else 0 + stash.borrowing_team_model_cap_reserved_tokens = team_reservation if team_cap is not None else 0 + stash.borrowing_team_model_reserved_window = ( + _reserved_window(atomic_response, enforced_team_share) if team_reservation else None + ) + stash.borrowing_team_model_cap_reserved_window = ( + _reserved_window(atomic_response, team_cap) if team_reservation else None + ) + self._cap_admitted_output(None if team_share is None else team_share.share, team_reservation, data, call_type) + if priority_descriptors and share is not None and not should_enforce_priority: + get_or_create_request_stash().rate_limit_response = atomic_response + elif priority_descriptors and not should_enforce_priority: priority_tracking_response: Final = await self.v3_limiter.should_rate_limit( descriptors=priority_descriptors, parent_otel_span=user_api_key_dict.parent_otel_span, @@ -706,12 +1176,167 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): else: get_or_create_request_stash().rate_limit_response = atomic_response + def _cap_admitted_output( + self, + enforced_team_share: RateLimitDescriptor | None, + team_reservation: int, + data: Mapping[str, object], + call_type: str | None, + ) -> None: + """Write the output cap the hold was estimated with onto the admitted request, unless only input tokens count.""" + tpm_share: Final = ( + None if enforced_team_share is None else (enforced_team_share["rate_limit"] or {}).get("tokens_per_unit") + ) + if team_reservation and tpm_share is not None and self.v3_limiter.get_rate_limit_type() != "input": + self.v3_limiter.bound_output_for_limit(data, tpm_share, call_type) + + async def _team_token_reservation( + self, + descriptor: RateLimitDescriptor, + data: Mapping[str, object], + model: str, + call_type: str | None, + user_api_key_dict: UserAPIKeyAuth, + ) -> int: + """Tokens to hold against the team's share before the call, as the regular limiter does for a hard limit.""" + tpm_share: Final = (descriptor["rate_limit"] or {}).get("tokens_per_unit") + if tpm_share is None or not data: + return 0 + as_dispatched: Final = dict(data, model=model) + if self.v3_limiter.get_rate_limit_type() != "input": + self.v3_limiter.bound_output_for_limit(as_dispatched, tpm_share, call_type) + try: + estimate: Final = await offload_token_count(self.v3_limiter.estimate_request_tokens)( + data=as_dispatched, + model=model, + min_configured_tpm_limit=tpm_share, + call_type=call_type, + team_id=user_api_key_dict.team_id, + rate_limit_type=self.v3_limiter.get_rate_limit_type(), + ) + except Exception as e: # noqa: BLE001 # an unreadable prompt still holds the minimum so admission runs + verbose_proxy_logger.warning( + "Dynamic rate limiter: token estimate failed for %s, holding one token: %s", model, e + ) + return 1 + return max(estimate, 1) + + async def async_fallback_target_allowed( + self, user_api_key_dict: UserAPIKeyAuth, model: str, request_data: Mapping[str, object], *, commit: bool = True + ) -> bool: + """Hold a router fallback target to the team's share on it once it saturates, and to its borrow ceiling before. + + With ``commit`` the attempt is counted toward the target's saturation, and toward the team's counter only + while the share is enforced, holding the prompt's estimated tokens there. An idle target is only checked. + """ + team_share: Final = self._borrowing_team_share(model=model, user_api_key_dict=user_api_key_dict) + model_group_info: Final = self.llm_router.registry().group(model) + if team_share is None or model_group_info is None: + return True + saturation: Final = await self._check_model_saturation(model, model_group_info) + enforced: Final = team_share.enforced(saturation >= _get_priority_settings().saturation_threshold) + cap: Final = team_share.ceiling + held: Final = enforced if enforced is not None else cap + reservation: Final = ( + await self._team_token_reservation( + team_share.share, + {**request_data, "model": model}, + model, + get_or_create_request_stash().call_type, + user_api_key_dict, + ) + if held is not None + else 0 + ) + team_descriptors: Final = tuple(d for d in (enforced, cap) if d is not None) + if not commit: + for descriptor in team_descriptors: + if not await self._share_has_room(descriptor, reservation, user_api_key_dict): + return False + return True + model_wide_descriptor: Final = self._create_model_tracking_descriptor( + model=model, model_group_info=model_group_info, high_limit_multiplier=1 + ) + response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=[model_wide_descriptor, *team_descriptors], + increments=[ + {"requests": 1, "tokens": 0}, + *({"requests": 1, "tokens": reservation} for _ in team_descriptors), + ], + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + if response["overall_code"] == "OVER_LIMIT": + return False + self._hold_fallback_share( + model, + team_share.share, + enforced, + reservation if enforced is not None else 0, + _reserved_window(response, enforced), + cap, + reservation if cap is not None else 0, + _reserved_window(response, cap), + ) + return True + + async def _share_has_room( + self, descriptor: RateLimitDescriptor, tokens: int, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + checked: Final = await self.v3_limiter.should_rate_limit( + [descriptor], parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True + ) + return all( + status["code"] != "OVER_LIMIT" + and (status["rate_limit_type"] != "tokens" or (status["limit_remaining"] or 0) >= tokens) + for status in checked["statuses"] + ) + + def _hold_fallback_share( + self, + model: str, + share: RateLimitDescriptor, + charged: RateLimitDescriptor | None, + reserved_tokens: int, + window: ReservationWindowIdentity | None, + cap: RateLimitDescriptor | None = None, + cap_reserved_tokens: int = 0, + cap_window: ReservationWindowIdentity | None = None, + ) -> None: + """Remember the hold under the requested name and the resolved group, so settlement finds it either way. + + ``charged`` is the share the target's usage settles to, or None for an idle target whose usage is forgiven. + ``cap`` is the team's borrow ceiling on the target, charged whether or not the target is saturated. Each + counter carries an earlier hold on the same target only while that hold sits in the counter's current window. + """ + stash: Final = get_or_create_request_stash() + earlier: Final = stash.borrowing_team_model_fallback_limits.get(model) + carried: Final = ( + earlier.reserved_tokens if earlier is not None and window is not None and earlier.window == window else 0 + ) + carried_cap: Final = ( + earlier.cap_reserved_tokens + if earlier is not None and cap_window is not None and earlier.cap_window == cap_window + else 0 + ) + hold: Final = BorrowingTeamModelHold( + descriptor=charged, + reserved_tokens=reserved_tokens + carried, + window=window, + cap=cap, + cap_reserved_tokens=cap_reserved_tokens + carried_cap, + cap_window=cap_window, + share=share, + ) + stash.borrowing_team_model_fallback_limits = MappingProxyType( + {**stash.borrowing_team_model_fallback_limits, model: hold, share["value"].rpartition(":")[2]: hold} + ) + @with_service_target(RATE_LIMITS_TARGET) async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, - data: dict, + data: Mapping[str, object], call_type: CallTypesLiteral, ) -> Exception | str | dict | None: """ @@ -753,50 +1378,77 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): claim_request_stash_for_data(data) model: Final = data["model"] - priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) + if not isinstance(model, str): + return None # Get model configuration - model_group_info: Final[ModelGroupInfo | None] = self.llm_router.registry().group(model) - if model_group_info is None: + requested_group_info: Final[ModelGroupInfo | None] = self.llm_router.registry().group(model) + if requested_group_info is None: verbose_proxy_logger.debug("No model group info for %s, allowing request", model) return None try: - # STEP 1: Check current saturation level - saturation: Final = await self._check_model_saturation(model, model_group_info) - - saturation_threshold: Final = _get_priority_settings().saturation_threshold - - verbose_proxy_logger.debug( - f"[Dynamic Rate Limiter] Model={model}, Saturation={saturation:.1%}, " - f"Threshold={saturation_threshold:.1%}, Priority={priority}" - ) - - # STEP 2: Check rate limits in THREE phases - # Phase 1: Read-only check of ALL limits (no increments) - # Phase 2: Decide which limits to enforce (based on saturation) - # Phase 3: Increment ALL counters only if request will be allowed - # This prevents partial increments and ensures accurate tracking - await self._check_rate_limits( - model=model, - model_group_info=model_group_info, - user_api_key_dict=user_api_key_dict, - priority=priority, - saturation=saturation, - ) - + await self._admit(model, requested_group_info, user_api_key_dict, data, call_type) + except ProxyRateLimitError as refusal: + raise self._throttled(refusal, model, user_api_key_dict) from None except HTTPException: raise except Exception as e: verbose_proxy_logger.error("Error in dynamic rate limiter: %s, allowing request", e) # Fail open on unexpected errors return None - return None + async def _admit( + self, + model: str, + model_group_info: ModelGroupInfo, + user_api_key_dict: UserAPIKeyAuth, + data: Mapping[str, object], + call_type: str | None, + ) -> None: + """Read saturation, then the three-phase rate limit check, raising the 429 when ``model`` is out of capacity. + A key or team with ``router_settings.fallback_on_saturation`` is sent to its fallbacks first once ``model`` is + at or above ``saturation_threshold``, and comes back here only when none of them admitted the request.""" + priority: Final = self._get_priority_from_user_api_key_dict(user_api_key_dict=user_api_key_dict) + saturation: Final = await self._check_model_saturation(model, model_group_info) + saturation_threshold: Final = _get_priority_settings().saturation_threshold + verbose_proxy_logger.debug( + f"[Dynamic Rate Limiter] Model={model}, Saturation={saturation:.1%}, " + f"Threshold={saturation_threshold:.1%}, Priority={priority}" + ) + if saturation >= saturation_threshold and fallback_on_saturation_requested(data): + raise self._saturation_fallback_error(model, saturation, priority) + await self._check_rate_limits( + model=model, + model_group_info=model_group_info, + user_api_key_dict=user_api_key_dict, + priority=priority, + saturation=saturation, + data=data, + call_type=call_type, + ) + + def _saturation_fallback_error(self, model: str, saturation: float, priority: str | None) -> ProxyRateLimitError: + resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(model) + return ProxyRateLimitError( + detail={ + "error": f"Model {model} is at {saturation:.0%} of its capacity, at or above the saturation " + f"threshold of {_get_priority_settings().saturation_threshold:.0%}, trying fallbacks first. " + f"Priority: {priority}" + }, + headers={"retry-after": str(self.v3_limiter.window_size), "x-litellm-priority": priority or "default"}, + model=resolved_model, + llm_provider=llm_provider, + descriptor_key=SATURATION_FALLBACK_DESCRIPTOR_KEY, + ) + @with_service_target(RATE_LIMITS_TARGET) async def async_post_call_success_hook( - self, data: dict, user_api_key_dict: UserAPIKeyAuth, response + self, + data: dict[str, object], # mutable-ok: the CustomLogger contract hands the v3 limiter a mutable dict + user_api_key_dict: UserAPIKeyAuth, + response, ) -> LLMResponseTypes: """ Post-call hook to add rate limit headers to response. @@ -827,7 +1479,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): return response @with_service_target(RATE_LIMITS_TARGET) - async def async_log_success_event(self, kwargs, response_obj: object, start_time, end_time) -> None: + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: """ Update token usage for priority-based rate limiting after successful API calls. @@ -846,36 +1500,45 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): verbose_proxy_logger.debug("INSIDE dynamic rate limiter ASYNC SUCCESS LOGGING") litellm_parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - - # Get metadata from standard_logging_object - standard_logging_object: Final = kwargs.get("standard_logging_object") or {} - settlement_metadata: Final = _SettlementMetadata.model_validate( - standard_logging_object.get("metadata") or {} + callback_kwargs: Final = callback_kwargs_mapping(kwargs) + settlement_metadata: Final = ( + _SettlementLoggingObject.model_validate(callback_kwargs.get("standard_logging_object") or {}).metadata + or _SettlementMetadata() ) - # Get model and priority - model_group: Final = get_model_group_from_litellm_kwargs(kwargs) + model_group: Final = get_model_group_from_litellm_kwargs(dict(kwargs)) + total_tokens: Final = _settled_tokens( + _usage_of(callback_kwargs, response_obj), self.v3_limiter.get_rate_limit_type() + ) + stash: Final = request_stash_for_callback(callback_kwargs) + team_model_tokens: Final = _team_model_settlement( + stash, model_group, total_tokens, self.v3_limiter.window_size + ) + served: Final = ( + None + if stash is None or model_group is None + else stash.borrowing_team_model_fallback_limits.get(model_group) + ) + if stash is not None: + stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_cap_reserved_tokens = 0 + stash.borrowing_team_model_reserved_window = None + stash.borrowing_team_model_cap_reserved_window = None + stash.borrowing_team_model_fallback_limits = MappingProxyType( + {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not served} + ) if not model_group: + if team_model_tokens: + await self.v3_limiter.async_increment_reservation_aware_tokens( + pipeline_operations=team_model_tokens, parent_otel_span=litellm_parent_otel_span + ) return # Get priority from user_api_key_auth_metadata in standard_logging_metadata # This is where user_api_key_dict.metadata is stored during pre-call key_priority: Final = settlement_metadata.priority - # Get total tokens from response - total_tokens = 0 - rate_limit_type: Final = self.v3_limiter.get_rate_limit_type() - - _usage: Final = _usage_of(response_obj) - if _usage is not None: - if rate_limit_type == "output": - total_tokens = _usage.completion_tokens - elif rate_limit_type == "input": - total_tokens = _usage.prompt_tokens - elif rate_limit_type == "total": - total_tokens = _usage.total_tokens - - if total_tokens == 0: + if total_tokens == 0 and not team_model_tokens: return # Model-wide token tracking (model_saturation_check) @@ -898,25 +1561,37 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): else: priority_key = f"{model_group}:default_pool" - team_model_tokens: Final = _team_model_token_increment( - get_or_create_request_stash().borrowing_team_model_limit, total_tokens, self.v3_limiter.window_size + class_share: Final = None if stash is None else stash.priority_class_share + settled_team_id: Final = settlement_metadata.user_api_key_team_id + priority_scope: Final = ( + RateLimitScope("priority_model", priority_key) + if class_share is None + else RateLimitScope( + class_share["key"], + class_share["value"] if settled_team_id is None else counter_value(model_group, settled_team_id), + ) ) - pipeline_operations: Final = ( - model_tokens, - CounterIncrement( - counter=RateLimitScope("priority_model", priority_key).counter("tokens"), - amount=total_tokens, - ttl=self.v3_limiter.window_size, - ), - *team_model_tokens, + usage_operations: Final = ( + () + if total_tokens == 0 + else ( + model_tokens, + CounterIncrement( + counter=priority_scope.counter("tokens"), + amount=total_tokens, + ttl=self.v3_limiter.window_size, + ), + ) ) + pipeline_operations: Final[tuple[ReservationOperation, ...]] = (*usage_operations, *team_model_tokens) # Execute token increments with TTL preservation if pipeline_operations: - await self.v3_limiter.async_increment_tokens_with_ttl_preservation( + await self.v3_limiter.async_increment_reservation_aware_tokens( pipeline_operations=pipeline_operations, parent_otel_span=litellm_parent_otel_span, ) + await self._record_team_share(stash, model_group, settlement_metadata, served) # Only log 'priority' if it's known safe; otherwise, redact. SAFE_PRIORITIES: Final = {"low", "medium", "high", "default"} @@ -931,5 +1606,99 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): except Exception as e: verbose_proxy_logger.exception("Error in dynamic rate limiter success event: %s", e) + @with_service_target(RATE_LIMITS_TARGET) + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], # mutable-ok: the base hook declares this parameter as a dict + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + """Give back the team-share tokens held at admission when the proxy refuses the request before the call runs, + as a later hook or the pre-call fallback walk does. ``async_log_failure_event`` is a completion callback and + never fires for those refusals, so without this the hold stays charged until the window resets. Idempotent: + the first release zeroes the stash's reservation, so a settlement that still follows charges only usage.""" + stash: Final = get_request_stash() + if stash is None: + return + refund: Final = _team_model_settlement(stash, None, 0, self.v3_limiter.window_size) + stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_cap_reserved_tokens = 0 + stash.borrowing_team_model_reserved_window = None + stash.borrowing_team_model_cap_reserved_window = None + if refund: + await self.v3_limiter.async_increment_reservation_aware_tokens( + pipeline_operations=refund, parent_otel_span=user_api_key_dict.parent_otel_span + ) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + """Settle the tokens held for a failed attempt to the usage it recorded, or give them all back when it recorded + none, on the admitting team counter or the fallback target that failed, leaving holds on other fallback + targets to the attempts still running.""" + from litellm.proxy.common_utils.callback_utils import get_model_group_from_litellm_kwargs + + callback_kwargs: Final = callback_kwargs_mapping(kwargs) + stash: Final = request_stash_for_callback(callback_kwargs) + if stash is None: + return + failed_group: Final = get_model_group_from_litellm_kwargs(dict(callback_kwargs)) + failed_hold: Final = ( + stash.borrowing_team_model_fallback_limits.get(failed_group) if failed_group is not None else None + ) + used: Final = _settled_tokens( + _usage_of(callback_kwargs, None if isinstance(response_obj, BaseException) else response_obj), + self.v3_limiter.get_rate_limit_type(), + ) + own_usage: Final = used if failed_hold is None else 0 + own_delta: Final = own_usage - stash.borrowing_team_model_reserved_tokens + own_cap_delta: Final = own_usage - stash.borrowing_team_model_cap_reserved_tokens + refund: Final = ( + *_window_guarded( + _team_model_token_increment(stash.borrowing_team_model_limit, own_delta, self.v3_limiter.window_size), + stash.borrowing_team_model_reserved_window, + ), + *_window_guarded( + _team_model_token_increment(stash.borrowing_team_model_cap, own_cap_delta, self.v3_limiter.window_size), + stash.borrowing_team_model_cap_reserved_window, + ), + *( + () + if failed_hold is None + else ( + *_window_guarded( + _team_model_token_increment( + failed_hold.descriptor, used - failed_hold.reserved_tokens, self.v3_limiter.window_size + ), + failed_hold.window, + ), + *_window_guarded( + _team_model_token_increment( + failed_hold.cap, used - failed_hold.cap_reserved_tokens, self.v3_limiter.window_size + ), + failed_hold.cap_window, + ), + ) + ), + ) + stash.borrowing_team_model_reserved_tokens = 0 + stash.borrowing_team_model_cap_reserved_tokens = 0 + stash.borrowing_team_model_reserved_window = None + stash.borrowing_team_model_cap_reserved_window = None + if failed_hold is not None: + stash.borrowing_team_model_fallback_limits = MappingProxyType( + {k: h for k, h in stash.borrowing_team_model_fallback_limits.items() if h is not failed_hold} + ) + if refund: + await self.v3_limiter.async_increment_reservation_aware_tokens(pipeline_operations=refund) + await self._record_team_share( + stash, + failed_group, + _SettlementLoggingObject.model_validate(callback_kwargs.get("standard_logging_object") or {}).metadata + or _SettlementMetadata(), + failed_hold, + ) + PROXY_DynamicRateLimitHandlerV3: Final = _PROXY_DynamicRateLimitHandlerV3 diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 7f266e67b41..a6697004fc7 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import ( TYPE_CHECKING, + Annotated, Any, Final, Literal, @@ -27,7 +28,7 @@ from typing import ( ) from fastapi import HTTPException -from pydantic import TypeAdapter, ValidationError +from pydantic import Field, TypeAdapter, ValidationError from starlette.status import HTTP_400_BAD_REQUEST, HTTP_503_SERVICE_UNAVAILABLE from typing_extensions import NotRequired, ReadOnly @@ -199,17 +200,54 @@ def inherited_team_model_limit( return team_limit +TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "team_model_limit_max_borrow_multiple" +MAX_BORROW_MULTIPLE_CEILING: Final = 1_000_000 +_MAX_BORROW_MULTIPLE: Final[TypeAdapter[float]] = TypeAdapter( + Annotated[float, Field(ge=1, le=MAX_BORROW_MULTIPLE_CEILING, allow_inf_nan=False)] +) + + @dataclass(frozen=True, slots=True) class BorrowingTeamModelLimits: """Inherited team per-model limits the dynamic rate limiter enforces at saturation instead of the regular limiter.""" tpm: int | None rpm: int | None + max_borrow_multiple: float | None = None + + def share(self, rate_limit_type: str) -> int | None: + return self.rpm if rate_limit_type == "requests" else self.tpm + + def borrow_ceiling(self) -> "BorrowingTeamModelLimits | None": + """The most the team may use while borrowing, or None when it may borrow up to the model's own limit.""" + if self.max_borrow_multiple is None: + return None + return BorrowingTeamModelLimits( + tpm=None if self.tpm is None else int(self.tpm * self.max_borrow_multiple), + rpm=None if self.rpm is None else int(self.rpm * self.max_borrow_multiple), + ) NO_BORROWING: Final = BorrowingTeamModelLimits(tpm=None, rpm=None) +def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None: + raw: Final[object] = (user_api_key_dict.team_metadata or {}).get(TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE) + if raw is None: + return None + try: + return _MAX_BORROW_MULTIPLE.validate_python(raw) + except ValidationError: + verbose_proxy_logger.warning( + "Ignoring %s=%r on team %s: expected a number from 1 to %d, so the team borrows up to the model limit", + TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE, + raw, + user_api_key_dict.team_id, + MAX_BORROW_MULTIPLE_CEILING, + ) + return None + + def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits: settings: Final = litellm.priority_reservation_settings if ( @@ -224,7 +262,9 @@ def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLi ) if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group): return NO_BORROWING - return limits + return BorrowingTeamModelLimits( + tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict) + ) def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]: @@ -530,6 +570,23 @@ class CallTypeRateLimiter(Protocol): ) -> Exception | str | dict[str, object] | None: ... +@dataclass(frozen=True, slots=True) +class BorrowingTeamModelHold: + """A router fallback target's team share with the tokens held on it, settled to actual usage or given back. + + An idle target holds no descriptor: its usage is not charged to the share, and settlement still finds the hold so + the admitting model's reservation is given back instead of taking the target's tokens. + """ + + descriptor: RateLimitDescriptor | None + reserved_tokens: int = 0 + window: ReservationWindowIdentity | None = None + cap: RateLimitDescriptor | None = None + cap_reserved_tokens: int = 0 + cap_window: ReservationWindowIdentity | None = None + share: RateLimitDescriptor | None = None + + @dataclass(slots=True) class RequestRateLimiterStash: """ @@ -554,6 +611,7 @@ class RequestRateLimiterStash: """ owner_litellm_call_id: str | None = None + call_type: str | None = None rate_limit_response: RateLimitResponse | None = None parallel_slot: ParallelSlotAcquisition | None = None parallel_slot_reused_at: datetime | None = None @@ -574,6 +632,17 @@ class RequestRateLimiterStash: batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None batch_tpd_refund_ops: tuple[ReservationOperation, ...] = () borrowing_team_model_limit: RateLimitDescriptor | None = None + borrowing_team_model_reserved_tokens: int = 0 + borrowing_team_model_reserved_window: ReservationWindowIdentity | None = None + borrowing_team_model_share: RateLimitDescriptor | None = None + borrowing_team_model_cap: RateLimitDescriptor | None = None + borrowing_team_model_cap_reserved_tokens: int = 0 + borrowing_team_model_cap_reserved_window: ReservationWindowIdentity | None = None + borrowing_team_model_fallback_limits: Mapping[str, BorrowingTeamModelHold] = field( + default_factory=lambda: MappingProxyType({}) + ) + priority_class_share: RateLimitDescriptor | None = None + dynamic_limiter_priority: str | None = None reservation_released: bool = False io_reservation_released: bool = False tpm_limited_tags: frozenset[str] = field(default_factory=frozenset) @@ -666,7 +735,7 @@ def get_or_create_request_stash() -> RequestRateLimiterStash: return stash -def claim_request_stash_for_data(data: dict) -> RequestRateLimiterStash: +def claim_request_stash_for_data(data: Mapping[str, object]) -> RequestRateLimiterStash: stash: Final = get_or_create_request_stash() owner_call_id: Final = data.get("litellm_call_id") if isinstance(owner_call_id, str): @@ -721,6 +790,23 @@ def _is_socket_turn(kwargs: Mapping[str, object]) -> bool: ) +def callback_kwargs_mapping(kwargs: object) -> Mapping[str, object]: + """A logging callback's kwargs as a typed mapping, empty when they are not a mapping.""" + return _callback_kwargs_mapping(kwargs) + + +def request_stash_for_callback(kwargs: Mapping[str, object]) -> RequestRateLimiterStash | None: + """The stash of the request a logging callback settles, or the connection's for a Responses WebSocket turn. + + A nested call (an LLM judge, a silent experiment) mints its own call id and gets None. + """ + call_id: Final = kwargs.get("litellm_call_id") + call_stash: Final = get_request_stash_for_call(call_id if isinstance(call_id, str) else None) + if call_stash is not None: + return call_stash + return get_request_stash() if _is_socket_turn(kwargs) else None + + _AGENT_DESCRIPTOR_KEYS: Final = frozenset({"agent", "agent_session"}) _SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS: Final = frozenset( @@ -1018,6 +1104,37 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if existing_cap is None or effective_cap < existing_cap: data[cap_field] = effective_cap # rebind-ok: downstream routing requires the bounded output cap + def estimate_request_tokens( + self, + data: object, + model: str, + min_configured_tpm_limit: int, + call_type: str | None, + team_id: str | None, + rate_limit_type: str = "total", + ) -> int: + """The reservation this limiter would hold for the request, for another limiter settling the same call.""" + if rate_limit_type == "total": + return self._estimate_tokens_for_request( + data=data, + model=model, + min_configured_tpm_limit=min_configured_tpm_limit, + call_type=call_type, + team_id=team_id, + ) + estimated_input, output_budget = self._estimate_input_and_output_tokens( + data=data, + min_configured_tpm_limit=min_configured_tpm_limit, + call_type=call_type, + team_id=team_id, + media_model_resolver=self._media_model_resolver, + ) + return output_budget if rate_limit_type == "output" else estimated_input + + def bound_output_for_limit(self, data: object, min_configured_limit: int, call_type: str | None) -> None: + """Cap a request without an explicit output cap the way this limiter does for its own small TPM limits.""" + self._apply_implicit_output_cap(data=data, min_configured_limit=min_configured_limit, call_type=call_type) + def _estimate_tokens_for_request( self, data: object, @@ -2575,6 +2692,50 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): }, ) + def _customer_hierarchy_rate_limit_descriptors( + self, user_api_key_dict: UserAPIKeyAuth + ) -> tuple[RateLimitDescriptor, ...]: + """One descriptor per customer org, team and member level that carries a TPM or RPM limit.""" + scopes: Final = user_api_key_dict.customer_hierarchy.scopes if user_api_key_dict.customer_hierarchy else () + return tuple( + RateLimitDescriptor( + key=scope.key, + value=scope.entity_id, + rate_limit={ + "requests_per_unit": scope.rpm_limit, + "tokens_per_unit": scope.tpm_limit, + "window_size": self.window_size, + }, + ) + for scope in scopes + if scope.rpm_limit is not None or scope.tpm_limit is not None + ) + + def _customer_hierarchy_model_rate_limit_descriptors( + self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None + ) -> tuple[RateLimitDescriptor, ...]: + """The ``model_per_customer_*`` descriptors for every level whose budget limits this model group.""" + model: Final = self._rate_limited_model(requested_model) + if model is None or user_api_key_dict.customer_hierarchy is None: + return () + limits: Final = tuple( + (scope, model.limit_in(scope.model_tpm_limit), model.limit_in(scope.model_rpm_limit)) + for scope in user_api_key_dict.customer_hierarchy.scopes + ) + return tuple( + RateLimitDescriptor( + key=f"model_per_{scope.key}", + value=f"{scope.entity_id}:{model.group}", + rate_limit={ + "requests_per_unit": rpm_limit, + "tokens_per_unit": tpm_limit, + "window_size": self.window_size, + }, + ) + for scope, tpm_limit, rpm_limit in limits + if tpm_limit is not None or rpm_limit is not None + ) + def _end_user_model_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None ) -> RateLimitDescriptor | None: @@ -3083,6 +3244,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) + descriptors.extend(self._customer_hierarchy_rate_limit_descriptors(user_api_key_dict)) + # Model rate limits requested_model: Final = data.get("model", None) model_candidates: Final = expand_model_latest_alias(requested_model if isinstance(requested_model, str) else "") @@ -3094,6 +3257,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) if (end_user_model := self._end_user_model_rate_limit_descriptor(user_api_key_dict, candidate)) is not None: descriptors.append(end_user_model) + descriptors.extend(self._customer_hierarchy_model_rate_limit_descriptors(user_api_key_dict, candidate)) # Per-request-tag rate limits scoped to this key self._add_tag_per_key_rate_limit_descriptor( @@ -3845,6 +4009,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): *await self._create_tag_rate_limit_descriptors(data), ] + async def raise_if_over_limit_before_body(self, user_api_key_dict: UserAPIKeyAuth) -> None: + """Refuse a request from its headers alone once a limit that needs no body is exhausted. + + Counters are only read, never incremented, so an admitted request is still charged by + ``async_pre_call_hook`` after its body is parsed. Limits that need the model are not checked here. + """ + descriptors: Final = await self._build_request_rate_limit_descriptors( + user_api_key_dict=user_api_key_dict, data={}, call_type=None + ) + if not descriptors: + return + response: Final = await self.should_rate_limit( + descriptors=descriptors, + parent_otel_span=user_api_key_dict.parent_otel_span, + read_only=True, + skip_tpm_check=self.tpm_reservation_enabled, + ) + if response["overall_code"] == "OVER_LIMIT": + self._handle_rate_limit_error(response=response, descriptors=descriptors) + async def _release_request_capacity_when_admitted( self, admission: asyncio.Task[RateLimitResponse], @@ -3946,6 +4130,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook") stash: Final = claim_request_stash_for_data(data) + stash.call_type = call_type reuses_request_slot: Final = call_type == CallTypes.call_mcp_tool.value and stash.parallel_slot is not None if reuses_request_slot: stash.parallel_slot_reused_at = self._get_current_time() @@ -4653,12 +4838,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): targets.append(("end_user", user_api_key_end_user_id)) if user_api_key_organization_id: targets.append(("organization", user_api_key_organization_id)) + customer_scope_owners: Final = ( + ("customer_org", standard_logging_metadata.get("user_api_key_customer_org_id")), + ("customer_team", standard_logging_metadata.get("user_api_key_customer_team_id")), + ("customer_member", standard_logging_metadata.get("user_api_key_customer_member_id")), + ) + targets.extend((scope, owner) for scope, owner in customer_scope_owners if owner) model_scope_owners: Final = ( ("model_per_key", user_api_key), ("model_per_team", user_api_key_team_id), ("model_per_end_user", f"{user_api_key_end_user_id}" if user_api_key_end_user_id else None), ("model_per_organization", user_api_key_organization_id), ("model_per_project", user_api_key_project_id), + *((f"model_per_{scope}", owner) for scope, owner in customer_scope_owners), ) if model_group: for group in dict.fromkeys(model_groups or (model_group,)): diff --git a/litellm/proxy/hooks/priority_class_share.py b/litellm/proxy/hooks/priority_class_share.py new file mode 100644 index 00000000000..229afe09856 --- /dev/null +++ b/litellm/proxy/hooks/priority_class_share.py @@ -0,0 +1,127 @@ +"""Splits a priority class's share of a model among the teams active on it. + +Enabled by ``priority_reservation_settings.priority_class_share_splits_equally_among_active_teams``. +""" + +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm._internal_context import with_service_target +from litellm.caching.caching import DualCache +from litellm.redis.stores.rate_limits import RATE_LIMITS_TARGET + +ACTIVE_TEAMS_KEY_PREFIX: Final = "dynamic_rate_limiter:active_teams:" +MEMBER_SEPARATOR: Final = "\x1f" +DEFAULT_CLASS: Final = "\x1edefault" +_POOL_PREFIX: Final = f"{MEMBER_SEPARATOR}pool{MEMBER_SEPARATOR}" + +_TOUCH_LUA: Final = ( + "redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1]) " + "redis.call('ZADD', KEYS[1], ARGV[2], ARGV[3]) " + "redis.call('EXPIRE', KEYS[1], ARGV[4]) " + "return redis.call('ZRANGE', KEYS[1], 0, -1)" +) +_MEMBERS: Final = TypeAdapter[tuple[str | bytes, ...]](tuple[str | bytes, ...]) + + +def pool_member(priority: str) -> str: + """The registry id that stands for every key without a team in ``priority``. It keeps the class present without + being a team, and starts with the separator so no team id can spell it.""" + return f"{_POOL_PREFIX}{priority}" + + +def team_member(team_id: str) -> str: + """The registry id of a real team. It never starts with the separator, so it cannot spell a pool marker.""" + return f"team{MEMBER_SEPARATOR}{team_id}" + + +def is_pool_member(member: str) -> bool: + return member.startswith(_POOL_PREFIX) + + +def counter_value(model: str, team_id: str) -> str: + """One counter per model and team. The length prefix keeps ``chat`` + ``prod:victim`` apart from + ``chat:prod`` + ``victim``.""" + return f"{len(model)}:{model}:{team_id}" + + +@dataclass(frozen=True, slots=True) +class TeamShare: + fraction: float + active_teams_in_class: int + active_classes: int + + +def team_share( + priority: str, members: Mapping[str, str], class_weights: Mapping[str, float], default_weight: float +) -> TeamShare: + """``members`` maps every team active on the model, the caller included, to its priority class. Classes with no + active member give their weight away, and each present class is split equally among its active teams. A pool + member keeps its class present but takes no slice, since keys without a team are held to the class pool.""" + present: Final = frozenset(members.values()) + weights: Final = {cls: class_weights.get(cls, default_weight) for cls in present} + total: Final = sum(weights.values()) + class_fraction: Final = weights[priority] / total if total > 0 else 1.0 / len(present) + teams_in_class: Final = sum(1 for team, cls in members.items() if cls == priority and not is_pool_member(team)) + return TeamShare( + fraction=class_fraction / max(1, teams_in_class), + active_teams_in_class=teams_in_class, + active_classes=len(present), + ) + + +def _members(raw: tuple[str | bytes, ...]) -> Mapping[str, str]: + """Oldest first, so a team seen under two classes in one window keeps its most recent one. The class is split off + from the right because a team id may itself contain the separator.""" + decoded: Final = (item.decode() if isinstance(item, bytes) else item for item in raw) + pairs: Final = (member.rsplit(MEMBER_SEPARATOR, 1) for member in decoded) + return MappingProxyType({pair[0]: pair[1] for pair in pairs if len(pair) == 2}) + + +class ActiveTeamRegistry: + """Which teams used a model in the current window, as one sorted set per model scored by last-seen time, in Redis + when the proxy has one and otherwise in process memory.""" + + def __init__(self, dual_cache: DualCache, time_provider: Callable[[], float] = time.time) -> None: + self._dual_cache = dual_cache + self._local: Mapping[str, Mapping[str, float]] = MappingProxyType({}) + self._now = time_provider + + @with_service_target(RATE_LIMITS_TARGET) + async def touch(self, model: str, team_id: str, priority: str, window: int) -> Mapping[str, str]: + """Record ``team_id`` as active on ``model`` and return every team active on it, the caller included. The + caller is always listed under ``priority``, whatever class another worker's clock last saw it in.""" + key: Final = f"{ACTIVE_TEAMS_KEY_PREFIX}{model}" + member: Final = f"{team_id}{MEMBER_SEPARATOR}{priority}" + now: Final = self._now() + redis_cache: Final = self._dual_cache.redis_cache + seen: Final = ( + self._touch_in_memory(key, member, now, window) + if redis_cache is None + else _members( + _MEMBERS.validate_python( + await redis_cache.async_register_script(_TOUCH_LUA)( + keys=(key,), args=(now - window, now, member, window) + ) + ) + ) + ) + return MappingProxyType({**seen, team_id: priority}) + + def _touch_in_memory(self, key: str, member: str, now: float, window: int) -> Mapping[str, str]: + """Prunes by each member's own last-seen time, so the map lives exactly as long as its newest member.""" + cutoff: Final = now - window + alive: Final = {seen_member: seen for seen_member, seen in self._local.get(key, {}).items() if seen > cutoff} + current: Final = MappingProxyType({**alive, member: now}) + others: Final = { + other_key: others_members + for other_key, others_members in self._local.items() + if other_key != key and any(seen > cutoff for seen in others_members.values()) + } + self._local = MappingProxyType({**others, key: current}) + return _members(tuple(sorted(current, key=current.__getitem__))) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 554fc7ed1d5..91554b0c759 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -36,6 +36,7 @@ from litellm.proxy.auth.auth_checks import ( get_key_object, get_team_object, ) +from litellm.proxy.auth.customer_spend import settle_customer_spend from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.db_spend_update_writer import ( @@ -76,6 +77,10 @@ from litellm.proxy.utils import ( BilledFailedAttempt, ProxyUpdateSpend, ) +from litellm.types.proxy.auth.customer_hierarchy import ( + customer_budget_scopes_from_request, + customer_hierarchy_ids_from_request, +) from litellm.types.proxy.auth.user_api_key_auth import committed_budget_spend_of_row from litellm.types.proxy.carried_budget_state import EntitySpendCounterKeys from litellm.types.router import Deployment @@ -86,6 +91,7 @@ from litellm.types.utils import ( from litellm.utils import get_end_user_id_for_cost_tracking if TYPE_CHECKING: + from litellm.proxy.spend_tracking.budget_reservation import BudgetCounter from litellm.proxy.utils import ProxyLogging @@ -104,6 +110,14 @@ _BATCH_ACCOUNTING: Final[ContextVar[BatchAccountingOutcome | None]] = ContextVar _BATCH_CALLBACK_FIELDS: Final = TypeAdapter(dict[str, object]) +class _CustomerCounterOptions(TypedDict, total=False): + customer_counters: ReadOnly["Sequence[BudgetCounter]"] + + +def _customer_counter_options(counters: "Sequence[BudgetCounter]") -> _CustomerCounterOptions: + return {"customer_counters": counters} if counters else {} + + @contextmanager def track_batch_accounting(call_id: str) -> Generator[BatchAccountingOutcome, None, None]: outcome: Final = BatchAccountingOutcome(call_id) @@ -674,6 +688,16 @@ class _ProxyDBLogger(CustomLogger): token: Final = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) end_user_id: Final = None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id + from litellm.proxy.proxy_server import customer_accounting_counters + + customer_counters: Final = ( + await customer_accounting_counters( + customer_hierarchy_ids_from_request(spend_rows[0][1]), + customer_budget_scopes_from_request(spend_rows[0][1]), + ) + if any(cost > 0 for cost, _ in spend_rows) + else () + ) accounting_rows: Final = tuple((cost, dict(row)) for cost, row in spend_rows) charged: Final = [ await self._spend_writer().update_database( @@ -688,6 +712,7 @@ class _ProxyDBLogger(CustomLogger): end_time=datetime.now(), org_id=user_api_key_dict.org_id, project_id=user_api_key_dict.project_id, + **_customer_counter_options(customer_counters), ) for row_cost, row_kwargs in accounting_rows ] @@ -696,7 +721,7 @@ class _ProxyDBLogger(CustomLogger): ) if counted_rows and not ProxyUpdateSpend.disable_spend_updates(): await self._increment_failure_spend_counters( - user_api_key_dict, token, end_user_id, counted_rows, start_time + user_api_key_dict, token, end_user_id, counted_rows, start_time, customer_counters ) async def _increment_failure_spend_counters( @@ -706,12 +731,15 @@ class _ProxyDBLogger(CustomLogger): end_user_id: str | None, counted_rows: Sequence[tuple[float, Mapping[str, object]]], start_time: datetime, + customer_counters: "Sequence[BudgetCounter]" = (), ) -> None: """The counters the success path raises for the same spend, sent before the client sees the error. A counter store slower than ``failure_counter_wait_seconds`` finishes them in the background instead of holding the error back. The reservation was already released, so every counter takes the full row cost.""" increment: Final = asyncio.create_task( - self._raise_failure_spend_counters(user_api_key_dict, token, end_user_id, counted_rows, start_time) + self._raise_failure_spend_counters( + user_api_key_dict, token, end_user_id, counted_rows, start_time, customer_counters + ) ) self._pending_increments.add(increment) increment.add_done_callback(self._pending_increments.discard) @@ -729,10 +757,11 @@ class _ProxyDBLogger(CustomLogger): end_user_id: str | None, counted_rows: Sequence[tuple[float, Mapping[str, object]]], start_time: datetime, + customer_counters: "Sequence[BudgetCounter]" = (), ) -> None: for row_cost, row_kwargs in counted_rows: await self._raise_row_spend_counters( - user_api_key_dict, token, end_user_id, row_cost, row_kwargs, start_time + user_api_key_dict, token, end_user_id, row_cost, row_kwargs, start_time, customer_counters ) await flush_post_call_redis_batches() @@ -744,6 +773,7 @@ class _ProxyDBLogger(CustomLogger): row_cost: float, row_kwargs: Mapping[str, object], start_time: datetime, + customer_counters: "Sequence[BudgetCounter]" = (), ) -> None: standard_logging_object: Final = row_kwargs.get("standard_logging_object") sl_object: Final = ( @@ -772,6 +802,7 @@ class _ProxyDBLogger(CustomLogger): ), project_id=user_api_key_dict.project_id, committed_spend_scope=_committed_spend_scope(user_api_key_dict), + **_customer_counter_options(customer_counters), **_ledger_counter_options(row_kwargs.get(ACCOUNTING_LEDGER_KWARG)), ) except Exception: # noqa: BLE001 # one row's counter failure must not keep the other rows' counters from landing @@ -1286,6 +1317,7 @@ class _IncrementSpendCounters(Protocol): project_id: str | None = None, committed_spend_scope: EntitySpendCounterKeys | None = None, windows_from_ledger: bool = False, + customer_counters: "Sequence[BudgetCounter]" = (), ) -> None: ... @@ -1339,24 +1371,34 @@ async def _update_database_and_spend_counters( """The reservation is reconciled before the spend is persisted, from its own read. One spend counter batch then spans the database write and the counter update, so the post-call counters are read with a single MGET after the write and their increments leave in a single pipeline.""" - from litellm.proxy.proxy_server import spend_counter_store + from litellm.proxy.proxy_server import customer_accounting_counters, spend_counter_store from litellm.proxy.spend_tracking.budget_reservation import get_reserved_counter_keys + request_fields: Final = _OBJECT_MAPPING.validate_python(kwargs) + customer_counters: Final = ( + await customer_accounting_counters( + customer_hierarchy_ids_from_request(request_fields), customer_budget_scopes_from_request(request_fields) + ) + if response_cost + else () + ) if budget_reservation is not None: await _reconcile_budget_reservation_before_db_update( budget_reservation=budget_reservation, response_cost=response_cost ) - counter_keys: Final = frozenset( - get_reserved_counter_keys(budget_reservation=budget_reservation) - ) | post_call_counter_keys( - token=user_api_key, - team_id=team_id, - user_id=user_id, - org_id=org_id, - end_user_id=end_user_id, - tags=request_tags, - model_access_groups=model_access_groups, - project_id=project_id, + counter_keys: Final = ( + frozenset(counter.counter_key for counter in customer_counters) + | frozenset(get_reserved_counter_keys(budget_reservation=budget_reservation)) + | post_call_counter_keys( + token=user_api_key, + team_id=team_id, + user_id=user_id, + org_id=org_id, + end_user_id=end_user_id, + tags=request_tags, + model_access_groups=model_access_groups, + project_id=project_id, + ) ) with spend_counter_batch_scope(spend_counter_store, counter_keys=counter_keys): return await _update_database_and_spend_counters_in_batch( @@ -1378,6 +1420,7 @@ async def _update_database_and_spend_counters( model_access_groups=model_access_groups, project_id=project_id, committed_spend_scope=committed_spend_scope, + customer_counters=customer_counters, ) @@ -1400,6 +1443,7 @@ async def _update_database_and_spend_counters_in_batch( project_id: str | None, spend_writer: DBSpendUpdateWriter | None = None, committed_spend_scope: EntitySpendCounterKeys | None = None, + customer_counters: "Sequence[BudgetCounter]" = (), ) -> bool: try: writer: Final = spend_writer if spend_writer is not None else proxy_logging_obj.db_spend_update_writer @@ -1422,6 +1466,7 @@ async def _update_database_and_spend_counters_in_batch( end_time=end_time, org_id=org_id, project_id=project_id, + **_customer_counter_options(customer_counters), ) except Exception: if budget_reservation is not None: @@ -1440,7 +1485,14 @@ async def _update_database_and_spend_counters_in_batch( await _release_budget_reservation(budget_reservation=budget_reservation) return False + from litellm.proxy.proxy_server import spend_counter_store + try: + await settle_customer_spend( + _OBJECT_MAPPING.validate_python(get_litellm_metadata_from_kwargs(kwargs=kwargs)), + response_cost, + spend_counter_store, + ) await increment_spend_counters( token=user_api_key, team_id=team_id, @@ -1467,6 +1519,7 @@ async def _update_database_and_spend_counters_in_batch( ) ), **_ledger_counter_options(_BATCH_CALLBACK_FIELDS.validate_python(kwargs).get(ACCOUNTING_LEDGER_KWARG)), + **_customer_counter_options(customer_counters), ) except Exception: if budget_reservation is not None: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8abc91a72ed..63e0e6a4cc1 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -19,7 +19,7 @@ from typing import ( ) from fastapi import HTTPException, Request, status -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers from typing_extensions import ReadOnly, TypedDict @@ -59,6 +59,7 @@ from litellm.litellm_core_utils.core_helpers import ( iter_items, keyword_arguments, normalize_drop_params, + require_attributes, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -93,6 +94,17 @@ from litellm.proxy.common_utils.callback_utils import ( key_logging_entries, strip_callback_config, ) +from litellm.proxy.common_utils.customer_usage import ( + CUSTOMER_USAGE_METADATA_KEY, + CustomerUsageDatabase, + CustomerUsageFailure, + ResolvedCustomerUsage, + customer_usage_fields, + customer_usage_http_status, + customer_usage_key_fields, + customer_usage_requested, + resolve_customer_usage, +) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export detach_request_parsed_body, @@ -100,6 +112,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY +from litellm.types.proxy.auth.customer_hierarchy import customer_budget_scopes_metadata # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE: Final = frozenset(str(v.value).lower() for v in SpecialHeaders) @@ -112,6 +125,68 @@ _CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names ) _TRANSPORT_ONLY_CREDENTIAL_KEYS: Final = frozenset({"provider_specific_header", "headers", "api_key"}) _REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +class _CustomerPolicyCarrier(BaseModel): + model_config = ConfigDict(frozen=True) + + user_api_key_auth: object = None + + +_CUSTOMER_REQUEST_MODELS: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None) + + +def customer_request_auth(data: Mapping[str, object]) -> UserAPIKeyAuth | None: + carriers: Final = tuple( + _CustomerPolicyCarrier.model_validate(customer_usage_fields(data.get(field))) + for field in ("metadata", "litellm_metadata") + ) + return next( + (carrier.user_api_key_auth for carrier in carriers if isinstance(carrier.user_api_key_auth, UserAPIKeyAuth)), + None, + ) + + +def check_customer_request_model_access( + data: Mapping[str, object], auth: UserAPIKeyAuth | None, llm_router: "Router | None" +) -> None: + if auth is None or not auth.customer_model_scopes: + return + from litellm.proxy.auth.auth_checks import check_customer_model_access + from litellm.proxy.auth.auth_utils import request_fallback_model_names + + requested: Final = _CUSTOMER_REQUEST_MODELS.validate_python(data.get("model")) + names: Final = (requested,) if isinstance(requested, str) else (requested or ()) + for model in (*names, *request_fallback_model_names(data)): + check_customer_model_access(model=model, valid_token=auth, llm_router=llm_router) + + +async def resolve_customer_request_usage( + metadata: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, *, ledger_enabled: bool +) -> ResolvedCustomerUsage | None: + from litellm.proxy.proxy_server import prisma_client + + key_metadata: Final = customer_usage_key_fields(user_api_key_dict) + usage: Final = await resolve_customer_usage( + metadata=metadata, + key_metadata=key_metadata, + administrator=user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN, + database=require_attributes( + prisma_client.db, + CustomerUsageDatabase, + "litellm_customerorganizationtable", + "litellm_customerteamtable", + "litellm_customermembertable", + ) + if prisma_client is not None and customer_usage_requested(metadata, key_metadata) + else None, + ledger_enabled=ledger_enabled, + ) + if isinstance(usage, CustomerUsageFailure): + raise HTTPException(status_code=customer_usage_http_status(usage.error), detail=usage.error.value) + return usage + + _REQUEST_ENTRIES: Final = TypeAdapter(tuple[object, ...]) # Matches any header of the form x--session-id (case-insensitive). @@ -1745,7 +1820,7 @@ class LiteLLMProxyRequestSetup: return None @staticmethod - def _get_spend_logs_metadata_from_request_headers(headers: dict) -> dict | None: + def get_spend_logs_metadata_from_request_headers(headers: Mapping[str, str]) -> object: """ Get the `spend_logs_metadata` from the request headers. """ @@ -1987,12 +2062,14 @@ class LiteLLMProxyRequestSetup: """ from litellm.proxy._types import LitellmMetadataFromRequestHeaders - metadata_from_headers: Final = LitellmMetadataFromRequestHeaders() request_body: Final = cast(Mapping[str, object], data) # cast-ok: request body is a str-keyed JSON object _drop_client_session_metadata(request_body) - spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) - if spend_logs_metadata is not None: - metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata + spend_logs_metadata: Final = LiteLLMProxyRequestSetup.get_spend_logs_metadata_from_request_headers(headers) + metadata_from_headers: Final = ( + LitellmMetadataFromRequestHeaders(spend_logs_metadata=spend_logs_metadata) + if spend_logs_metadata is not None + else LitellmMetadataFromRequestHeaders() + ) ######################################################################################### # Finally update the requests metadata with the `metadata_from_headers` @@ -2108,6 +2185,7 @@ class LiteLLMProxyRequestSetup: ) -> StandardLoggingUserAPIKeyMetadata: stripped_metadata: Final = strip_callback_config(user_api_key_dict.metadata) auth_metadata: Final = cast("dict[str, str] | None", stripped_metadata) # cast-ok: metadata is free-form JSON + customer: Final = user_api_key_dict.customer_hierarchy user_api_key_logged_metadata: Final = StandardLoggingUserAPIKeyMetadata( user_api_key_hash=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), user_api_key_alias=user_api_key_dict.key_alias, @@ -2125,6 +2203,14 @@ class LiteLLMProxyRequestSetup: user_api_key_org_alias=user_api_key_dict.organization_alias, user_api_key_team_alias=user_api_key_dict.team_alias, user_api_key_end_user_id=user_api_key_dict.end_user_id, + user_api_key_customer_member_id=customer.customer_member_id if customer else None, + user_api_key_customer_team_id=customer.customer_team_id if customer else None, + user_api_key_customer_org_id=customer.customer_org_id if customer else None, + user_api_key_customer_org_alias=customer.customer_org_alias if customer else None, + user_api_key_customer_team_alias=customer.customer_team_alias if customer else None, + user_api_key_customer_member_alias=customer.customer_member_alias if customer else None, + user_api_key_customer_budget_scopes=customer_budget_scopes_metadata(customer), + user_api_key_customer_hierarchy=customer.model_dump(mode="json") if customer else None, user_api_key_user_email=user_api_key_dict.user_email, user_api_key_request_route=user_api_key_dict.request_route, user_api_key_budget_reset_at=( @@ -2989,9 +3075,25 @@ async def add_litellm_data_to_request( if _metadata_variable_name == "metadata": data.pop("litellm_metadata", None) + _customer_metadata: Final = customer_usage_fields(_REQUEST_FIELDS.validate_python(data)[_metadata_variable_name]) + _customer_usage: Final = await resolve_customer_request_usage( + _customer_metadata, + user_api_key_dict, + ledger_enabled=general_settings_view is None + or general_settings_view.get("use_spend_logs_ledger", True) is not False, + ) + if _customer_usage is not None: + data[_metadata_variable_name][CUSTOMER_USAGE_METADATA_KEY] = ( # rebind-ok: data is an out-param + _customer_usage.attribution.model_dump_json() + ) + + _customer_request_auth: Final = user_api_key_dict.with_customer_model_scopes( + _customer_usage.model_scopes if _customer_usage is not None else () + ) + data = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=data, - user_api_key_dict=user_api_key_dict, + user_api_key_dict=_customer_request_auth, _metadata_variable_name=_metadata_variable_name, ) data[_metadata_variable_name]["litellm_api_version"] = version @@ -3208,6 +3310,8 @@ async def add_litellm_data_to_request( if holds_api_key_secret_reference(data): # pyright: ignore[reportUnknownArgumentType] # data is an untyped dict raise HTTPException(status_code=400, detail=API_KEY_SECRET_REFERENCE_REJECTION) + check_customer_request_model_access(_REQUEST_FIELDS.validate_python(data), _customer_request_auth, llm_router) + ## ENFORCED PARAMS CHECK # loop through each enforced param # example enforced_params ['user', 'metadata', 'metadata.generation_name'] diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 10bdd1d2353..fb7c98a4b75 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, Depends, HTTPException, status from typing_extensions import ReadOnly, TypedDict +import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import iter_items @@ -35,6 +36,7 @@ from litellm.proxy.management_helpers.resource_display_names import ( agent_display_names, key_display_names, mcp_server_display_names, + vector_store_display_names, ) from litellm.proxy.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw @@ -67,6 +69,9 @@ class _AccessGroupRecord(Protocol): @property def access_agent_ids(self) -> Sequence[str] | None: ... + @property + def access_vector_store_ids(self) -> Sequence[str] | None: ... + @property def assigned_team_ids(self) -> Sequence[str] | None: ... @@ -216,6 +221,7 @@ def _explicitly_set_fields(data: AccessGroupUpdateRequest) -> Mapping[str, objec class _ResourceNames: mcp_servers: Mapping[str, str] agents: Mapping[str, str] + vector_stores: Mapping[str, str] teams: Mapping[str, str | None] keys: Mapping[str, str] @@ -233,6 +239,7 @@ def _record_to_response( "assigned_team_ids": assigned_team_ids, "access_mcp_servers": _label(record.access_mcp_server_ids or (), names.mcp_servers), "access_agents": _label(record.access_agent_ids or (), names.agents), + "access_vector_stores": _label(record.access_vector_store_ids or (), names.vector_stores), "assigned_teams": _label(assigned_team_ids, names.teams), "assigned_keys": _label(record.assigned_key_ids or (), names.keys), } @@ -252,7 +259,7 @@ async def _responses_for( if not records: return () teams: Final = await _teams_touching(TeamRepository(prisma_client).table, records) - mcp_servers, agents, keys = await asyncio.gather( + mcp_servers, agents, vector_stores, keys = await asyncio.gather( mcp_server_display_names( prisma_client, _ids_across(records, lambda record: record.access_mcp_server_ids), @@ -261,11 +268,17 @@ async def _responses_for( agent_display_names( prisma_client, _ids_across(records, lambda record: record.access_agent_ids), global_agent_registry ), + vector_store_display_names( + prisma_client, + _ids_across(records, lambda record: record.access_vector_store_ids), + litellm.vector_store_registry.vector_stores if litellm.vector_store_registry is not None else (), + ), key_display_names(prisma_client, _ids_across(records, lambda record: record.assigned_key_ids)), ) names: Final = _ResourceNames( mcp_servers=mcp_servers, agents=agents, + vector_stores=vector_stores, teams=MappingProxyType({team.team_id: team.team_alias for team in teams}), keys=keys, ) @@ -578,6 +591,7 @@ async def create_access_group( "mcp_tool_permissions": _tool_permissions_json(data.mcp_tool_permissions), "access_agent_ids": data.access_agent_ids or [], "access_skill_names": list(data.access_skill_names or ()), + "access_vector_store_ids": list(data.access_vector_store_ids or ()), "assigned_team_ids": data.assigned_team_ids or [], "assigned_key_ids": data.assigned_key_ids or [], "created_by": user_api_key_dict.user_id, @@ -676,6 +690,7 @@ async def update_access_group( "access_mcp_server_ids", "access_agent_ids", "access_skill_names", + "access_vector_store_ids", ): value = list(iter_items(value or ())) update_data[field] = ( diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py index 0ab559ce009..091b88bb046 100644 --- a/litellm/proxy/management_endpoints/budget_management_endpoints.py +++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py @@ -14,32 +14,40 @@ All /budget management endpoints #### BUDGET TABLE MANAGEMENT #### import math from collections.abc import Mapping +from datetime import datetime, timezone from types import MappingProxyType -from typing import Final +from typing import Annotated, Final from fastapi import APIRouter, Depends, HTTPException +from pydantic import TypeAdapter from litellm._internal_context import service_target from litellm._logging import verbose_proxy_logger +from litellm.models.budget import initialize_budget_limits from litellm.proxy._types import * +from litellm.proxy.auth.customer_hierarchy_auth import forget_customer_hierarchies, forget_customer_organization_aliases from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time +from litellm.proxy.common_utils.timezone_utils import get_budget_reset_settings, get_budget_reset_time from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # legacy module exports _user_has_admin_view, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export user_api_key_has_admin_view, validate_budget_duration, ) +from litellm.proxy.management_helpers.customer_hierarchy import budget_limit_json +from litellm.proxy.management_helpers.utils import validate_budget_windows_update from litellm.proxy.utils import jsonify_object from litellm.repositories.budget_repository import BudgetRepository +from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE +from litellm.repositories.table_repositories import ( + CustomerMemberRepository, + CustomerOrganizationRepository, + CustomerTeamRepository, +) router: Final = APIRouter() +_WRITE_VALUES: Final = TypeAdapter(Mapping[str, object]) -@router.post( - "/budget/new", - tags=["budget management"], - dependencies=[Depends(user_api_key_auth)], -) async def new_budget( budget_obj: BudgetNewRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -49,6 +57,7 @@ async def new_budget( Parameters: - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.) + - budget_limits: Optional[list] - Concurrent budget windows with independent caps and reset times. - budget_id: Optional[str] - The id of the budget. If not provided, a new id will be generated. - max_budget: Optional[float] - The max budget for the budget. - soft_budget: Optional[float] - The soft budget for the budget. @@ -97,15 +106,30 @@ async def new_budget( raise HTTPException(status_code=400, detail={"error": str(e)}) # if no budget_reset_at date is set, but a budget_duration is given, then set budget_reset_at initially to the first completed duration interval in future - if budget_obj.budget_reset_at is None and budget_obj.budget_duration is not None: - budget_obj.budget_reset_at = get_budget_reset_time(budget_duration=budget_obj.budget_duration) + reset: Final = ( + {"budget_reset_at": get_budget_reset_time(budget_duration=budget_obj.budget_duration)} + if budget_obj.budget_reset_at is None and budget_obj.budget_duration is not None + else {} + ) - budget_obj_json: Final = budget_obj.model_dump(exclude_none=True) + budget_obj_json: Final = {**budget_obj.model_dump(exclude_none=True), **reset} budget_obj_jsonified: Final[dict[str, object]] = jsonify_object(budget_obj_json) # mutable-ok: prisma create input + window_data: Final = ( + { + "budget_limits": budget_limit_json( + initialize_budget_limits( + budget_obj.budget_limits, get_budget_reset_settings(), datetime.now(timezone.utc) + ) + ) + } + if isinstance(budget_obj, BudgetWriteRequest) and budget_obj.budget_limits is not None + else {} + ) try: response: Final = await BudgetRepository(prisma_client).table.create( data={ **budget_obj_jsonified, + **window_data, "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, } @@ -121,11 +145,6 @@ async def new_budget( return response -@router.post( - "/budget/update", - tags=["budget management"], - dependencies=[Depends(user_api_key_auth)], -) async def update_budget( budget_obj: BudgetNewRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -135,6 +154,7 @@ async def update_budget( Parameters: - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.) + - budget_limits: Optional[list] - Replace concurrent budget windows. Null or an empty list clears them. - budget_id: Optional[str] - The id of the budget. If not provided, a new id will be generated. - max_budget: Optional[float] - The max budget for the budget. - soft_budget: Optional[float] - The soft budget for the budget. @@ -193,17 +213,61 @@ async def update_budget( else {} ) - budget_obj_jsonified: Final[Mapping[str, object]] = jsonify_object( - { - **budget_obj.model_dump(exclude_unset=True), - **recomputed_reset_at, - "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, - } + budget_obj_jsonified: Final = _WRITE_VALUES.validate_python( + jsonify_object( + { + **budget_obj.model_dump(exclude_unset=True), + **recomputed_reset_at, + "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, + } + ) ) + window_data: Final = ( + { + "budget_limits": budget_limit_json( + initialize_budget_limits( + budget_obj.budget_limits, get_budget_reset_settings(), datetime.now(timezone.utc) + ) + ) + } + if isinstance(budget_obj, BudgetWriteRequest) and budget_obj.budget_limits is not None + else {} + ) + if isinstance(budget_obj, BudgetWriteRequest) and budget_obj.budget_limits: + await validate_budget_windows_update(prisma_client.db, budget_obj.budget_id) response: Final = await BudgetRepository(prisma_client).table.update( - where={"budget_id": budget_obj.budget_id}, - data=budget_obj_jsonified, + where={"budget_id": budget_obj.budget_id}, data={**budget_obj_jsonified, **window_data} + ) + members: Final = await CustomerMemberRepository(prisma_client).table.find_many( + where={ + "OR": ( + {"budget_id": budget_obj.budget_id}, + {"budget_override_id": budget_obj.budget_id}, + {"parent_org": {"is": {"budget_id": budget_obj.budget_id}}}, + ) + } + ) + await forget_customer_hierarchies(tuple(member.customer_member_id for member in members), user_api_key_cache) + teams: Final = await CustomerTeamRepository(prisma_client).table.find_many( + where={"OR": ({"budget_id": budget_obj.budget_id}, {"budget_override_id": budget_obj.budget_id})} + ) + team_ids: Final = tuple(team.customer_team_id for team in teams) + for offset in range(0, len(team_ids), IN_LIST_CHUNK_SIZE): + await forget_customer_hierarchies( + tuple( + member.customer_member_id + for member in await CustomerMemberRepository(prisma_client).table.find_many( + where={"parent_team_ids": {"has_some": team_ids[offset : offset + IN_LIST_CHUNK_SIZE]}} + ) + ), + user_api_key_cache, + ) + organizations: Final = await CustomerOrganizationRepository(prisma_client).table.find_many( + where={"budget_id": budget_obj.budget_id} + ) + await forget_customer_organization_aliases( + tuple(organization.alias for organization in organizations), user_api_key_cache ) with service_target("team_member_budgets"): try: @@ -214,6 +278,22 @@ async def update_budget( return response +@router.post("/budget/new", tags=["budget management"], description=new_budget.__doc__) +async def new_budget_endpoint( + budget_obj: BudgetWriteRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> object: + return await new_budget(budget_obj, user_api_key_dict) + + +@router.post("/budget/update", tags=["budget management"], description=update_budget.__doc__) +async def update_budget_endpoint( + budget_obj: BudgetWriteRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> object: + return await update_budget(budget_obj, user_api_key_dict) + + @router.post( "/budget/info", tags=["budget management"], @@ -293,11 +373,12 @@ async def budget_settings( "model_max_budget": {"type": "Object"}, "model_tpm_limit": {"type": "Object"}, "model_rpm_limit": {"type": "Object"}, + "budget_limits": {"type": "Object"}, } return_val: Final = [] - for field_name, field_info in BudgetNewRequest.model_fields.items(): + for field_name, field_info in BudgetWriteRequest.model_fields.items(): if field_name in allowed_args: _stored_in_db = True @@ -371,6 +452,18 @@ async def delete_budget( detail={"error": f"{CommonProxyErrors.not_allowed_access.value}, your role={user_api_key_dict.user_role}"}, ) + customer_org_count: Final = await CustomerOrganizationRepository(prisma_client).table.count( + where={"budget_template_ids": {"has": data.id}} + ) + if customer_org_count: + raise HTTPException( + status_code=409, + detail={ + "error": "Budget is a customer organization default budget. Delete it through " + "/management/v1/customer_organizations/{customer_org_id}/default_budgets/{budget_id}" + }, + ) + response: Final = await BudgetRepository(prisma_client).table.delete(where={"budget_id": data.id}) return response diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 983c3d24513..174931be397 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -656,6 +656,8 @@ async def upsert_budget_and_membership( A patch whose every field already matches the shared row changes nothing and is a no-op rather than a clone. """ + from litellm.proxy.management_helpers.utils import reject_budget_windows + ignore_default_model_budget_clear: Final = ( existing_budget_id is not None and existing_budget_id == team_default_budget_id @@ -694,7 +696,10 @@ async def upsert_budget_and_membership( if existing_budget_id is not None and not is_shared_default: existing_budget: Final = await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) - merged: Final = existing_budget.model_dump() if existing_budget is not None else {} + merged: Final = ( + _BUDGET_COLUMNS.validate_python(existing_budget.model_dump()) if existing_budget is not None else {} + ) + reject_budget_windows(merged) merged.update(write_data) if not _has_meaningful_budget_limit(merged): await _disconnect() @@ -713,6 +718,7 @@ async def upsert_budget_and_membership( if source_row is not None else MappingProxyType({}) ) + reject_budget_windows(source) if is_shared_default and all(source.get(field) == value for field, value in effective_patch.items()): return diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 24612ce99bd..530f717ffe6 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -12,7 +12,6 @@ All /customer management endpoints #### END-USER/CUSTOMER MANAGEMENT #### from collections.abc import Mapping, Sequence from collections.abc import Set as AbstractSet -from datetime import datetime, timedelta from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, Protocol, TypeVar, overload @@ -28,9 +27,9 @@ if TYPE_CHECKING: import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import ( end_user_cache_key, end_user_restricted_registry_cache_key, @@ -42,6 +41,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: handle_update_object_permission_common, set_object_permission, ) +from litellm.proxy.management_helpers.utils import validate_scalar_budget_attachment from litellm.proxy.spend_tracking.committed_spend import drop_committed_spend from litellm.proxy.utils import handle_exception_on_proxy from litellm.redis.stores.spend_counters import end_user_counter @@ -287,9 +287,7 @@ def new_budget_request(data: NewCustomerRequest) -> BudgetNewRequest | None: budget_request: Final = BudgetNewRequest.model_validate(budget_kv_pairs) validate_budget_duration(budget_request.budget_duration) if budget_request.budget_reset_at is None and budget_request.budget_duration is not None: - budget_request.budget_reset_at = datetime.utcnow() + timedelta( - seconds=duration_in_seconds(duration=budget_request.budget_duration) - ) + budget_request.budget_reset_at = get_budget_reset_time(budget_duration=budget_request.budget_duration) return budget_request return None @@ -466,6 +464,7 @@ async def new_end_user( new_end_user_obj["budget_id"] = budget_record.budget_id elif data.budget_id is not None: + await validate_scalar_budget_attachment(prisma_client, data.budget_id) new_end_user_obj["budget_id"] = data.budget_id _user_data: Final = _STR_OBJECT_DICT.validate_python(data.dict(exclude_none=True)) @@ -679,6 +678,7 @@ async def update_end_user( if prisma_client is None: raise Exception("Not connected to DB!") + await validate_scalar_budget_attachment(prisma_client, data.budget_id) sent_fields: Final = data.fields_set() non_default_values: Final[dict[str, object]] = { k: v for k, v in data_json.items() if _should_update_field(k, v, sent_fields) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index f973fadb53c..a2c32cdb008 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -79,9 +79,11 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l from litellm.proxy.management_endpoints.key_management_endpoints import ( # noqa: F401 # legacy module exports _check_permissions_caller_permission, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_permissions_caller_permission, + customer_key_actor, generate_key_helper_fn, prepare_metadata_fields, raise_on_logging_credential_change, + validated_customer_key_metadata, ) from litellm.proxy.management_helpers.bulk_user_deletion import delete_users_from_current_teams from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: F401 # legacy module exports @@ -560,17 +562,15 @@ async def new_user( }' ``` """ + actor: Final = customer_key_actor(user_api_key_dict) + if data.auto_create_key and actor is not None: + validated_customer_key_metadata(data.metadata, None, actor) try: from litellm.proxy.proxy_server import _license_check, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) - if prisma_client is None: - raise HTTPException( - status_code=500, - detail=CommonProxyErrors.db_not_connected_error.value, - ) validate_budget_duration(data.budget_duration) # Check for duplicate user_id or email @@ -629,7 +629,12 @@ async def new_user( teams = check_if_default_team_set() organization_ids: Final = cast(list[str] | None, data_json.pop("organizations", None)) - response: Final = await generate_key_helper_fn(request_type="user", **data_json, llm_router=None) + response: Final = await generate_key_helper_fn( + request_type="user", + **data_json, + llm_router=None, + actor=actor, + ) created_user_id: Final = cast(str | None, response.get("user_id", None)) if created_user_id is not None: forget_missing_user(created_user_id) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 3bd17c4b5d8..7753341b48c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -82,6 +82,12 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) +from litellm.proxy.common_utils.customer_usage import ( + CustomerUsageFailure, + customer_usage_fields, + customer_usage_http_status, + normalize_customer_key_metadata, +) from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time, keeps_due_budget_reset @@ -153,7 +159,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, ) -from litellm.proxy.management_helpers.utils import management_endpoint_wrapper +from litellm.proxy.management_helpers.utils import management_endpoint_wrapper, validate_scalar_budget_attachment from litellm.proxy.mcp_server.db import ( rotate_mcp_server_credentials_master_key, rotate_mcp_user_credentials_master_key, @@ -1479,6 +1485,25 @@ async def _stored_user_role(prisma_client: PrismaClient | None, user_id: str | N return user_row.user_role if user_row is not None else None +def validated_customer_key_metadata( + metadata: object, + existing_metadata: object, + actor: UserAPIKeyAuth | None, +) -> Mapping[str, object] | None: + binding: Final = normalize_customer_key_metadata( + customer_usage_fields(metadata) if metadata is not None else None, + customer_usage_fields(existing_metadata) if existing_metadata is not None else None, + administrator=actor is not None and actor.user_role == LitellmUserRoles.PROXY_ADMIN.value, + ) + if isinstance(binding, CustomerUsageFailure): + raise HTTPException(status_code=customer_usage_http_status(binding.error), detail=binding.error.value) + return binding + + +def customer_key_actor(value: object) -> UserAPIKeyAuth | None: + return value if isinstance(value, UserAPIKeyAuth) else None + + async def _common_key_generation_helper( data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -1493,6 +1518,8 @@ async def _common_key_generation_helper( prisma_client, ) + validated_customer_key_metadata(data.metadata, None, user_api_key_dict) + acting_as_team_admin: Final = _team_admin_requests_member_key( data=data, team_table=team_table, @@ -1520,6 +1547,8 @@ async def _common_key_generation_helper( ) validate_budget_duration(data.budget_duration) + if prisma_client is not None: + await validate_scalar_budget_attachment(prisma_client, data.budget_id) raise_on_invalid_key_logging_config(data.metadata) raise_on_logging_credential_change(data.metadata, None, user_api_key_dict) @@ -1857,7 +1886,7 @@ async def _common_key_generation_helper( try: response = await generate_key_helper_fn( - request_type="key", **data_json, table_name="key", llm_router=llm_router + request_type="key", **data_json, table_name="key", llm_router=llm_router, actor=user_api_key_dict ) except KeyProjectBindingError: if prisma_client is not None: @@ -3062,6 +3091,7 @@ async def prepare_key_update_data( *, prisma_client: PrismaClient | None = None, llm_router: Router | None = None, + user_api_key_dict: UserAPIKeyAuth | None = None, ): if data.router_settings is not None or ( "router_settings" not in data.model_fields_set @@ -3080,6 +3110,13 @@ async def prepare_key_update_data( llm_router=llm_router, ) data_json: Final[dict] = data.model_dump(exclude_unset=True) + if "metadata" in data_json: + customer_binding: Final = validated_customer_key_metadata( + data.metadata, + _StoredKeyMetadata.model_validate(existing_key_row, from_attributes=True).metadata, + user_api_key_dict, + ) + data_json["metadata"] = dict(customer_binding) if customer_binding is not None else None data_json.pop("key", None) data_json.pop("new_key", None) data_json.pop("grace_period", None) # Request-only param, not a DB column @@ -3477,7 +3514,11 @@ async def _process_single_key_update( # Prepare update data raise_on_logging_credential_change(key_request.metadata, existing_key_row, user_api_key_dict) non_default_values = await prepare_key_update_data( - data=key_request, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router + data=key_request, + existing_key_row=existing_key_row, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) await _enforce_custom_key_policy( @@ -3665,7 +3706,9 @@ async def _validate_end_user_budget_id_change( raise HTTPException(status_code=403, detail=forbidden_detail) if requested_budget_id == "": return - budget_row: Final = await BudgetRepository(_require_prisma_client(prisma_client)).find_by_id(requested_budget_id) + budget_row: Final = await validate_scalar_budget_attachment( + _require_prisma_client(prisma_client), requested_budget_id + ) if budget_row is None: missing_detail: Final = {"error": f"end_user_budget_id={requested_budget_id} does not match any budget."} raise HTTPException(status_code=400, detail=missing_detail) @@ -3896,6 +3939,7 @@ async def _validate_update_key_data( user_api_key_dict=user_api_key_dict, prisma_client=checked_prisma_client, ) + await validate_scalar_budget_attachment(checked_prisma_client, data.budget_id) enforce_output_token_estimates_are_admin_only( data=data, @@ -4231,7 +4275,11 @@ async def update_key_fn( _enforce_upperbound_key_params(data, fill_defaults=False) raise_on_logging_credential_change(data.metadata, existing_key_row, user_api_key_dict) non_default_values: Final = await prepare_key_update_data( - data=data, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router + data=data, + existing_key_row=existing_key_row, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) # Only validate key_alias format if it's actually being changed @@ -5384,7 +5432,9 @@ async def generate_key_helper_fn( budget_limits: list | None = None, # multiple concurrent budget windows *, llm_router: Router | None = None, + actor: UserAPIKeyAuth | None = None, ): + customer_metadata: Final = validated_customer_key_metadata(_DECODED_JSON.validate_python(metadata), None, actor) from litellm.proxy.proxy_server import premium_user, prisma_client if prisma_client is None: @@ -5436,7 +5486,7 @@ async def generate_key_helper_fn( router_settings_json: Final = safe_dumps(router_settings) if router_settings is not None else safe_dumps({}) metadata_json: Final = metadata_json_with_limits( - metadata, + customer_metadata, model_rpm_limit=model_rpm_limit, model_tpm_limit=model_tpm_limit, mcp_rpm_limit=mcp_rpm_limit, @@ -6419,7 +6469,11 @@ async def _execute_virtual_key_regeneration( _enforce_upperbound_key_params(data, fill_defaults=False) raise_on_logging_credential_change(data.metadata, key_in_db, user_api_key_dict) non_default_values = await prepare_key_update_data( - data=data, existing_key_row=key_in_db, prisma_client=prisma_client, llm_router=llm_router + data=data, + existing_key_row=key_in_db, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) # Only validate key_alias format if it's actually being changed new_key_alias: Final = non_default_values.get("key_alias") diff --git a/litellm/proxy/management_endpoints/management_v1/__init__.py b/litellm/proxy/management_endpoints/management_v1/__init__.py index a02d989cc28..41e0c2b574e 100644 --- a/litellm/proxy/management_endpoints/management_v1/__init__.py +++ b/litellm/proxy/management_endpoints/management_v1/__init__.py @@ -8,6 +8,15 @@ from litellm.proxy.management.users.endpoints import router as current_user_rout from litellm.proxy.management_endpoints.management_v1.budgets import ( router as budgets_router, ) +from litellm.proxy.management_endpoints.management_v1.customer_budget_increase_requests import ( + router as customer_budget_increase_requests_router, +) +from litellm.proxy.management_endpoints.management_v1.customer_usage import ( + router as customer_usage_router, +) +from litellm.proxy.management_endpoints.management_v1.customers import ( + router as customers_router, +) from litellm.proxy.management_endpoints.management_v1.spend_logs import ( router as spend_logs_router, ) @@ -20,6 +29,9 @@ from litellm.proxy.management_endpoints.management_v1.users import ( router: Final = APIRouter() router.include_router(budgets_router) +router.include_router(customers_router) +router.include_router(customer_budget_increase_requests_router) +router.include_router(customer_usage_router) router.include_router(spend_logs_router) router.include_router(teams_router) router.include_router(users_router) diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 4c8d8fdf302..4688c4a88d9 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, Request from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.models.budget import BudgetLimits from litellm.proxy._types import ( CommonProxyErrors, UserAPIKeyAuth, @@ -62,6 +63,7 @@ class BudgetListItem(LiteLLMBaseModel): rpm_limit: int | None = None tpd_limit: int | None = None budget_duration: str | None = None + budget_limits: BudgetLimits | None = None budget_reset_at: datetime | None = None created_at: datetime updated_at: datetime diff --git a/litellm/proxy/management_endpoints/management_v1/customer_budget_increase_requests.py b/litellm/proxy/management_endpoints/management_v1/customer_budget_increase_requests.py new file mode 100644 index 00000000000..d82477c91d3 --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/customer_budget_increase_requests.py @@ -0,0 +1,236 @@ +"""Temporary budget increases for paused customer members. + +Same request and approval shape as MCP server and skill submissions: a key acting for the member files a request that waits +in ``pending_review``, a proxy admin approves or rejects it, and an approval lands as the member's budget override. +""" + +from collections.abc import Awaitable, Callable +from datetime import datetime +from typing import Annotated, Final, Literal, TypeAlias, TypeVar +from uuid import uuid4 + +from fastapi import APIRouter, Depends, Query, Request +from pydantic import TypeAdapter + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.customer_hierarchy_auth import forget_customer_hierarchies +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_endpoints.management_v1.customers import ( + customer_actor, + get_customer_hierarchy, + get_user_api_key_cache, + list_response, + resolve, +) +from litellm.proxy.management_helpers.customer_hierarchy import ( + CustomerHierarchy, + CustomerHierarchyStore, + ErrorCode, + Failure, + Page, + PrismaRecord, +) +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.repositories.prisma_protocols import TableActions +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + Budget, + BudgetIncreaseRejection, + BudgetIncreaseRequest, + BudgetIncreaseRequestCreate, + CustomerMember, + IncreaseRequestStatus, + MemberPatch, +) +from litellm.types.proxy.management_endpoints.management_v1 import ListResponse + +router: Final = APIRouter( + prefix=f"{MANAGEMENT_V1_PREFIX}/customer_budget_increase_requests", tags=["customer management"] +) + +Hierarchy: TypeAlias = Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)] +Caller: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)] +Admin: TypeAlias = Annotated[str, Depends(customer_actor)] +Cache: TypeAlias = Annotated[UserApiKeyCache, Depends(get_user_api_key_cache)] + +T: Final = TypeVar("T") +_REQUEST: Final = TypeAdapter(BudgetIncreaseRequest) +_REQUESTS: Final = TypeAdapter(tuple[BudgetIncreaseRequest, ...]) + + +def _requests_table(store: CustomerHierarchyStore) -> TableActions[PrismaRecord]: + return store.db.litellm_customerbudgetincreaserequest # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportAny] # prisma tables are generated at build time + + +async def _in_transaction( + hierarchy: CustomerHierarchy, operation: Callable[[CustomerHierarchyStore], Awaitable[T]] +) -> T: + """Every request-table read and write runs through the hierarchy's transaction, which also maps Prisma errors.""" + return resolve(await hierarchy.transaction(operation)) + + +def _allows_increase(member: CustomerMember) -> bool: + return member.effective_budget.cap_policy == "pause_allow_increase" + + +def _active_override(member: CustomerMember, now: datetime) -> tuple[Budget, datetime | None]: + expires_at: Final = member.budget_override_expires_at + if member.budget_override is None or (expires_at is not None and expires_at <= now): + return Budget(), None + return member.budget_override, expires_at + + +def _grant(member: CustomerMember, request: BudgetIncreaseRequest, now: datetime) -> MemberPatch: + """The approved cap rides on the member's active override and ends no later than that override already did, + so model and rate grants are never renewed or revived by a budget approval.""" + current, current_expiry = _active_override(member, now) + requested_expiry: Final = request.expires_at if request.expires_at is not None else current_expiry + expiry: Final = ( + requested_expiry + if current_expiry is None or requested_expiry is None + else min(current_expiry, requested_expiry) + ) + return MemberPatch( + budget_override=Budget.model_validate({**current.model_dump(), "max_budget": request.requested_max_budget}), + budget_override_expires_at=expiry, + ) + + +def _check_key_scope(caller: UserAPIKeyAuth, member: CustomerMember) -> None: + """A key whose metadata pins ``customer_org_id`` may only file for members of that organization.""" + key_customer_org_id: Final = (caller.metadata or {}).get("customer_org_id") # pyright: ignore[reportUnknownMemberType] # UserAPIKeyAuth.metadata is an untyped dict + if key_customer_org_id is not None and key_customer_org_id != member.parent_org_id: + resolve(Failure(ErrorCode.KEY_NOT_IN_ORG)) + + +@router.post("", response_model=BudgetIncreaseRequest, status_code=201) +@management_endpoint_wrapper +async def request_budget_increase( + data: BudgetIncreaseRequestCreate, caller: Caller, hierarchy: Hierarchy +) -> BudgetIncreaseRequest: + """File a temporary increase for a member whose budget is set to pause_allow_increase. Waits for an admin.""" + member: Final = resolve(await hierarchy.get_member(data.customer_member_id)) + _check_key_scope(caller, member) + if not _allows_increase(member): + resolve(Failure(ErrorCode.INCREASE_NOT_ALLOWED)) + now: Final = hierarchy.clock() + if data.expires_at <= now: + resolve(Failure(ErrorCode.EXPIRY_IN_PAST)) + pending_count: Final = await _in_transaction( + hierarchy, + lambda store: _requests_table(store).count( + where={"customer_member_id": data.customer_member_id, "status": "pending_review"} + ), + ) + if pending_count > 0: + resolve(Failure(ErrorCode.REQUEST_ALREADY_PENDING)) + row: Final = await _in_transaction( + hierarchy, + lambda store: _requests_table(store).create( + data={ + "request_id": str(uuid4()), + "customer_member_id": data.customer_member_id, + "requested_max_budget": data.requested_max_budget, + "expires_at": data.expires_at, + "reason": data.reason, + "status": "pending_review", + "submitted_by": caller.user_id or caller.key_alias or "unknown", + "submitted_at": now, + } + ), + ) + return _REQUEST.validate_python(row, from_attributes=True) + + +@router.get("", response_model=ListResponse[BudgetIncreaseRequest]) +@management_endpoint_wrapper +async def list_budget_increase_requests( + request: Request, + actor: Admin, + hierarchy: Hierarchy, + status: Annotated[IncreaseRequestStatus | None, Query()] = None, + customer_member_id: Annotated[str | None, Query(max_length=256)] = None, + page: Annotated[int, Query(ge=1)] = 1, + page_size: Annotated[int, Query(ge=1, le=200)] = 50, +) -> ListResponse[BudgetIncreaseRequest]: + """The admin review queue. Proxy admins only.""" + where: Final = { + **({"status": status} if status is not None else {}), + **({"customer_member_id": customer_member_id} if customer_member_id is not None else {}), + } + rows: Final = await _in_transaction( + hierarchy, + lambda store: _requests_table(store).find_many( + where=where, order=[{"submitted_at": "desc"}], skip=(page - 1) * page_size, take=page_size + ), + ) + total: Final = await _in_transaction(hierarchy, lambda store: _requests_table(store).count(where=where)) + return list_response( + Page(data=_REQUESTS.validate_python(rows, from_attributes=True), total_count=total), request, page, page_size + ) + + +async def _claim( + store: CustomerHierarchyStore, + request_id: str, + actor: str, + status: Literal["approved", "rejected"], + notes: str | None, +) -> BudgetIncreaseRequest: + """Flip the request out of ``pending_review`` only if it is still there, so two reviewers cannot both win.""" + claimed: Final = await _requests_table(store).update_many( + where={"request_id": request_id, "status": "pending_review"}, + data={"status": status, "reviewed_by": actor, "reviewed_at": store.now, "review_notes": notes}, + ) + row: Final = await _requests_table(store).find_unique(where={"request_id": request_id}) + if row is None: + resolve(Failure(ErrorCode.NOT_FOUND)) + if claimed != 1: + resolve(Failure(ErrorCode.REQUEST_NOT_PENDING)) + return _REQUEST.validate_python(row, from_attributes=True) + + +async def _approve(store: CustomerHierarchyStore, request_id: str, actor: str) -> BudgetIncreaseRequest: + approved: Final = await _claim(store, request_id, actor, "approved", None) + member: Final = resolve(await store.get_member(approved.customer_member_id)) + resolve(await store.update_member(approved.customer_member_id, _grant(member, approved, store.now), actor)) + return approved + + +@router.get("/{request_id}", response_model=BudgetIncreaseRequest) +@management_endpoint_wrapper +async def get_budget_increase_request(request_id: str, actor: Admin, hierarchy: Hierarchy) -> BudgetIncreaseRequest: + """One request. Proxy admins only.""" + row: Final = await _in_transaction( + hierarchy, lambda store: _requests_table(store).find_unique(where={"request_id": request_id}) + ) + if row is None: + resolve(Failure(ErrorCode.NOT_FOUND)) + return _REQUEST.validate_python(row, from_attributes=True) + + +@router.put("/{request_id}/approve", response_model=BudgetIncreaseRequest) +@management_endpoint_wrapper +async def approve_budget_increase_request( + request_id: str, actor: Admin, hierarchy: Hierarchy, cache: Cache +) -> BudgetIncreaseRequest: + """Grant the increase: the member's override takes the requested cap until the request's expiry. Proxy admins only. + + The claim and the override land in one transaction, so a racing reject sees ``request_not_pending`` instead of + leaving a rejected request with the cap still granted.""" + approved: Final = await _in_transaction(hierarchy, lambda store: _approve(store, request_id, actor)) + await forget_customer_hierarchies((approved.customer_member_id,), cache) + return approved + + +@router.put("/{request_id}/reject", response_model=BudgetIncreaseRequest) +@management_endpoint_wrapper +async def reject_budget_increase_request( + request_id: str, actor: Admin, hierarchy: Hierarchy, body: BudgetIncreaseRejection | None = None +) -> BudgetIncreaseRequest: + """Decline the increase and leave the member paused, with optional review notes. Proxy admins only.""" + notes: Final = None if body is None else body.review_notes + return await _in_transaction(hierarchy, lambda store: _claim(store, request_id, actor, "rejected", notes)) diff --git a/litellm/proxy/management_endpoints/management_v1/customer_usage.py b/litellm/proxy/management_endpoints/management_v1/customer_usage.py new file mode 100644 index 00000000000..510b3a30be1 --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/customer_usage.py @@ -0,0 +1,310 @@ +from collections.abc import Sequence +from enum import Enum +from functools import partial +from typing import Annotated, Final, Protocol + +from fastapi import APIRouter, Depends, Query +from pydantic import AfterValidator, TypeAdapter + +from litellm.litellm_core_utils.core_helpers import require_attributes +from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params +from litellm.proxy.management_endpoints.common_daily_activity import ( + DailySpendRecord, + InvalidDateRange, + daily_activity_repository, + daily_activity_scope, + get_daily_activity, + get_daily_activity_aggregated, +) +from litellm.proxy.management_endpoints.common_daily_activity import raise_public as raise_activity_error +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_endpoints.management_v1.customers import ( + raise_public, + reject_duplicate_list_params, + require_customer_admin, +) +from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error +from litellm.proxy.management_helpers.customer_hierarchy import ErrorCode, Failure, PrismaRecord +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.proxy.utils import PrismaClient +from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.prisma_protocols import DatabaseClient, TableActions +from litellm.types.proxy.management_endpoints.common_daily_activity import SpendAnalyticsPaginatedResponse +from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail + + +class CustomerEntity(str, Enum): + ORGANIZATION = "customer_org_id" + TEAM = "customer_team_id" + MEMBER = "customer_member_id" + + +class CustomerMetadataDatabase(Protocol): + @property + def litellm_customerorganizationtable(self) -> TableActions[PrismaRecord]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[PrismaRecord]: ... + + @property + def litellm_customermembertable(self) -> TableActions[PrismaRecord]: ... + + +_TEXT: Final = TypeAdapter(str) +_OPTIONAL_TEXT: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def get_customer_usage_prisma() -> PrismaClient: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise_public(Failure(ErrorCode.DATABASE_UNAVAILABLE)) + return prisma_client + + +def metadata_table(prisma_client: PrismaClient, entity: CustomerEntity) -> TableActions[PrismaRecord]: + database: Final = require_attributes(prisma_client, DatabaseClient, "db").db + tables: Final = require_attributes( + database, + CustomerMetadataDatabase, + "litellm_customerorganizationtable", + "litellm_customerteamtable", + "litellm_customermembertable", + ) + if entity is CustomerEntity.ORGANIZATION: + return tables.litellm_customerorganizationtable + if entity is CustomerEntity.TEAM: + return tables.litellm_customerteamtable + return tables.litellm_customermembertable + + +async def page_aliases( + table: TableActions[PrismaRecord], entity: CustomerEntity, records: Sequence[DailySpendRecord] +) -> dict[str, dict[str, object]]: # mutable-ok: get_daily_activity requires dictionary-returning metadata callbacks + identifiers: Final = tuple(_OPTIONAL_TEXT.validate_python(getattr(row, entity.value, None)) for row in records) + return await customer_aliases(table, entity, tuple(value for value in identifiers if value)) + + +async def customer_aliases( + table: TableActions[PrismaRecord], entity: CustomerEntity, identifiers: Sequence[str] +) -> dict[str, dict[str, object]]: # mutable-ok: shared daily activity helper accepts dictionary-valued metadata + rows: Final = await find_many_in(table, entity.value, identifiers) + values: Final = tuple(row.model_dump() for row in rows) + return { + _TEXT.validate_python(row[entity.value]): { + "alias": _OPTIONAL_TEXT.validate_python(row.get("alias")) or _TEXT.validate_python(row[entity.value]) + } + for row in values + } + + +async def customer_activity( + prisma_client: PrismaClient, + entity: CustomerEntity, + entity_ids: str | None, + excluded_ids: str | None, + start_date: str | None, + end_date: str | None, + model: str | None, + api_key: str | None, + page: int, + page_size: int, +) -> SpendAnalyticsPaginatedResponse: + return await get_daily_activity( + prisma_client=prisma_client, + table_name="litellm_dailycustomermemberspend", + entity_id_field=entity.value, + entity_id=entity_ids.split(",") if entity_ids else None, + entity_metadata_field=None, + exclude_entity_ids=excluded_ids.split(",") if excluded_ids else None, + start_date=start_date, + end_date=end_date, + model=model, + api_key=api_key, + page=page, + page_size=page_size, + resolve_entity_metadata=partial(page_aliases, metadata_table(prisma_client, entity), entity), + ) + + +router: Final = APIRouter( + prefix=MANAGEMENT_V1_PREFIX, + tags=["customer usage"], + dependencies=( + Depends(require_customer_admin), + Depends(reject_unknown_query_params), + Depends(reject_duplicate_list_params), + ), +) + + +def bounded_organizations(value: str | None) -> str | None: + if value is None: + return None + ids: Final = tuple(part.strip() for part in value.split(",") if part.strip()) + if not ids or len(ids) > 100: + raise ValueError("Provide between 1 and 100 customer organization IDs") + return ",".join(ids) + + +@router.get( + "/customer_organizations/daily/activity/aggregated", + response_model=SpendAnalyticsPaginatedResponse, + description=( + "Get complete daily usage totals for up to 100 customer organizations in one response. " + "Uses shared SQL daily-spend aggregation with per-organization breakdowns. " + "Dates are inclusive UTC days, limited to 400 days. Model/API-key filters apply before aggregation. " + "Provide either customer_org_ids or customer_org_id for one exact ID, including commas or whitespace. " + "Key breakdowns show the top key; totals include every key. No raw-row pagination is required. " + "Budget counters are returned separately by organization reads. Proxy admins only." + ), +) +async def customer_organization_daily_activity_aggregated( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + start_date: Annotated[str, Query(description="First UTC calendar day, YYYY-MM-DD, inclusive")], + end_date: Annotated[ + str, Query(description="Last UTC calendar day, YYYY-MM-DD, inclusive, at most 400 days after start") + ], + customer_org_ids: Annotated[ + str | None, + Query(description="Comma-separated IDs for up to 100 visible customer organizations"), + AfterValidator(bounded_organizations), + ] = None, + customer_org_id: Annotated[ + str | None, Query(min_length=1, description="One exact customer organization ID, without splitting or trimming") + ] = None, + model: str | None = None, + api_key: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + date_error: Final = aggregated_date_range_error(start_date, end_date) + if date_error is not None: + raise_activity_error(InvalidDateRange(reason=date_error)) + if (customer_org_ids is None) == (customer_org_id is None): + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}invalid-query-parameter", + title="Invalid organization filter", + status=400, + detail="Provide exactly one of customer_org_ids or customer_org_id.", + ) + ) + ids: Final = [customer_org_id] if customer_org_id is not None else (customer_org_ids or "").split(",") + scope: Final = daily_activity_scope( + "litellm_dailycustomermemberspend", + CustomerEntity.ORGANIZATION.value, + ids, + None, + api_key, + start_date, + end_date, + model, + None, + ) + aliases: Final = await customer_aliases( + metadata_table(prisma_client, CustomerEntity.ORGANIZATION), CustomerEntity.ORGANIZATION, ids + ) + return await get_daily_activity_aggregated( + daily_activity_repository(prisma_client), + scope, + entity_metadata_field=aliases, + include_entity_breakdown=True, + api_key_limit=1, + ) + + +@router.get( + "/customer_organizations/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer organization activity with model, provider, key and organization breakdowns. " + "Filters accept comma-separated organization IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_organization_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_org_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_org_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.ORGANIZATION, + customer_org_ids, + exclude_customer_org_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) + + +@router.get( + "/customer_teams/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer team activity with model, provider, key and team breakdowns. " + "Filters accept comma-separated team IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_team_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_team_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_team_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.TEAM, + customer_team_ids, + exclude_customer_team_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) + + +@router.get( + "/customer_members/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer member activity with model, provider, key and member breakdowns. " + "Filters accept comma-separated member IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_member_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_member_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_member_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.MEMBER, + customer_member_ids, + exclude_customer_member_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) diff --git a/litellm/proxy/management_endpoints/management_v1/customers.py b/litellm/proxy/management_endpoints/management_v1/customers.py new file mode 100644 index 00000000000..a678a535385 --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/customers.py @@ -0,0 +1,592 @@ +from collections.abc import Mapping +from math import ceil +from types import MappingProxyType +from typing import Annotated, Final, NoReturn, TypeAlias, TypeVar + +from fastapi import APIRouter, Depends, Query, Request, Response + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.customer_hierarchy_auth import forget_customer_hierarchies, forget_customer_organization_aliases +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.list_api.common import ( + PROBLEM_TYPE_BASE, + ManagementProblem, + build_list_links, + reject_unknown_query_params, +) +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_helpers.customer_hierarchy import ( + CustomerHierarchy, + CustomerPortalLimits, + ErrorCode, + Failure, + Page, +) +from litellm.proxy.management_helpers.customer_upsert import CustomerUpsert +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + BudgetGroupsUpdate, + BudgetTemplate, + CustomerMember, + MemberCreate, + MemberPatch, + MemberUpsert, + MemberUpsertResponse, + Organization, + OrganizationCreate, + OrganizationPatch, + OrganizationUpsert, + OrganizationUpsertResponse, + Team, + TeamCreate, + TeamPatch, + TemplatePatch, +) +from litellm.types.proxy.management_endpoints.management_v1 import ListMeta, ListResponse, ProblemDetail + +T: Final = TypeVar("T") + + +_ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyType( + { + ErrorCode.CAPACITY_BUSY: ( + 503, + "Customer portal busy", + "Customer portal capacity is being updated. Retry the request.", + ), + ErrorCode.FREE_TIER_LIMIT: ( + 403, + "Free customer portal limit reached", + "Free customer portal limit reached: up to 5 customer organizations, 5 teams per organization and " + "5 members per team without a LiteLLM Enterprise license. Upgrade to Enterprise to remove the limits: " + "https://www.litellm.ai/enterprise", + ), + ErrorCode.NOT_FOUND: (404, "Not found", "Customer entity not found."), + ErrorCode.PARENT_NOT_FOUND: (404, "Parent not found", "Customer parent not found."), + ErrorCode.ALREADY_EXISTS: (409, "Already exists", "Customer entity ID already exists."), + ErrorCode.INVALID_DEFAULT_BUDGET: ( + 422, + "Invalid default budget", + "default_budget_template must name one of default_budgets, or default_budgets must hold one template.", + ), + ErrorCode.TEMPLATE_NOT_IN_ORG: ( + 422, + "Template not in organization", + "Budget template must belong to the member's organization.", + ), + ErrorCode.HAS_CHILDREN: (409, "Has children", "Delete the customer's children before deleting the parent."), + ErrorCode.INVALID_TEMPLATE_NAMES: ( + 422, + "Invalid template names", + "Budget template names must be unique within the organization.", + ), + ErrorCode.TEMPLATE_IN_USE: (409, "Template in use", "A default or referenced template cannot be deleted."), + ErrorCode.BUDGET_REQUIRED: (422, "Budget required", "A member requires a template or a budget override."), + ErrorCode.INCREASE_NOT_ALLOWED: ( + 422, + "Increase not allowed", + "The member's budget does not allow temporary increase requests.", + ), + ErrorCode.REQUEST_NOT_PENDING: ( + 409, + "Request not pending", + "Only a pending increase request can be approved or rejected.", + ), + ErrorCode.REQUEST_ALREADY_PENDING: ( + 409, + "Request already pending", + "The member already has an increase request waiting for review.", + ), + ErrorCode.EXPIRY_IN_PAST: (422, "Expiry in past", "A temporary increase must expire in the future."), + ErrorCode.KEY_NOT_IN_ORG: ( + 403, + "Key not in organization", + "The key may only act for members of its own customer organization.", + ), + ErrorCode.FORBIDDEN: (403, "Forbidden", "Only proxy admins can manage customer hierarchies."), + ErrorCode.DATABASE_UNAVAILABLE: (503, "Database not connected", "Database not connected."), + ErrorCode.CONFIGURATION_CONFLICT: ( + 409, + "Budget groups changed", + "Budget groups changed since you started editing. Reload before saving.", + ), + ErrorCode.ALIAS_ALREADY_EXISTS: ( + 409, + "Customer org alias already exists", + "Customer org alias is already used by another customer org.", + ), + ErrorCode.CUSTOMER_ORG_MISMATCH: ( + 409, + "Customer org mismatch", + "Customer member and customer team must belong to the specified customer org.", + ), + } +) +assert _ERROR_DETAILS.keys() == frozenset(ErrorCode), "every ErrorCode needs a public problem" + + +def error_details(code: ErrorCode) -> tuple[int, str, str]: + return _ERROR_DETAILS[code] + + +def raise_public(failure: Failure) -> NoReturn: + status, title, detail = error_details(failure.code) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}customer-hierarchy-{failure.code.value.replace('_', '-')}", + title=title, + status=status, + detail=detail, + ) + ) + + +def require_customer_admin(auth: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]) -> UserAPIKeyAuth: + if auth.user_role != LitellmUserRoles.PROXY_ADMIN: + raise_public(Failure(ErrorCode.FORBIDDEN)) + return auth + + +def customer_actor(auth: Annotated[UserAPIKeyAuth, Depends(require_customer_admin)]) -> str: + return auth.user_id or "proxy_admin" + + +def get_customer_hierarchy() -> CustomerHierarchy: + from litellm.proxy.proxy_server import premium_user, prisma_client + + if prisma_client is None: + raise_public(Failure(ErrorCode.DATABASE_UNAVAILABLE)) + return CustomerHierarchy(prisma_client, premium_user=premium_user) + + +def resolve(result: T | Failure) -> T: + if isinstance(result, Failure): + raise_public(result) + return result + + +def list_response(result: Page[T], request: Request, page: int, page_size: int) -> ListResponse[T]: + total_pages: Final = ceil(result.total_count / page_size) + return ListResponse[T].model_validate( + { + "data": result.data, + "meta": ListMeta(total_count=result.total_count, page=page, page_size=page_size, total_pages=total_pages), + "links": build_list_links(request, page, total_pages), + } + ) + + +def reject_duplicate_list_params(request: Request) -> None: + repeated: Final = tuple( + sorted(name for name in request.query_params if len(request.query_params.getlist(name)) > 1) + ) + if repeated: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}duplicate-query-parameter", + title="Duplicate query parameter", + status=400, + detail=f"Repeated query parameter(s): {', '.join(repeated)}. Each may appear once.", + ) + ) + + +router: Final = APIRouter( + prefix=MANAGEMENT_V1_PREFIX, + tags=["customer hierarchy"], + dependencies=(Depends(require_customer_admin), Depends(reject_unknown_query_params)), +) + + +def get_user_api_key_cache() -> UserApiKeyCache: + from litellm.proxy.proxy_server import user_api_key_cache + + return user_api_key_cache + + +AuthCache: TypeAlias = Annotated[UserApiKeyCache, Depends(get_user_api_key_cache)] + + +@router.get( + "/customer_portal/limits", + response_model=CustomerPortalLimits, + description="Get customer portal free-tier limits and current capacity. Proxy admins only.", +) +@management_endpoint_wrapper +async def get_customer_portal_limits( + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> CustomerPortalLimits: + return resolve(await hierarchy.portal_limits()) + + +@router.post( + "/customer_organizations", + status_code=201, + response_model=Organization, + description="Create a customer organization with resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def create_customer_organization( + data: OrganizationCreate, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> Organization: + return resolve(await hierarchy.create_organization(data, actor)) + + +@router.post( + "/customer_organizations/upsert", + response_model=OrganizationUpsertResponse, + description=( + "Create or update a customer org by customer_org_id, or by alias when no ID is supplied. " + "Returns 200 with created=true for creation and created=false for updates. Repeated identical calls are no-ops. " + "Supplied default_budgets are merged by template name, preserving existing template IDs and omitted templates. " + "A single supplied template becomes the default; multiple templates require default_budget_template to name one. " + "Omitted fields retain their values. Conflicting or ambiguous aliases return 409. Proxy admins only." + ), +) +@management_endpoint_wrapper +async def upsert_customer_organization( + data: OrganizationUpsert, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> OrganizationUpsertResponse: + outcome: Final = resolve(await CustomerUpsert(hierarchy).organization(data, actor)) + await forget_customer_hierarchies( + resolve(await hierarchy.member_ids_in_organization(outcome.data.customer_org_id)), cache + ) + await forget_customer_organization_aliases((outcome.previous_alias, outcome.data.alias), cache) + return outcome.data + + +@router.get( + "/customer_organizations/{customer_org_id}", + response_model=Organization, + description="Get a customer organization with resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def get_customer_organization( + customer_org_id: str, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> Organization: + return resolve(await hierarchy.get_organization(customer_org_id)) + + +@router.patch( + "/customer_organizations/{customer_org_id}", + response_model=Organization, + description="Update a customer organization alias or overall budget. Proxy admins only.", +) +@management_endpoint_wrapper +async def update_customer_organization( + customer_org_id: str, + data: OrganizationPatch, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> Organization: + previous: Final = resolve(await hierarchy.get_organization(customer_org_id)) + organization: Final = resolve(await hierarchy.update_organization(customer_org_id, data, actor)) + await forget_customer_hierarchies(resolve(await hierarchy.member_ids_in_organization(customer_org_id)), cache) + await forget_customer_organization_aliases((previous.alias, organization.alias), cache) + return organization + + +@router.delete( + "/customer_organizations/{customer_org_id}", + status_code=204, + description="Delete a customer organization and its owned budgets. Set cascade=true to also delete its customer teams and members; otherwise parents with children return 409. Proxy admins only.", +) +@management_endpoint_wrapper +async def delete_customer_organization( + customer_org_id: str, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, + cascade: Annotated[bool, Query(description="Also delete this customer organization's teams and members.")] = False, +) -> Response: + previous: Final = resolve(await hierarchy.get_organization(customer_org_id)) + resolve(await hierarchy.delete_organization(customer_org_id, cascade=cascade)) + await forget_customer_organization_aliases((previous.alias,), cache) + return Response(status_code=204) + + +@router.get( + "/customer_organizations", + response_model=ListResponse[Organization], + dependencies=(Depends(reject_duplicate_list_params),), + description="List customer organizations with page-number pagination and resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def list_customer_organizations( + request: Request, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + page: Annotated[int, Query(ge=1)] = 1, + page_size: Annotated[int, Query(ge=1)] = 50, + q: Annotated[str, Query(description="Case-insensitive substring of the customer ID or alias")] = "", +) -> ListResponse[Organization]: + size: Final = min(page_size, 100) + result: Final = resolve(await hierarchy.list_organizations(page, size, q.strip())) + return list_response(result, request, page, size) + + +@router.post( + "/customer_teams", + status_code=201, + response_model=Team, + description="Create a customer team with resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def create_customer_team( + data: TeamCreate, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> Team: + return resolve(await hierarchy.create_team(data, actor)) + + +@router.get( + "/customer_teams/{customer_team_id}", + response_model=Team, + description="Get a customer team with resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def get_customer_team( + customer_team_id: str, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> Team: + return resolve(await hierarchy.get_team(customer_team_id)) + + +@router.patch( + "/customer_teams/{customer_team_id}", + response_model=Team, + description="Update a customer team and its budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def update_customer_team( + customer_team_id: str, + data: TeamPatch, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> Team: + team: Final = resolve(await hierarchy.update_team(customer_team_id, data, actor)) + await forget_customer_hierarchies(resolve(await hierarchy.member_ids_in_team(customer_team_id)), cache) + return team + + +@router.delete( + "/customer_teams/{customer_team_id}", + status_code=204, + description="Delete a customer team and its owned budgets. Set cascade=true to also delete its customer members; otherwise parents with children return 409. Proxy admins only.", +) +@management_endpoint_wrapper +async def delete_customer_team( + customer_team_id: str, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cascade: Annotated[bool, Query(description="Also delete this customer team's members.")] = False, +) -> Response: + resolve(await hierarchy.delete_team(customer_team_id, cascade=cascade)) + return Response(status_code=204) + + +@router.get( + "/customer_teams", + response_model=ListResponse[Team], + dependencies=(Depends(reject_duplicate_list_params),), + description="List customer teams with page-number pagination and resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def list_customer_teams( + request: Request, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + page: Annotated[int, Query(ge=1)] = 1, + page_size: Annotated[int, Query(ge=1)] = 50, + parent_org_id: str | None = None, +) -> ListResponse[Team]: + size: Final = min(page_size, 100) + result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size)) + return list_response(result, request, page, size) + + +@router.post( + "/customer_members", + status_code=201, + response_model=CustomerMember, + description="Create a customer member with resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def create_customer_member( + data: MemberCreate, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> CustomerMember: + return resolve(await hierarchy.create_member(data, actor)) + + +@router.post( + "/customer_members/upsert", + response_model=MemberUpsertResponse, + description=( + "Create or update a customer member by customer_member_id within a customer org identified by " + "customer_org_id or customer_org_alias. If both are supplied they must match. " + "A new customer member with no parent_team_id joins the customer org's default customer team, created if necessary. An existing customer member keeps its customer teams, and a supplied parent_team_id becomes its first customer team. " + "Alias, budget_id, budget_override and budget_override_expires_at follow the existing customer member models. " + "Omitted update fields retain their values. Moving between customer teams in the same customer org is allowed. " + "Moving between customer orgs or selecting another customer org's customer team returns 409. " + "Returns 200 with created=true for creation and created=false for updates. Repeated identical calls are no-ops. " + "Proxy admins only." + ), +) +@management_endpoint_wrapper +async def upsert_customer_member( + data: MemberUpsert, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> MemberUpsertResponse: + member: Final = resolve(await CustomerUpsert(hierarchy).member(data, actor)) + await forget_customer_hierarchies((member.customer_member_id,), cache) + return member + + +@router.get( + "/customer_members/{customer_member_id}", + response_model=CustomerMember, + description="Get a customer member with resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def get_customer_member( + customer_member_id: str, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> CustomerMember: + return resolve(await hierarchy.get_member(customer_member_id)) + + +@router.patch( + "/customer_members/{customer_member_id}", + response_model=CustomerMember, + description="Update a customer member and its budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def update_customer_member( + customer_member_id: str, + data: MemberPatch, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> CustomerMember: + member: Final = resolve(await hierarchy.update_member(customer_member_id, data, actor)) + await forget_customer_hierarchies((customer_member_id,), cache) + return member + + +@router.delete( + "/customer_members/{customer_member_id}", + status_code=204, + description="Delete a customer member and its owned budgets. Parents with children return 409. Proxy admins only.", +) +@management_endpoint_wrapper +async def delete_customer_member( + customer_member_id: str, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> Response: + resolve(await hierarchy.delete_member(customer_member_id)) + await forget_customer_hierarchies((customer_member_id,), cache) + return Response(status_code=204) + + +@router.get( + "/customer_members", + response_model=ListResponse[CustomerMember], + dependencies=(Depends(reject_duplicate_list_params),), + description="List customer members with page-number pagination and resolved budgets. Proxy admins only.", +) +@management_endpoint_wrapper +async def list_customer_members( + request: Request, + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + page: Annotated[int, Query(ge=1)] = 1, + page_size: Annotated[int, Query(ge=1)] = 50, + parent_team_id: str | None = None, + parent_org_id: str | None = None, +) -> ListResponse[CustomerMember]: + size: Final = min(page_size, 100) + result: Final = resolve(await hierarchy.list_members(parent_team_id, page, size, parent_org_id=parent_org_id)) + return list_response(result, request, page, size) + + +@router.post( + "/customer_organizations/{customer_org_id}/default_budgets", + status_code=201, + response_model=Organization, + description="Add a named budget template to a customer organization. Proxy admins only.", +) +@management_endpoint_wrapper +async def add_customer_budget_template( + customer_org_id: str, + data: BudgetTemplate, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> Organization: + return resolve(await hierarchy.add_template(customer_org_id, data, actor)) + + +@router.patch( + "/customer_organizations/{customer_org_id}/default_budgets", + response_model=Organization, + description="Atomically configure budget groups and their default. Requires the current budget_groups_revision. " + "Existing groups retain omitted settings and window reset dates. Stale revisions and referenced deletions " + "return 409 without applying changes. Proxy admins only.", +) +@management_endpoint_wrapper +async def configure_customer_budget_groups( + customer_org_id: str, + data: BudgetGroupsUpdate, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> Organization: + organization: Final = resolve(await hierarchy.configure_templates(customer_org_id, data, actor)) + await forget_customer_hierarchies(resolve(await hierarchy.member_ids_in_organization(customer_org_id)), cache) + return organization + + +@router.patch( + "/customer_organizations/{customer_org_id}/default_budgets/{budget_id}", + response_model=Organization, + description="Edit a budget template in place. Members inherit the updated values. Proxy admins only.", +) +@management_endpoint_wrapper +async def update_customer_budget_template( + customer_org_id: str, + budget_id: str, + data: TemplatePatch, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], + cache: AuthCache, +) -> Organization: + organization: Final = resolve(await hierarchy.update_template(customer_org_id, budget_id, data, actor)) + await forget_customer_hierarchies(resolve(await hierarchy.member_ids_using_template(budget_id)), cache) + return organization + + +@router.delete( + "/customer_organizations/{customer_org_id}/default_budgets/{budget_id}", + status_code=204, + description="Delete an unreferenced, non-default customer budget template. Proxy admins only.", +) +@management_endpoint_wrapper +async def delete_customer_budget_template( + customer_org_id: str, + budget_id: str, + actor: Annotated[str, Depends(customer_actor)], + hierarchy: Annotated[CustomerHierarchy, Depends(get_customer_hierarchy)], +) -> Response: + resolve(await hierarchy.delete_template(customer_org_id, budget_id, actor)) + return Response(status_code=204) diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 853f29ecb62..fb9851c3d15 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -507,6 +507,9 @@ async def new_organization( except Exception: pass + from litellm.proxy.management_helpers.utils import validate_scalar_budget_attachment + + await validate_scalar_budget_attachment(prisma_client, data.budget_id) if data.budget_id is None: """ Every organization needs a budget attached. diff --git a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py index 7cd36d51511..1effbbd8e0c 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py @@ -28,6 +28,7 @@ from litellm.constants import ( MAX_COMPETITOR_NAMES, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import subclass_overrides_hook from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, @@ -181,7 +182,7 @@ async def apply_policies( continue if not isinstance(callback, CustomGuardrail): continue - if "apply_guardrail" not in type(callback).__dict__: + if not subclass_overrides_hook(type(callback), "apply_guardrail", CustomGuardrail): verbose_proxy_logger.debug( "apply_policies: guardrail '%s' has no apply_guardrail, skipping", guardrail_name, diff --git a/litellm/proxy/management_endpoints/team_admin_field_permissions.py b/litellm/proxy/management_endpoints/team_admin_field_permissions.py index 08a84ca91c3..f80aa2a2e29 100644 --- a/litellm/proxy/management_endpoints/team_admin_field_permissions.py +++ b/litellm/proxy/management_endpoints/team_admin_field_permissions.py @@ -108,6 +108,7 @@ PROXY_CONTROLLED_TEAM_METADATA_KEYS: Final[frozenset[str]] = frozenset( "phoenix_project_name_override", "phoenix_project_name", "opik", + "team_model_limit_max_borrow_multiple", ) ) _NOT_COLUMNS: Final[frozenset[str]] = frozenset({"team_id", "metadata"}) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f3d3c01b3a4..6da15cca9ad 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -621,7 +621,7 @@ async def refresh_cached_team( ) batches: Final = active_request_redis_batches() if batches is not None: - await batches.flush_all() + await batches.flush_pre_call() await publish_auth_cache_invalidation(cache_key=f"team_id:{team_row.team_id}") if team_table.team_alias is not None: await publish_auth_cache_invalidation(cache_key=f"team_alias:{team_table.team_alias}") @@ -5403,6 +5403,9 @@ async def _resolve_team_access_group_resources( ), "access_group_agent_ids": list({a for group in resolved_groups for a in (group.access_agent_ids or [])}), "access_group_skill_names": _effective_access_group_skill_names(resolved_groups, unresolved_ids), + "access_group_vector_store_ids": tuple( + dict.fromkeys(chain.from_iterable(group.access_vector_store_ids or () for group in resolved_groups)) + ), "unresolved_access_group_ids": unresolved_ids, "access_group_details": tuple( TeamAccessGroupModelGrant( @@ -5412,6 +5415,7 @@ async def _resolve_team_access_group_resources( mcp_server_ids=tuple(group.access_mcp_server_ids or ()), agent_ids=tuple(group.access_agent_ids or ()), skill_names=tuple(group.access_skill_names or ()), + vector_store_ids=tuple(group.access_vector_store_ids or ()), ) for group in resolved_groups ), diff --git a/litellm/proxy/management_helpers/customer_budget_usage.py b/litellm/proxy/management_helpers/customer_budget_usage.py new file mode 100644 index 00000000000..2802149f18c --- /dev/null +++ b/litellm/proxy/management_helpers/customer_budget_usage.py @@ -0,0 +1,103 @@ +from collections.abc import Mapping, Sequence +from datetime import datetime +from itertools import chain +from typing import TYPE_CHECKING, Final, TypeVar + +from litellm.proxy.common_utils.timezone_utils import BudgetResetSettings +from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.prisma_protocols import TableActions +from litellm.types.proxy.auth.customer_hierarchy import CustomerScopeKey +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + Budget, + CustomerBudgetUsage, + CustomerMember, + Organization, + Team, +) + +if TYPE_CHECKING: + from prisma.models import LiteLLM_BudgetWindowSpend + + from litellm.proxy.spend_tracking.budget_reservation import BudgetCounter + +Entity: Final = TypeVar("Entity", Organization, Team, CustomerMember) + + +def _identity(entity: Organization | Team | CustomerMember) -> tuple[CustomerScopeKey, str, Budget | None]: + if isinstance(entity, Organization): + return "customer_org", entity.customer_org_id, entity.budget + if isinstance(entity, Team): + return "customer_team", entity.customer_team_id, entity.effective_budget + return "customer_member", entity.customer_member_id, entity.effective_budget + + +def _scalar_usage( + entity: Organization | Team | CustomerMember, budget: Budget | None +) -> tuple[CustomerBudgetUsage, ...]: + if budget is None or budget.max_budget is None: + return () + return ( + CustomerBudgetUsage( + source="scalar", + budget_duration=budget.budget_duration, + max_budget=budget.max_budget, + spend=entity.spend, + reset_at=entity.budget_reset_at, + ), + ) + + +def _usage( + counters: Sequence["BudgetCounter"], + rows: Mapping[tuple[str, str, str], "LiteLLM_BudgetWindowSpend"], +) -> tuple[CustomerBudgetUsage, ...]: + from litellm.proxy.db.spend_counter_reseed import recorded_window_spend + + return tuple( + CustomerBudgetUsage( + source="window", + budget_duration=counter.window_duration, + max_budget=counter.max_budget, + spend=recorded_window_spend( + rows.get((counter.entity_type, counter.entity_id, counter.window_duration or "")), + counter.entity_type, + counter.window_start, + ), + window_start=counter.window_start, + reset_at=counter.reset_at, + ) + for counter in counters + ) + + +async def customer_budget_usage( + entities: Sequence[Entity], + table: TableActions["LiteLLM_BudgetWindowSpend"], + now: datetime, + settings: BudgetResetSettings, +) -> tuple[Entity, ...]: + from litellm.proxy.spend_tracking.budget_reservation import customer_window_counters + + if not entities: + return () + identities: Final = tuple(_identity(entity) for entity in entities) + counters: Final = tuple( + customer_window_counters(scope, entity_id, budget.budget_limits or (), now=now, reset_settings=settings) + if budget is not None + else () + for scope, entity_id, budget in identities + ) + windows: Final = tuple(chain.from_iterable(counters)) + rows: Final = await find_many_in( + table, + "entity_id", + tuple(counter.entity_id for counter in windows), + where={"entity_type": identities[0][0]}, + ) + by_window: Final = {(row.entity_type, row.entity_id, row.window_duration): row for row in rows} + return tuple( + entity.model_copy( + update={"budget_usage": _scalar_usage(entity, identity[2]) + _usage(entity_windows, by_window)} + ) + for entity, identity, entity_windows in zip(entities, identities, counters, strict=True) + ) diff --git a/litellm/proxy/management_helpers/customer_hierarchy.py b/litellm/proxy/management_helpers/customer_hierarchy.py new file mode 100644 index 00000000000..6d0b857a0e1 --- /dev/null +++ b/litellm/proxy/management_helpers/customer_hierarchy.py @@ -0,0 +1,1313 @@ +from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence +from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from itertools import chain +from typing import TYPE_CHECKING, Final, Generic, Protocol, TypeVar +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator + +from litellm.models.budget import BudgetLimit, BudgetLimits, LiteLLM_BudgetTable, initialize_budget_limits +from litellm.proxy.common_utils.timezone_utils import ( + BudgetResetSettings, + get_budget_reset_settings, + get_budget_reset_time, +) +from litellm.proxy.db.db_span import db_span +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.list_api.common import escape_like +from litellm.proxy.management_helpers.customer_budget_usage import customer_budget_usage +from litellm.repositories.budget_repository import create_budget_row, update_budget_row +from litellm.repositories.chunked_in import count_in, delete_many_in, find_many_in +from litellm.repositories.prisma_protocols import TableActions +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + Budget, + BudgetGroupChange, + BudgetGroupsUpdate, + BudgetTemplate, + CustomerMember, + MemberCreate, + MemberPatch, + Organization, + OrganizationBudget, + OrganizationCreate, + OrganizationPatch, + Team, + TeamCreate, + TeamPatch, + TemplatePatch, +) + +if TYPE_CHECKING: + from prisma import Json, Prisma + from prisma.models import LiteLLM_BudgetWindowSpend + + +class ErrorCode(str, Enum): + FREE_TIER_LIMIT = "free_tier_limit" + CAPACITY_BUSY = "capacity_busy" + NOT_FOUND = "not_found" + PARENT_NOT_FOUND = "parent_not_found" + ALREADY_EXISTS = "already_exists" + INVALID_DEFAULT_BUDGET = "invalid_default_budget" + TEMPLATE_NOT_IN_ORG = "template_not_in_org" + HAS_CHILDREN = "has_children" + FORBIDDEN = "forbidden" + DATABASE_UNAVAILABLE = "database_unavailable" + INVALID_TEMPLATE_NAMES = "invalid_template_names" + TEMPLATE_IN_USE = "template_in_use" + BUDGET_REQUIRED = "budget_required" + ALIAS_ALREADY_EXISTS = "alias_already_exists" + CUSTOMER_ORG_MISMATCH = "customer_org_mismatch" + INCREASE_NOT_ALLOWED = "increase_not_allowed" + REQUEST_NOT_PENDING = "request_not_pending" + REQUEST_ALREADY_PENDING = "request_already_pending" + EXPIRY_IN_PAST = "expiry_in_past" + KEY_NOT_IN_ORG = "key_not_in_org" + CONFIGURATION_CONFLICT = "configuration_conflict" + + +@dataclass(frozen=True, slots=True) +class Failure: + code: ErrorCode + + +FREE_ORGANIZATION_LIMIT: Final = 5 +FREE_TEAMS_PER_ORGANIZATION_LIMIT: Final = 5 +FREE_MEMBERS_PER_TEAM_LIMIT: Final = 5 +FREE_CAPACITY_ADVISORY_LOCK: Final = 714023581 +FREE_CAPACITY_LOCK_TIMEOUT_MS: Final = 1000 + + +class CustomerPortalLimits(BaseModel): + model_config = ConfigDict(frozen=True) + + limits_apply: bool + organization_limit: int = FREE_ORGANIZATION_LIMIT + teams_per_organization_limit: int = FREE_TEAMS_PER_ORGANIZATION_LIMIT + members_per_team_limit: int = FREE_MEMBERS_PER_TEAM_LIMIT + organization_count: int + any_limit_reached: bool + + +class _CapacityCounts(BaseModel): + model_config = ConfigDict(frozen=True) + + full_organizations: int + full_teams: int + + +_LOCK_ORGANIZATION_SQL: Final = ( + 'SELECT 1 AS locked FROM "LiteLLM_CustomerOrganizationTable" WHERE customer_org_id = $1 FOR UPDATE' +) +_LOCK_TEMPLATES_SQL: Final = ( + 'SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_id IN ' + '(SELECT unnest(budget_template_ids) FROM "LiteLLM_CustomerOrganizationTable" WHERE customer_org_id = $1) ' + "ORDER BY budget_id FOR UPDATE" +) + +_LOCK_CUSTOMER_IDENTITIES_SQL: Final = ( + "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtextextended('litellm:customer-identities', 0))" +) + + +class TransactionOperation(str, Enum): + READ_WRITE = "read_write" + DELETE = "delete" + + +class PrismaRecord(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +class TransactionSource(Protocol): + def tx(self) -> AbstractAsyncContextManager["Prisma"]: ... + + +T: Final = TypeVar("T") + + +@dataclass(frozen=True, slots=True) +class Page(Generic[T]): + data: tuple[T, ...] + total_count: int + + +class BudgetRecord(Budget): + model_config = ConfigDict(frozen=True, extra="ignore") + + temp_budget_increase: float | None = Field(default=None, exclude=True) + temp_budget_expiry: datetime | None = Field(default=None, exclude=True) + + +class BudgetWindow(BaseModel): + """The recurring window a budget row carries; customer rows read theirs from the budget they point at.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + budget_duration: str | None = None + budget_reset_at: datetime | None = None + + +class OrganizationMemberCount(BaseModel): + model_config = ConfigDict(frozen=True) + + customer_org_id: str + member_count: int + + +_VALUES: Final = TypeAdapter(Mapping[str, object]) +_MEMBER_COUNTS: Final = TypeAdapter(tuple[OrganizationMemberCount, ...]) +_BUDGET: Final[TypeAdapter[BudgetRecord | None]] = TypeAdapter(BudgetRecord | None) +_EXPIRY: Final[TypeAdapter[datetime | None]] = TypeAdapter(datetime | None) +_WINDOW: Final[TypeAdapter[BudgetWindow | None]] = TypeAdapter(BudgetWindow | None) + + +def window_reset_at(value: object) -> datetime | None: + window: Final = _WINDOW.validate_python(value) + return None if window is None else window.budget_reset_at + + +class _StoredWindow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + budget_duration: str | None = None + budget_reset_at: datetime | None = None + + +def budget_window_values( + budget: Budget, patch: Collection[str], current: _StoredWindow | None = None +) -> Mapping[str, object]: + """``budget_reset_at`` follows ``budget_duration``: stamped when a duration is written, cleared when it is removed. + Resubmitting the duration a row already runs keeps its clock, or an edit landing after the window came due + would push the reset out another window without the job ever clearing the spend.""" + if "budget_duration" not in patch: + return {} + duration: Final = budget.budget_duration + if current is not None and current.budget_duration == duration and current.budget_reset_at is not None: + return {} + return {"budget_reset_at": get_budget_reset_time(duration) if duration is not None else None} + + +def budget_values(budget: Budget) -> Mapping[str, object]: + return _VALUES.validate_python(budget.model_dump(include=set(Budget.model_fields))) + + +def group_budget_patch(group: BudgetGroupChange, current: OrganizationBudget | None) -> Budget: + patch: Final = Budget.model_validate(group.model_dump(exclude_unset=True, exclude={"budget_id", "name"})) + if patch.budget_limits is None: + return patch + resets: Final = ( + {limit.budget_duration: limit.reset_at for limit in (current.budget_limits or ())} if current else {} + ) + return patch.model_copy( + update={ + "budget_limits": tuple( + limit.model_copy(update={"reset_at": resets.get(limit.budget_duration, limit.reset_at)}) + for limit in patch.budget_limits + ) + } + ) + + +_JSON_BUDGET_COLUMNS: Final = frozenset({"model_tpm_limit", "model_rpm_limit"}) + + +_MODEL_LIMIT_MAP: Final = TypeAdapter(dict[str, int]) +_BUDGET_LIMITS: Final[TypeAdapter[tuple[BudgetLimit, ...]]] = TypeAdapter(BudgetLimits) + + +def budget_limit_json(value: object) -> "Json": + from prisma import Json + + limits: Final = _BUDGET_LIMITS.validate_python(value) + return Json( + [ + { + "budget_duration": limit.budget_duration, + "max_budget": limit.max_budget, + "reset_at": limit.reset_at.isoformat() if limit.reset_at is not None else None, + } + for limit in limits + ] + ) + + +def _budget_json_column(key: str, value: object) -> object: + from prisma import Json + + if key == "budget_limits": + return budget_limit_json(value) if value is not None else Json(None) + if key in _JSON_BUDGET_COLUMNS: + return Json(_MODEL_LIMIT_MAP.validate_python(value)) + return value + + +def budget_row_values(values: Mapping[str, object]) -> Mapping[str, object]: + """Prisma writes Json columns only through ``Json``: a budget without windows carries ``Json(None)``, the same + JSON null the key path writes, since a bare ``None`` on a ``Json?`` column is rejected by the query engine.""" + return { + key: _budget_json_column(key, value) + for key, value in values.items() + if not (key in _JSON_BUDGET_COLUMNS and value is None) + } + + +def related_budget(value: object, now: datetime | None = None, inherited_maximum: float | None = None) -> Budget | None: + record: Final = _BUDGET.validate_python(value) + if record is None: + return None + values: Final = budget_values(record) + if now is None: + return Budget.model_validate(values) + budget: Final = LiteLLM_BudgetTable( + max_budget=record.max_budget, + temp_budget_increase=record.temp_budget_increase, + temp_budget_expiry=record.temp_budget_expiry, + ) + maximum: Final = budget.effective_max_budget(utc_datetime(now)) + if maximum is not None: + return Budget.model_validate({**values, "max_budget": maximum}) + if inherited_maximum is None: + return Budget.model_validate(values) + return Budget.model_validate( + { + **values, + "max_budget": inherited_maximum + budget.active_temp_budget_increase(utc_datetime(now)), + } + ) + + +class OrganizationRecord(Organization): + budget_template_ids: tuple[str, ...] = () + + @model_validator(mode="after") + def valid_ownership(self) -> "OrganizationRecord": + if ( + self.default_budget_template_id is not None + and self.default_budget_template_id not in self.budget_template_ids + ): + raise ValueError("Default budget template must belong to the organization") + return self + + +class StoredBudget(BudgetRecord): + budget_id: str + name: str | None = None + + +class _MemberDeletionRow(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + customer_member_id: str + budget_override_id: str | None = None + + +class _TeamDeletionRow(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + customer_team_id: str + budget_id: str | None = None + budget_override_id: str | None = None + + +_DELETE_BATCH_SIZE: Final = 100 +_BUDGET_REFERENCE_FIELDS: Final = ( + "customer_organization_defaults", + "customer_organizations", + "customer_member_overrides", + "customer_teams", + "customer_team_overrides", + "customer_members", + "organization", + "projects", + "keys", + "end_users", + "tags", + "model_access_groups", + "team_membership", + "organization_membership", +) + + +def organization_response( + row: OrganizationRecord, budgets: Mapping[str, StoredBudget], member_count: int +) -> Organization: + return Organization.model_validate( + { + **row.model_dump(exclude={"budget_template_ids"}), + "member_count": member_count, + "default_budgets": tuple( + OrganizationBudget.model_validate( + { + **budgets[budget_id].model_dump(), + "name": budgets[budget_id].name or budget_id, + } + ) + for budget_id in row.budget_template_ids + ), + } + ) + + +def team_response(row: PrismaRecord, now: datetime) -> Team: + values: Final = row.model_dump() + budget: Final = related_budget(values.get("budget"), now) + override: Final = related_budget( + values.get("budget_override"), now, budget.max_budget if budget is not None else None + ) + expiry: Final = _EXPIRY.validate_python(values.get("budget_override_expires_at")) + return Team.model_validate( + { + **values, + "budget": related_budget(values.get("budget")), + "budget_override": related_budget(values.get("budget_override")), + "budget_reset_at": member_window_reset_at(values.get("budget"), values.get("budget_override"), expiry, now), + "effective_budget": effective_member_budget(budget, override, expiry, now), + } + ) + + +def override_expiry( + row: Team | CustomerMember | None, payload: TeamPatch | MemberPatch, override_id: str | None +) -> datetime | None: + if override_id is None: + return None + if "budget_override_expires_at" in payload.model_fields_set: + return payload.budget_override_expires_at + return row.budget_override_expires_at if row is not None and row.budget_override_id is not None else None + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def utc_datetime(value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + + +def effective_member_budget( + template: Budget | None, override: Budget | None, expires_at: datetime | None, now: datetime +) -> Budget: + inherited: Final = template if template is not None else Budget(allowed_models=()) + if override is None: + return inherited + if expires_at is not None and utc_datetime(expires_at) <= utc_datetime(now): + return inherited + values: Final = _VALUES.validate_python( + override.model_dump(exclude_none=True, exclude={"allowed_models", "budget_limits"}) + ) + models: Final[Mapping[str, object]] = {"allowed_models": override.allowed_models} if override.allowed_models else {} + inherited_windows: Final = () if override.max_budget is not None else inherited.budget_limits or () + windows: Final = { + **{limit.budget_duration: limit for limit in inherited_windows}, + **{limit.budget_duration: limit for limit in override.budget_limits or ()}, + } + return Budget.model_validate( + {**budget_values(inherited), **values, **models, "budget_limits": tuple(windows.values()) or None} + ) + + +def member_window_reset_at( + template: object, override: object, expires_at: datetime | None, now: datetime +) -> datetime | None: + """An unexpired override with its own duration runs the member's window; otherwise the template does.""" + active: Final = _WINDOW.validate_python(override) + unexpired: Final = expires_at is None or utc_datetime(expires_at) > utc_datetime(now) + if active is not None and active.budget_duration is not None and unexpired: + return active.budget_reset_at + return window_reset_at(template) + + +class CustomerRowWindow(BaseModel): + """The budget relations a customer row carries, read off a Prisma row or a dumped mapping.""" + + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + budget: BudgetWindow | None = None + budget_override: BudgetWindow | None = None + budget_override_expires_at: datetime | None = None + + +def customer_row_reset_at(row: object, now: datetime) -> datetime | None: + """When the row's current spend window resets: its budget row's clock, or an unexpired override's own.""" + window: Final = CustomerRowWindow.model_validate(row, from_attributes=True) + return member_window_reset_at(window.budget, window.budget_override, window.budget_override_expires_at, now) + + +def member_response(row: PrismaRecord, now: datetime) -> CustomerMember: + values: Final = row.model_dump() + override: Final = related_budget(values.get("budget_override")) + template: Final = related_budget(values.get("budget"), now) + effective_override: Final = related_budget( + values.get("budget_override"), now, template.max_budget if template is not None else None + ) + expiry: Final = _EXPIRY.validate_python(values.get("budget_override_expires_at")) + return CustomerMember.model_validate( + { + **values, + "budget_override": override, + "budget_reset_at": member_window_reset_at(values.get("budget"), values.get("budget_override"), expiry, now), + "effective_budget": effective_member_budget(template, effective_override, expiry, now), + } + ) + + +def default_template_index(templates: tuple[BudgetTemplate, ...], default_name: str | None) -> int | Failure | None: + if default_name is not None: + named: Final = tuple(index for index, template in enumerate(templates) if template.name == default_name) + return named[0] if named else Failure(ErrorCode.INVALID_DEFAULT_BUDGET) + if not templates: + return None + if len(templates) == 1: + return 0 + return Failure(ErrorCode.INVALID_DEFAULT_BUDGET) + + +def alias_patch(payload: OrganizationPatch | TeamPatch | MemberPatch) -> Mapping[str, object]: + return {"alias": payload.alias} if "alias" in payload.model_fields_set else {} + + +def default_template_patch( + organization: Organization, payload: OrganizationPatch +) -> Mapping[str, str | None] | Failure: + if "default_budget_template_id" not in payload.model_fields_set: + return {} + selected: Final = payload.default_budget_template_id + if selected is not None and all(template.budget_id != selected for template in organization.default_budgets): + return Failure(ErrorCode.TEMPLATE_NOT_IN_ORG) + return {"default_budget_template_id": selected} + + +_BUDGET_INCLUDE: Final = {"budget": True} +_MEMBER_INCLUDE: Final = {"budget": True, "budget_override": True} +_ID: Final = TypeAdapter(str) + + +def _member_ids(rows: Sequence[PrismaRecord]) -> tuple[str, ...]: + return tuple(_ID.validate_python(row.model_dump()["customer_member_id"]) for row in rows) + + +class CustomerHierarchyStore: + def __init__( + self, db: "Prisma", now: datetime, settings: BudgetResetSettings | None = None, *, premium_user: bool + ) -> None: + self.db = db + self.premium_user = premium_user + self.now = now + self.settings = settings if settings is not None else get_budget_reset_settings() + self.organizations: TableActions[PrismaRecord] = db.litellm_customerorganizationtable # pyright: ignore[reportAttributeAccessIssue] # protocol accepts immutable query mappings + self.teams: TableActions[PrismaRecord] = db.litellm_customerteamtable # pyright: ignore[reportAttributeAccessIssue] # protocol accepts immutable query mappings + self.members: TableActions[PrismaRecord] = db.litellm_customermembertable # pyright: ignore[reportAttributeAccessIssue] # protocol accepts immutable query mappings + self.budgets: TableActions[PrismaRecord] = db.litellm_budgettable # pyright: ignore[reportAttributeAccessIssue] # protocol accepts immutable query mappings + self.window_spend: TableActions[LiteLLM_BudgetWindowSpend] = db.litellm_budgetwindowspend # pyright: ignore[reportAttributeAccessIssue] # protocol accepts immutable query mappings + + async def lock_free_capacity(self, scope: str | None = None) -> None: + await self.db.query_raw("SELECT set_config('lock_timeout', $1, true)", f"{FREE_CAPACITY_LOCK_TIMEOUT_MS}ms") + if scope is None: + await self.db.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock($1)", FREE_CAPACITY_ADVISORY_LOCK) + return + await self.db.query_raw( + "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtextextended($1, $2))", + scope, + FREE_CAPACITY_ADVISORY_LOCK, + ) + + async def member_capacity_reached(self, team_ids: Sequence[str]) -> bool: + if self.premium_user or not team_ids: + return False + for team_id in sorted(frozenset(team_ids)): + await self.lock_free_capacity(f"members:{team_id}") + if await self.members.count(where={"parent_team_ids": {"has": team_id}}) >= FREE_MEMBERS_PER_TEAM_LIMIT: + return True + return False + + async def portal_limits(self) -> CustomerPortalLimits: + organization_count: Final = await self.organizations.count() + counts: Final = TypeAdapter(tuple[_CapacityCounts, ...]).validate_python( + await self.db.query_raw( + 'SELECT (SELECT COUNT(*) FROM (SELECT parent_org_id FROM "LiteLLM_CustomerTeamTable" ' + "GROUP BY parent_org_id HAVING COUNT(*) >= $1) AS full_orgs) AS full_organizations, " + '(SELECT COUNT(*) FROM (SELECT team_id FROM "LiteLLM_CustomerMemberTable", ' + "unnest(parent_team_ids) AS team_id GROUP BY team_id HAVING COUNT(*) >= $2) AS full_members) AS full_teams", + FREE_TEAMS_PER_ORGANIZATION_LIMIT, + FREE_MEMBERS_PER_TEAM_LIMIT, + ) + ) + return CustomerPortalLimits( + limits_apply=not self.premium_user, + organization_count=organization_count, + any_limit_reached=not self.premium_user + and ( + organization_count >= FREE_ORGANIZATION_LIMIT + or counts[0].full_organizations > 0 + or counts[0].full_teams > 0 + ), + ) + + async def organization_responses(self, rows: Sequence[PrismaRecord]) -> tuple[Organization, ...]: + records: Final = tuple( + OrganizationRecord.model_validate( + { + **row.model_dump(), + "budget": related_budget(row.model_dump().get("budget"), self.now), + "budget_reset_at": window_reset_at(row.model_dump().get("budget")), + } + ) + for row in rows + ) + budget_ids: Final = tuple(chain.from_iterable(record.budget_template_ids for record in records)) + budget_rows: Final = await find_many_in(self.budgets, "budget_id", budget_ids) + resolved: Final = tuple(StoredBudget.model_validate(row.model_dump()) for row in budget_rows) + budgets: Final = {budget.budget_id: budget for budget in resolved} + + counts: Final = await self.organization_member_counts(tuple(record.customer_org_id for record in records)) + responses: Final = tuple( + organization_response(record, budgets, counts.get(record.customer_org_id, 0)) for record in records + ) + return await customer_budget_usage(responses, self.window_spend, self.now, self.settings) + + async def organization_member_counts(self, organization_ids: tuple[str, ...]) -> Mapping[str, int]: + if not organization_ids: + return {} + async with db_span("find_many", "LiteLLM_CustomerMemberTable"): + rows: Final = _MEMBER_COUNTS.validate_python( + await self.db.query_raw( + "SELECT parent_org_id AS customer_org_id, COUNT(customer_member_id) AS member_count " + 'FROM "LiteLLM_CustomerMemberTable" ' + "WHERE parent_org_id = ANY($1::text[]) GROUP BY parent_org_id", + list(organization_ids), + ) + ) + return {row.customer_org_id: row.member_count for row in rows} + + async def get_organization(self, organization_id: str) -> Organization | Failure: + row: Final = await self.organizations.find_unique( + where={"customer_org_id": organization_id}, include=_BUDGET_INCLUDE + ) + if row is None: + return Failure(ErrorCode.NOT_FOUND) + organizations: Final = await self.organization_responses((row,)) + return organizations[0] + + async def get_team(self, team_id: str) -> Team | Failure: + row: Final = await self.teams.find_unique(where={"customer_team_id": team_id}, include=_MEMBER_INCLUDE) + if row is None: + return Failure(ErrorCode.NOT_FOUND) + responses: Final = await customer_budget_usage( + (team_response(row, self.now),), self.window_spend, self.now, self.settings + ) + return responses[0] + + async def get_member(self, member_id: str) -> CustomerMember | Failure: + row: Final = await self.members.find_unique(where={"customer_member_id": member_id}, include=_MEMBER_INCLUDE) + if row is None: + return Failure(ErrorCode.NOT_FOUND) + responses: Final = await customer_budget_usage( + (member_response(row, self.now),), self.window_spend, self.now, self.settings + ) + return responses[0] + + async def create_budget(self, budget: Budget, actor: str, name: str | None = None) -> str: + budget_id: Final = str(uuid4()) + await create_budget_row( + self.budgets, + budget_row_values( + { + **budget_values(budget), + **budget_window_values(budget, budget.model_fields_set), + **( + {"budget_limits": initialize_budget_limits(budget.budget_limits, self.settings, self.now)} + if budget.budget_limits is not None + else {} + ), + "allowed_models": budget.allowed_models or (), + "budget_id": budget_id, + "name": name, + } + ), + actor, + ) + return budget_id + + async def update_budget(self, budget_id: str | None, budget: Budget, actor: str) -> str: + if budget_id is None: + return await self.create_budget(budget, actor) + if budget.budget_limits: + from litellm.proxy.management_helpers.utils import validate_budget_windows_update + + await validate_budget_windows_update(self.db, budget_id) + patch: Final = _VALUES.validate_python(budget.model_dump(exclude_unset=True)) + stored: Final = await self.budgets.find_unique(where={"budget_id": budget_id}) + current: Final = _StoredWindow.model_validate(stored, from_attributes=True) if stored is not None else None + await update_budget_row( + self.budgets, + budget_id, + budget_row_values( + { + **patch, + **budget_window_values(budget, patch, current), + **( + {"budget_limits": initialize_budget_limits(budget.budget_limits, self.settings, self.now)} + if budget.budget_limits is not None + else {} + ), + **({"allowed_models": budget.allowed_models or ()} if "allowed_models" in patch else {}), + } + ), + actor, + ) + return budget_id + + async def write_template_metadata( + self, organization_id: str, template_ids: Sequence[str], default_id: str | None, actor: str + ) -> None: + await self.organizations.update( + where={"customer_org_id": organization_id}, + data={ + "budget_template_ids": tuple(template_ids), + "default_budget_template_id": default_id, + "updated_by": actor, + }, + ) + + async def lock_organization(self, organization_id: str) -> None: + await self.db.query_raw(_LOCK_ORGANIZATION_SQL, organization_id) + + async def lock_customer_identities(self) -> None: + await self.db.query_raw(_LOCK_CUSTOMER_IDENTITIES_SQL) + + async def claim_organization_alias(self, alias: str | None, organization_id: str | None) -> Failure | None: + await self.lock_customer_identities() + if alias is None: + return None + duplicates: Final = await self.organizations.find_many( + where={"alias": alias, "customer_org_id": {"not": organization_id}} + if organization_id + else {"alias": alias}, + take=1, + ) + return Failure(ErrorCode.ALIAS_ALREADY_EXISTS) if duplicates else None + + async def create_organization(self, payload: OrganizationCreate, actor: str) -> Organization | Failure: + alias_failure: Final = await self.claim_organization_alias(payload.alias, None) + if alias_failure is not None: + return alias_failure + if not self.premium_user: + await self.lock_free_capacity() + if await self.organizations.count() >= FREE_ORGANIZATION_LIMIT and ( + payload.customer_org_id is None + or await self.organizations.find_unique(where={"customer_org_id": payload.customer_org_id}) is None + ): + return Failure(ErrorCode.FREE_TIER_LIMIT) + default_index: Final = default_template_index(payload.default_budgets, payload.default_budget_template) + if isinstance(default_index, Failure): + return default_index + names: Final = tuple(template.name for template in payload.default_budgets) + if len(set(names)) != len(names): + return Failure(ErrorCode.INVALID_TEMPLATE_NAMES) + organization_id: Final = payload.customer_org_id or str(uuid4()) + overall_id: Final = await self.create_budget(payload.budget, actor) if payload.budget is not None else None + await self.organizations.create( + data={ + "customer_org_id": organization_id, + "alias": payload.alias, + "budget_id": overall_id, + "created_by": actor, + "updated_by": actor, + } + ) + created: Final = tuple( + [ + await self.create_budget( + Budget.model_validate(template.model_dump(exclude={"name"})), actor, template.name + ) + for template in payload.default_budgets + ] + ) + await self.write_template_metadata( + organization_id, + created, + created[default_index] if default_index is not None else None, + actor, + ) + return await self.get_organization(organization_id) + + async def overall_budget_patch( + self, current_id: str | None, payload: OrganizationPatch | TeamPatch, actor: str + ) -> str | None: + if "budget" not in payload.model_fields_set: + return current_id + if payload.budget is None: + return None + return await self.update_budget(current_id, payload.budget, actor) + + async def delete_released_budget(self, previous_id: str | None, current_id: str | None) -> None: + if previous_id is not None and current_id is None: + await self._delete_unused_budgets((previous_id,)) + + async def update_organization( + self, organization_id: str, payload: OrganizationPatch, actor: str + ) -> Organization | Failure: + alias_failure: Final = await self.claim_organization_alias(payload.alias, organization_id) + if alias_failure is not None: + return alias_failure + await self.lock_organization(organization_id) + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return organization + default_patch: Final = default_template_patch(organization, payload) + if isinstance(default_patch, Failure): + return default_patch + budget_id: Final = await self.overall_budget_patch(organization.budget_id, payload, actor) + await self.organizations.update( + where={"customer_org_id": organization_id}, + data={**alias_patch(payload), **default_patch, "budget_id": budget_id, "updated_by": actor}, + ) + await self.delete_released_budget(organization.budget_id, budget_id) + return await self.get_organization(organization_id) + + async def add_template(self, organization_id: str, payload: BudgetTemplate, actor: str) -> Organization | Failure: + await self.lock_organization(organization_id) + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return organization + names: Final = {template.budget_id: template.name for template in organization.default_budgets} + if payload.name in names.values(): + return Failure(ErrorCode.INVALID_TEMPLATE_NAMES) + budget_id: Final = await self.create_budget( + Budget.model_validate(payload.model_dump(exclude={"name"})), actor, payload.name + ) + default_id: Final = organization.default_budget_template_id if names else budget_id + await self.write_template_metadata(organization_id, (*names, budget_id), default_id, actor) + return await self.get_organization(organization_id) + + async def update_template( + self, organization_id: str, budget_id: str, payload: TemplatePatch, actor: str + ) -> Organization | Failure: + await self.lock_organization(organization_id) + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return organization + names: Final = {template.budget_id: template.name for template in organization.default_budgets} + if budget_id not in names: + return Failure(ErrorCode.TEMPLATE_NOT_IN_ORG) + name: Final = payload.name if payload.name is not None else names[budget_id] + if any(existing_name == name for existing_id, existing_name in names.items() if existing_id != budget_id): + return Failure(ErrorCode.INVALID_TEMPLATE_NAMES) + patch: Final = Budget.model_validate(payload.model_dump(exclude_unset=True, exclude={"name"})) + await self.update_budget(budget_id, patch, actor) + if name != names[budget_id]: + await update_budget_row(self.budgets, budget_id, {"name": name}, actor) + await self.write_template_metadata( + organization_id, tuple(names), organization.default_budget_template_id, actor + ) + return await self.get_organization(organization_id) + + async def delete_template(self, organization_id: str, budget_id: str, actor: str) -> None | Failure: + await self.lock_organization(organization_id) + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return organization + names: Final = {template.budget_id: template.name for template in organization.default_budgets} + if budget_id not in names: + return Failure(ErrorCode.TEMPLATE_NOT_IN_ORG) + if organization.default_budget_template_id == budget_id: + return Failure(ErrorCode.TEMPLATE_IN_USE) + if await self.members.count(where={"budget_id": budget_id}): + return Failure(ErrorCode.TEMPLATE_IN_USE) + remaining: Final = tuple(key for key in names if key != budget_id) + await self.write_template_metadata(organization_id, remaining, organization.default_budget_template_id, actor) + await self._delete_unused_budgets((budget_id,)) + return None + + async def resolve_member_template(self, organization_id: str, template_id: str | None) -> str | Failure | None: + await self.lock_organization(organization_id) + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return Failure(ErrorCode.PARENT_NOT_FOUND) + selected: Final = template_id if template_id is not None else organization.default_budget_template_id + if selected is None: + return None + if any(template.budget_id == selected for template in organization.default_budgets): + return selected + return Failure(ErrorCode.TEMPLATE_NOT_IN_ORG) + + async def configure_templates( + self, organization_id: str, payload: BudgetGroupsUpdate, actor: str + ) -> Organization | Failure: + await self.lock_organization(organization_id) + await self.db.query_raw(_LOCK_TEMPLATES_SQL, organization_id) + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return organization + if payload.expected_revision != organization.budget_groups_revision: + return Failure(ErrorCode.CONFIGURATION_CONFLICT) + originals: Final = {group.budget_id: group for group in organization.default_budgets} + retained: Final = tuple(group.budget_id for group in payload.default_budgets if group.budget_id is not None) + if len(set(retained)) != len(retained) or not set(retained).issubset(originals): + return Failure(ErrorCode.TEMPLATE_NOT_IN_ORG) + names: Final = tuple(group.name for group in payload.default_budgets) + if len(set(names)) != len(names): + return Failure(ErrorCode.INVALID_TEMPLATE_NAMES) + if payload.default_budget_template is not None and payload.default_budget_template not in names: + return Failure(ErrorCode.INVALID_DEFAULT_BUDGET) + removed: Final = tuple(budget_id for budget_id in originals if budget_id not in retained) + if await count_in(self.members, "budget_id", removed): + return Failure(ErrorCode.TEMPLATE_IN_USE) + ids: Final = tuple( + [ + await self.write_configured_template(group, originals.get(group.budget_id or ""), actor) + for group in payload.default_budgets + ] + ) + default_id: Final = ( + ids[names.index(payload.default_budget_template)] if payload.default_budget_template else None + ) + await self.write_template_metadata(organization_id, ids, default_id, actor) + await self._delete_unused_budgets(removed) + return await self.get_organization(organization_id) + + async def write_configured_template( + self, group: BudgetGroupChange, current: OrganizationBudget | None, actor: str + ) -> str: + patch: Final = group_budget_patch(group, current) + if current is None: + return await self.create_budget(patch, actor, group.name) + if patch.model_fields_set: + await self.update_budget(current.budget_id, patch, actor) + if group.name != current.name: + await update_budget_row(self.budgets, current.budget_id, {"name": group.name}, actor) + return current.budget_id + + async def create_team(self, payload: TeamCreate, actor: str) -> Team | Failure: + parent: Final = await self.organizations.find_unique(where={"customer_org_id": payload.parent_org_id}) + if parent is None: + return Failure(ErrorCode.PARENT_NOT_FOUND) + if not self.premium_user: + await self.lock_free_capacity(f"teams:{payload.parent_org_id}") + if await self.teams.count( + where={"parent_org_id": payload.parent_org_id} + ) >= FREE_TEAMS_PER_ORGANIZATION_LIMIT and ( + payload.customer_team_id is None + or await self.teams.find_unique(where={"customer_team_id": payload.customer_team_id}) is None + ): + return Failure(ErrorCode.FREE_TIER_LIMIT) + budget_id: Final = await self.create_budget(payload.budget, actor) if payload.budget is not None else None + override_id: Final = ( + await self.create_budget(payload.budget_override, actor) if payload.budget_override is not None else None + ) + team_id: Final = payload.customer_team_id or str(uuid4()) + await self.teams.create( + data={ + "customer_team_id": team_id, + "alias": payload.alias, + "parent_org_id": payload.parent_org_id, + "budget_id": budget_id, + "budget_override_id": override_id, + "budget_override_expires_at": override_expiry(None, payload, override_id), + "created_by": actor, + "updated_by": actor, + } + ) + return await self.get_team(team_id) + + async def update_team(self, team_id: str, payload: TeamPatch, actor: str) -> Team | Failure: + team: Final = await self.get_team(team_id) + if isinstance(team, Failure): + return team + budget_id: Final = await self.overall_budget_patch(team.budget_id, payload, actor) + override_id: Final = await self.override_patch(team.budget_override_id, payload, actor) + await self.teams.update( + where={"customer_team_id": team_id}, + data={ + **alias_patch(payload), + "budget_id": budget_id, + "budget_override_id": override_id, + "budget_override_expires_at": override_expiry(team, payload, override_id), + "updated_by": actor, + }, + ) + await self.delete_released_budget(team.budget_id, budget_id) + await self.delete_released_budget(team.budget_override_id, override_id) + return await self.get_team(team_id) + + async def member_organization(self, payload: MemberCreate) -> str | Failure: + """The one organization every listed team belongs to, which the payload's ``parent_org_id`` must match.""" + teams: Final = await find_many_in(self.teams, "customer_team_id", payload.parent_team_ids) + organizations: Final = frozenset(_ID.validate_python(team.model_dump()["parent_org_id"]) for team in teams) + if len(teams) != len(set(payload.parent_team_ids)) or len(organizations) != 1: + return Failure(ErrorCode.PARENT_NOT_FOUND) + organization_id: Final = next(iter(organizations)) + if payload.parent_org_id is not None and payload.parent_org_id != organization_id: + return Failure(ErrorCode.PARENT_NOT_FOUND) + return organization_id + + async def create_member( + self, payload: MemberCreate, actor: str, *, budget_required: bool = True + ) -> CustomerMember | Failure: + organization_id: Final = await self.member_organization(payload) + if isinstance(organization_id, Failure): + return organization_id + if await self.member_capacity_reached(payload.parent_team_ids) and ( + payload.customer_member_id is None + or await self.members.find_unique(where={"customer_member_id": payload.customer_member_id}) is None + ): + return Failure(ErrorCode.FREE_TIER_LIMIT) + template_id: Final = await self.resolve_member_template(organization_id, payload.budget_id) + if isinstance(template_id, Failure): + return template_id + if budget_required and template_id is None and payload.budget_override is None: + return Failure(ErrorCode.BUDGET_REQUIRED) + override_id: Final = ( + await self.create_budget(payload.budget_override, actor) if payload.budget_override is not None else None + ) + member_id: Final = payload.customer_member_id or str(uuid4()) + await self.members.create( + data={ + "customer_member_id": member_id, + "alias": payload.alias, + "parent_org_id": organization_id, + "parent_team_ids": tuple(dict.fromkeys(payload.parent_team_ids)), + "budget_id": template_id, + "budget_override_id": override_id, + "budget_override_expires_at": override_expiry(None, payload, override_id), + "created_by": actor, + "updated_by": actor, + } + ) + return await self.get_member(member_id) + + async def member_template_patch(self, member: CustomerMember, payload: MemberPatch) -> str | Failure | None: + if "budget_id" not in payload.model_fields_set: + return member.budget_id + if payload.budget_id is None: + return None + return await self.resolve_member_template(member.parent_org_id, payload.budget_id) + + async def override_patch(self, current_id: str | None, payload: TeamPatch | MemberPatch, actor: str) -> str | None: + if "budget_override" not in payload.model_fields_set: + return current_id + if payload.budget_override is None: + return None + return await self.update_budget(current_id, payload.budget_override, actor) + + async def update_member(self, member_id: str, payload: MemberPatch, actor: str) -> CustomerMember | Failure: + member: Final = await self.get_member(member_id) + if isinstance(member, Failure): + return member + selected: Final = await self.member_template_patch(member, payload) + if isinstance(selected, Failure): + return selected + override: Final = ( + payload.budget_override if "budget_override" in payload.model_fields_set else member.budget_override + ) + had_budget: Final = member.budget_id is not None or member.budget_override is not None + if had_budget and selected is None and override is None: + return Failure(ErrorCode.BUDGET_REQUIRED) + override_id: Final = await self.override_patch(member.budget_override_id, payload, actor) + await self.members.update( + where={"customer_member_id": member_id}, + data={ + **alias_patch(payload), + **({"budget_id": selected} if "budget_id" in payload.model_fields_set else {}), + **( + { + "budget_override_id": override_id, + "budget_override_expires_at": override_expiry(member, payload, override_id), + } + if payload.model_fields_set & {"budget_override", "budget_override_expires_at"} + else {} + ), + "updated_by": actor, + }, + ) + await self.delete_released_budget(member.budget_override_id, override_id) + return await self.get_member(member_id) + + async def list_organizations(self, page: int, page_size: int, search: str = "") -> Page[Organization]: + where: Final = ( + { + "OR": tuple( + {field: {"contains": escape_like(search), "mode": "insensitive"}} + for field in ("alias", "customer_org_id") + ) + } + if search + else {} + ) + count: Final = await self.organizations.count(where=where) + rows: Final = await self.organizations.find_many( + where=where, + skip=(page - 1) * page_size, + take=page_size, + order=[{"created_at": "asc"}, {"customer_org_id": "asc"}], + include=_BUDGET_INCLUDE, + ) + return Page(await self.organization_responses(rows), count) + + async def list_teams(self, parent_org_id: str | None, page: int, page_size: int) -> Page[Team]: + where: Final = {"parent_org_id": parent_org_id} if parent_org_id is not None else {} + count: Final = await self.teams.count(where=where) + rows: Final = await self.teams.find_many( + where=where, + skip=(page - 1) * page_size, + take=page_size, + order=[{"created_at": "asc"}, {"customer_team_id": "asc"}], + include=_MEMBER_INCLUDE, + ) + return Page( + await customer_budget_usage( + tuple(team_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings + ), + count, + ) + + async def list_members( + self, parent_team_id: str | None, page: int, page_size: int, *, parent_org_id: str | None = None + ) -> Page[CustomerMember]: + where: Final[Mapping[str, object]] = { + **({"parent_team_ids": {"has": parent_team_id}} if parent_team_id is not None else {}), + **({"parent_org_id": parent_org_id} if parent_org_id is not None else {}), + } + count: Final = await self.members.count(where=where) + rows: Final = await self.members.find_many( + where=where, + skip=(page - 1) * page_size, + take=page_size, + order=[{"created_at": "asc"}, {"customer_member_id": "asc"}], + include=_MEMBER_INCLUDE, + ) + return Page( + await customer_budget_usage( + tuple(member_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings + ), + count, + ) + + async def _delete_unused_budgets(self, budget_ids: Sequence[str]) -> None: + where: Final[Mapping[str, object]] = {field: {"none": {}} for field in _BUDGET_REFERENCE_FIELDS} + await delete_many_in( + self.budgets, + "budget_id", + budget_ids, + atomicity="caller_transaction", + where=where, + chunk_size=_DELETE_BATCH_SIZE, + ) + + async def _delete_team_member_batch(self, team_id: str) -> bool: + rows: Final = await self.members.find_many(where={"parent_team_ids": {"has": team_id}}, take=_DELETE_BATCH_SIZE) + members: Final = tuple(_MemberDeletionRow.model_validate(row) for row in rows) + if not members: + return False + member_ids: Final = tuple(member.customer_member_id for member in members) + budget_ids: Final = tuple( + member.budget_override_id for member in members if member.budget_override_id is not None + ) + await delete_many_in(self.members, "customer_member_id", member_ids, atomicity="caller_transaction") + await self._delete_unused_budgets(budget_ids) + return True + + async def _delete_team_members(self, team_id: str) -> None: + while True: + if not await self._delete_team_member_batch(team_id): + return + + async def _delete_organization_team_batch(self, organization_id: str) -> bool: + rows: Final = await self.teams.find_many(where={"parent_org_id": organization_id}, take=_DELETE_BATCH_SIZE) + teams: Final = tuple(_TeamDeletionRow.model_validate(row) for row in rows) + if not teams: + return False + for team in teams: + await self._delete_team_members(team.customer_team_id) + team_ids: Final = tuple(team.customer_team_id for team in teams) + budget_ids: Final = tuple( + budget_id + for team in teams + for budget_id in (team.budget_id, team.budget_override_id) + if budget_id is not None + ) # comprehension-ok: two bounded sources per team + await delete_many_in(self.teams, "customer_team_id", team_ids, atomicity="caller_transaction") + await self._delete_unused_budgets(budget_ids) + return True + + async def _delete_organization_teams(self, organization_id: str) -> None: + while True: + if not await self._delete_organization_team_batch(organization_id): + return + + async def member_ids_in_team(self, team_id: str) -> tuple[str, ...]: + return _member_ids(await self.members.find_many(where={"parent_team_ids": {"has": team_id}})) + + async def member_ids_in_organization(self, organization_id: str) -> tuple[str, ...]: + return _member_ids(await self.members.find_many(where={"parent_org_id": organization_id})) + + async def member_ids_using_template(self, budget_id: str) -> tuple[str, ...]: + return _member_ids(await self.members.find_many(where={"budget_id": budget_id})) + + async def delete_member(self, member_id: str) -> None | Failure: + member: Final = await self.get_member(member_id) + if isinstance(member, Failure): + return member + await self.members.delete(where={"customer_member_id": member_id}) + if member.budget_override_id is not None: + await self._delete_unused_budgets((member.budget_override_id,)) + return None + + async def delete_team(self, team_id: str, *, cascade: bool = False) -> None | Failure: + team: Final = await self.get_team(team_id) + if isinstance(team, Failure): + return team + if cascade: + await self._delete_team_members(team_id) + elif await self.members.count(where={"parent_team_ids": {"has": team_id}}): + return Failure(ErrorCode.HAS_CHILDREN) + await self.teams.delete(where={"customer_team_id": team_id}) + await self._delete_unused_budgets(tuple(b for b in (team.budget_id, team.budget_override_id) if b is not None)) + return None + + async def delete_organization(self, organization_id: str, *, cascade: bool = False) -> None | Failure: + organization: Final = await self.get_organization(organization_id) + if isinstance(organization, Failure): + return organization + if cascade: + await self._delete_organization_teams(organization_id) + elif await self.teams.count(where={"parent_org_id": organization_id}): + return Failure(ErrorCode.HAS_CHILDREN) + await self.organizations.delete(where={"customer_org_id": organization_id}) + budget_ids: Final = tuple(template.budget_id for template in organization.default_budgets) + ( + (organization.budget_id,) if organization.budget_id is not None else () + ) + await self._delete_unused_budgets(budget_ids) + return None + + +class CustomerHierarchy: + def __init__(self, db: TransactionSource, *, premium_user: bool, clock: Callable[[], datetime] = utc_now) -> None: + self.db = db + self.clock = clock + self.premium_user = premium_user + + async def portal_limits(self) -> CustomerPortalLimits | Failure: + return await self.transaction(lambda store: store.portal_limits()) + + async def transaction( + self, + operation: Callable[[CustomerHierarchyStore], Awaitable[T]], + *, + operation_type: TransactionOperation = TransactionOperation.READ_WRITE, + ) -> T | Failure: + from prisma.errors import DataError, ForeignKeyViolationError, RecordNotFoundError, UniqueViolationError + + try: + async with self.db.tx() as tx: + return await operation(CustomerHierarchyStore(tx, self.clock(), premium_user=self.premium_user)) + except UniqueViolationError: + return Failure(ErrorCode.ALREADY_EXISTS) + except RecordNotFoundError: + return Failure(ErrorCode.NOT_FOUND) + except ForeignKeyViolationError: + return Failure( + ErrorCode.HAS_CHILDREN if operation_type is TransactionOperation.DELETE else ErrorCode.PARENT_NOT_FOUND + ) + except DataError as error: + if PrismaDBExceptionHandler.is_lock_timeout_error(error): + return Failure(ErrorCode.CAPACITY_BUSY) + raise + + async def create_organization(self, payload: OrganizationCreate, actor: str) -> Organization | Failure: + return await self.transaction(lambda store: store.create_organization(payload, actor)) + + async def get_organization(self, entity_id: str) -> Organization | Failure: + return await self.transaction(lambda store: store.get_organization(entity_id)) + + async def update_organization( + self, entity_id: str, payload: OrganizationPatch, actor: str + ) -> Organization | Failure: + return await self.transaction(lambda store: store.update_organization(entity_id, payload, actor)) + + async def delete_organization(self, entity_id: str, *, cascade: bool = False) -> None | Failure: + return await self.transaction( + lambda store: store.delete_organization(entity_id, cascade=cascade), + operation_type=TransactionOperation.DELETE, + ) + + async def create_team(self, payload: TeamCreate, actor: str) -> Team | Failure: + return await self.transaction(lambda store: store.create_team(payload, actor)) + + async def get_team(self, entity_id: str) -> Team | Failure: + return await self.transaction(lambda store: store.get_team(entity_id)) + + async def update_team(self, entity_id: str, payload: TeamPatch, actor: str) -> Team | Failure: + return await self.transaction(lambda store: store.update_team(entity_id, payload, actor)) + + async def delete_team(self, entity_id: str, *, cascade: bool = False) -> None | Failure: + return await self.transaction( + lambda store: store.delete_team(entity_id, cascade=cascade), operation_type=TransactionOperation.DELETE + ) + + async def create_member( + self, payload: MemberCreate, actor: str, *, budget_required: bool = True + ) -> CustomerMember | Failure: + return await self.transaction( + lambda store: store.create_member(payload, actor, budget_required=budget_required) + ) + + async def get_member(self, entity_id: str) -> CustomerMember | Failure: + return await self.transaction(lambda store: store.get_member(entity_id)) + + async def update_member(self, entity_id: str, payload: MemberPatch, actor: str) -> CustomerMember | Failure: + return await self.transaction(lambda store: store.update_member(entity_id, payload, actor)) + + async def delete_member(self, entity_id: str) -> None | Failure: + return await self.transaction( + lambda store: store.delete_member(entity_id), operation_type=TransactionOperation.DELETE + ) + + async def member_ids_in_team(self, entity_id: str) -> tuple[str, ...] | Failure: + return await self.transaction(lambda store: store.member_ids_in_team(entity_id)) + + async def member_ids_in_organization(self, entity_id: str) -> tuple[str, ...] | Failure: + return await self.transaction(lambda store: store.member_ids_in_organization(entity_id)) + + async def member_ids_using_template(self, budget_id: str) -> tuple[str, ...] | Failure: + return await self.transaction(lambda store: store.member_ids_using_template(budget_id)) + + async def list_organizations( + self, page: int = 1, page_size: int = 50, search: str = "" + ) -> Page[Organization] | Failure: + return await self.transaction(lambda store: store.list_organizations(page, page_size, search)) + + async def list_teams( + self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50 + ) -> Page[Team] | Failure: + return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size)) + + async def list_members( + self, parent_team_id: str | None = None, page: int = 1, page_size: int = 50, *, parent_org_id: str | None = None + ) -> Page[CustomerMember] | Failure: + return await self.transaction( + lambda store: store.list_members(parent_team_id, page, page_size, parent_org_id=parent_org_id) + ) + + async def add_template(self, organization_id: str, payload: BudgetTemplate, actor: str) -> Organization | Failure: + return await self.transaction(lambda store: store.add_template(organization_id, payload, actor)) + + async def configure_templates( + self, organization_id: str, payload: BudgetGroupsUpdate, actor: str + ) -> Organization | Failure: + return await self.transaction(lambda store: store.configure_templates(organization_id, payload, actor)) + + async def update_template( + self, organization_id: str, budget_id: str, payload: TemplatePatch, actor: str + ) -> Organization | Failure: + return await self.transaction(lambda store: store.update_template(organization_id, budget_id, payload, actor)) + + async def delete_template(self, organization_id: str, budget_id: str, actor: str) -> None | Failure: + return await self.transaction( + lambda store: store.delete_template(organization_id, budget_id, actor), + operation_type=TransactionOperation.DELETE, + ) diff --git a/litellm/proxy/management_helpers/customer_upsert.py b/litellm/proxy/management_helpers/customer_upsert.py new file mode 100644 index 00000000000..6285a7580e5 --- /dev/null +++ b/litellm/proxy/management_helpers/customer_upsert.py @@ -0,0 +1,300 @@ +from dataclasses import dataclass +from typing import Final + +from pydantic import BaseModel + +from litellm.proxy.auth.customer_member_auto_create import DEFAULT_TEAM_ALIAS, default_team_id +from litellm.proxy.management_helpers.customer_hierarchy import ( + FREE_TEAMS_PER_ORGANIZATION_LIMIT, + CustomerHierarchy, + CustomerHierarchyStore, + ErrorCode, + Failure, + default_template_index, +) +from litellm.types.proxy.management_endpoints.customer_hierarchy import ( + Budget, + BudgetTemplate, + CustomerMember, + MemberCreate, + MemberPatch, + MemberUpsert, + MemberUpsertResponse, + Organization, + OrganizationPatch, + OrganizationUpsert, + OrganizationUpsertResponse, + Team, +) + + +@dataclass(frozen=True, slots=True) +class OrganizationOutcome: + data: OrganizationUpsertResponse + previous_alias: str | None + + +def matches_patch(current: BaseModel | None, patch: BaseModel | None) -> bool: + if current is None or patch is None: + return current is patch + values: Final = current.model_dump() + return all(values.get(key) == value for key, value in patch.model_dump(exclude_unset=True).items()) + + +async def organization_by_identity( + store: CustomerHierarchyStore, customer_org_id: str | None, alias: str | None +) -> Organization | Failure: + if customer_org_id is not None: + await store.lock_organization(customer_org_id) + return await store.get_organization(customer_org_id) + rows: Final = await store.organizations.find_many(where={"alias": alias}, take=2) + if len(rows) > 1: + return Failure(ErrorCode.ALIAS_ALREADY_EXISTS) + if not rows: + return Failure(ErrorCode.NOT_FOUND) + organization: Final = Organization.model_validate(rows[0].model_dump()) + await store.lock_organization(organization.customer_org_id) + return await store.get_organization(organization.customer_org_id) + + +def template_budget(template: BudgetTemplate) -> Budget: + return Budget.model_validate(template.model_dump(exclude_unset=True, exclude={"name"})) + + +def templates_match(organization: Organization, payload: OrganizationUpsert, default_index: int | None) -> bool: + if not payload.default_budgets: + return True + existing: Final = {template.name: template for template in organization.default_budgets} + return all( + template.name in existing and matches_patch(existing[template.name], template_budget(template)) + for template in payload.default_budgets + ) and ( + default_index is not None + and payload.default_budgets[default_index].name in existing + and existing[payload.default_budgets[default_index].name].budget_id == organization.default_budget_template_id + ) + + +async def write_templates( + store: CustomerHierarchyStore, + organization: Organization, + payload: OrganizationUpsert, + default_index: int, + actor: str, +) -> None: + existing: Final = {template.name: template.budget_id for template in organization.default_budgets} + written: Final = tuple( + [ + await store.update_budget(existing[template.name], template_budget(template), actor) + if template.name in existing + else await store.create_budget(template_budget(template), actor, template.name) + for template in payload.default_budgets + ] + ) + current_ids: Final = tuple(template.budget_id for template in organization.default_budgets) + await store.write_template_metadata( + organization.customer_org_id, + (*current_ids, *(budget_id for budget_id in written if budget_id not in current_ids)), + written[default_index], + actor, + ) + + +async def upsert_organization( + store: CustomerHierarchyStore, payload: OrganizationUpsert, actor: str +) -> OrganizationOutcome | Failure: + await store.lock_customer_identities() + default_index: Final = default_template_index(payload.default_budgets, payload.default_budget_template) + if isinstance(default_index, Failure): + return default_index + names: Final = tuple(template.name for template in payload.default_budgets) + if len(set(names)) != len(names): + return Failure(ErrorCode.INVALID_TEMPLATE_NAMES) + existing: Final = await organization_by_identity(store, payload.customer_org_id, payload.alias) + if isinstance(existing, Failure): + if existing.code is not ErrorCode.NOT_FOUND: + return existing + created: Final = await store.create_organization(payload, actor) + if isinstance(created, Failure): + return created + return OrganizationOutcome( + OrganizationUpsertResponse.model_validate({**created.model_dump(), "created": True}), None + ) + patch: Final = OrganizationPatch.model_validate( + payload.model_dump( + exclude_unset=True, exclude={"customer_org_id", "default_budgets", "default_budget_template"} + ) + ) + alias_failure: Final = await store.claim_organization_alias(payload.alias, existing.customer_org_id) + if alias_failure is not None: + return alias_failure + alias_unchanged: Final = "alias" not in patch.model_fields_set or existing.alias == patch.alias + budget_unchanged: Final = "budget" not in patch.model_fields_set or matches_patch(existing.budget, patch.budget) + if alias_unchanged and budget_unchanged and templates_match(existing, payload, default_index): + return OrganizationOutcome( + OrganizationUpsertResponse.model_validate({**existing.model_dump(), "created": False}), existing.alias + ) + updated: Final = await store.update_organization(existing.customer_org_id, patch, actor) + if isinstance(updated, Failure): + return updated + if default_index is not None: + await write_templates(store, updated, payload, default_index, actor) + result: Final = await store.get_organization(existing.customer_org_id) + if isinstance(result, Failure): + return result + return OrganizationOutcome( + OrganizationUpsertResponse.model_validate({**result.model_dump(), "created": False}), existing.alias + ) + + +async def customer_team( + store: CustomerHierarchyStore, organization_id: str, team_id: str | None, actor: str +) -> Team | Failure: + if team_id is not None: + explicit_team: Final = await store.get_team(team_id) + if isinstance(explicit_team, Failure): + return Failure(ErrorCode.PARENT_NOT_FOUND) + return ( + explicit_team + if explicit_team.parent_org_id == organization_id + else Failure(ErrorCode.CUSTOMER_ORG_MISMATCH) + ) + if not store.premium_user: + await store.lock_free_capacity(f"teams:{organization_id}") + if ( + await store.teams.find_unique(where={"customer_team_id": default_team_id(organization_id)}) is None + and await store.teams.count(where={"parent_org_id": organization_id}) >= FREE_TEAMS_PER_ORGANIZATION_LIMIT + ): + return Failure(ErrorCode.FREE_TIER_LIMIT) + await store.teams.upsert( + where={"customer_team_id": default_team_id(organization_id)}, + data={ + "create": { + "customer_team_id": default_team_id(organization_id), + "alias": DEFAULT_TEAM_ALIAS, + "parent_org_id": organization_id, + "created_by": actor, + "updated_by": actor, + }, + "update": {}, + }, + ) + team: Final = await store.get_team(default_team_id(organization_id)) + if isinstance(team, Failure): + return team + return team if team.parent_org_id == organization_id else Failure(ErrorCode.CUSTOMER_ORG_MISMATCH) + + +async def member_team_ids( + store: CustomerHierarchyStore, + organization_id: str, + existing: CustomerMember | Failure, + team_id: str | None, + actor: str, +) -> tuple[str, ...] | Failure: + if not isinstance(existing, Failure) and team_id is None: + return existing.parent_team_ids + team: Final = await customer_team(store, organization_id, team_id, actor) + if isinstance(team, Failure): + return team + if isinstance(existing, Failure): + return (team.customer_team_id,) + return (team.customer_team_id, *(tid for tid in existing.parent_team_ids if tid != team.customer_team_id)) + + +async def upsert_member( + store: CustomerHierarchyStore, payload: MemberUpsert, actor: str +) -> MemberUpsertResponse | Failure: + await store.lock_customer_identities() + organization: Final = await organization_by_identity(store, payload.customer_org_id, payload.customer_org_alias) + if isinstance(organization, Failure): + return Failure(ErrorCode.PARENT_NOT_FOUND) if organization.code is ErrorCode.NOT_FOUND else organization + if payload.customer_org_alias is not None and organization.alias != payload.customer_org_alias: + return Failure(ErrorCode.CUSTOMER_ORG_MISMATCH) + existing: Final = await store.get_member(payload.customer_member_id) + if not isinstance(existing, Failure): + if existing.parent_org_id != organization.customer_org_id: + return Failure(ErrorCode.CUSTOMER_ORG_MISMATCH) + elif existing.code is not ErrorCode.NOT_FOUND: + return existing + patch: Final = MemberPatch.model_validate( + payload.model_dump( + exclude_unset=True, + exclude={"customer_member_id", "customer_org_id", "customer_org_alias", "parent_team_id"}, + ) + ) + requested_budget: Final = ( + patch.budget_id + if "budget_id" in patch.model_fields_set + else organization.default_budget_template_id + if isinstance(existing, Failure) + else existing.budget_id + ) + selected: Final = ( + requested_budget or organization.default_budget_template_id + if isinstance(existing, Failure) + else requested_budget + ) + if selected is not None and not any(template.budget_id == selected for template in organization.default_budgets): + return Failure(ErrorCode.TEMPLATE_NOT_IN_ORG) + override: Final = ( + patch.budget_override + if "budget_override" in patch.model_fields_set or isinstance(existing, Failure) + else existing.budget_override + ) + if selected is None and override is None: + return Failure(ErrorCode.BUDGET_REQUIRED) + team_ids: Final = await member_team_ids( + store, organization.customer_org_id, existing, payload.parent_team_id, actor + ) + if isinstance(team_ids, Failure): + return team_ids + if isinstance(existing, Failure): + member: Final = await store.create_member( + MemberCreate.model_validate( + { + **patch.model_dump(exclude_unset=True), + "customer_member_id": payload.customer_member_id, + "parent_team_ids": team_ids, + } + ), + actor, + ) + if isinstance(member, Failure): + return member + return MemberUpsertResponse.model_validate({**member.model_dump(), "created": True}) + alias_matches: Final = "alias" not in patch.model_fields_set or existing.alias == patch.alias + override_matches: Final = "budget_override" not in patch.model_fields_set or matches_patch( + existing.budget_override, patch.budget_override + ) + expiry_matches: Final = "budget_override_expires_at" not in patch.model_fields_set or ( + existing.budget_override_expires_at == patch.budget_override_expires_at + ) + if ( + existing.parent_team_ids == team_ids + and selected == existing.budget_id + and alias_matches + and override_matches + and expiry_matches + ): + return MemberUpsertResponse.model_validate({**existing.model_dump(), "created": False}) + if await store.member_capacity_reached(tuple(tid for tid in team_ids if tid not in existing.parent_team_ids)): + return Failure(ErrorCode.FREE_TIER_LIMIT) + await store.members.update( + where={"customer_member_id": payload.customer_member_id}, data={"parent_team_ids": team_ids} + ) + updated: Final = await store.update_member(payload.customer_member_id, patch, actor) + if isinstance(updated, Failure): + return updated + return MemberUpsertResponse.model_validate({**updated.model_dump(), "created": False}) + + +class CustomerUpsert: + def __init__(self, hierarchy: CustomerHierarchy) -> None: + self.hierarchy = hierarchy + + async def organization(self, payload: OrganizationUpsert, actor: str) -> OrganizationOutcome | Failure: + return await self.hierarchy.transaction(lambda store: upsert_organization(store, payload, actor)) + + async def member(self, payload: MemberUpsert, actor: str) -> MemberUpsertResponse | Failure: + return await self.hierarchy.transaction(lambda store: upsert_member(store, payload, actor)) diff --git a/litellm/proxy/management_helpers/resource_display_names.py b/litellm/proxy/management_helpers/resource_display_names.py index f3a97da1b12..9c9c99b7b43 100644 --- a/litellm/proxy/management_helpers/resource_display_names.py +++ b/litellm/proxy/management_helpers/resource_display_names.py @@ -1,4 +1,5 @@ -"""Display names for ids stored on management objects. DB rows win; config-declared servers and agents fill the gaps.""" +"""Display names for ids stored on management objects. DB rows win; config-declared servers and agents fill the gaps. +A vector store declared in the config file keeps its config name, as it does in the vector store list.""" from collections.abc import Mapping, Sequence from types import MappingProxyType @@ -6,9 +7,15 @@ from typing import Final from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry from litellm.proxy.utils import PrismaClient -from litellm.repositories.table_repositories import AgentsRepository, MCPServerRepository +from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.table_repositories import ( + AgentsRepository, + ManagedVectorStoresRepository, + MCPServerRepository, +) from litellm.repositories.verification_token_repository import VerificationTokenRepository from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.types.vector_stores import LiteLLM_ManagedVectorStore async def mcp_server_display_names( @@ -52,6 +59,23 @@ async def agent_display_names( return MappingProxyType({**from_registry, **from_db}) +async def vector_store_display_names( + prisma_client: PrismaClient, + vector_store_ids: Sequence[str], + registered_stores: Sequence[LiteLLM_ManagedVectorStore], +) -> Mapping[str, str]: + """vector_store_id -> vector_store_name for the stores that have one.""" + wanted: Final = frozenset(vector_store_ids) + rows: Final = await find_many_in(ManagedVectorStoresRepository(prisma_client).table, "vector_store_id", wanted) + from_db: Final = {row.vector_store_id: row.vector_store_name for row in rows} + from_config: Final = { + store_id: store.get("vector_store_name") + for store in registered_stores + if store.get("is_config") and (store_id := store.get("vector_store_id")) is not None and store_id in wanted + } + return MappingProxyType({store_id: name for store_id, name in {**from_db, **from_config}.items() if name}) + + async def key_display_names(prisma_client: PrismaClient, tokens: Sequence[str]) -> Mapping[str, str]: """token hash -> key_alias for the keys that have one.""" if not tokens: diff --git a/litellm/proxy/management_helpers/team_roster_sync.py b/litellm/proxy/management_helpers/team_roster_sync.py index 08be455a55d..fb3e4d01197 100644 --- a/litellm/proxy/management_helpers/team_roster_sync.py +++ b/litellm/proxy/management_helpers/team_roster_sync.py @@ -38,6 +38,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import ( TEAM_ADVISORY_LOCK_SQL, USER_ROWS_LOCK_SQL, ) +from litellm.proxy.management_helpers.utils import reject_budget_windows from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in from litellm.repositories.team_repository import TeamRepository @@ -196,6 +197,8 @@ async def _member_budget_ids( if default_budget_id is None: return MappingProxyType({}) default_budget: Final = await _budget_tx_db(tx).find_unique(where={"budget_id": default_budget_id}) + if default_budget is not None: + reject_budget_windows(default_budget.model_dump()) return MappingProxyType(dict.fromkeys(user_ids, default_budget_id if default_budget is not None else None)) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index d2b4e78c8f4..5d5b391876b 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -10,12 +10,14 @@ from typing import Any, Final, ParamSpec, Protocol, TypeVar from fastapi import HTTPException, Request from pydantic import BaseModel, TypeAdapter from starlette.exceptions import HTTPException as StarletteHTTPException +from typing_extensions import LiteralString import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.integrations.otel.model.config import is_otel_v2_enabled +from litellm.litellm_core_utils.core_helpers import attribute_or_none, require_attributes from litellm.proxy._types import ( # key request types; user request types; team request types; customer request types BudgetNewRequest, DeleteCustomerRequest, @@ -126,6 +128,63 @@ def _budget_table(prisma_client: PrismaClient, tx: MemberWriteTx | None) -> _Pri return tx.litellm_budgettable if tx is not None else BudgetRepository(prisma_client).table +def reject_budget_windows(row: Mapping[str, object]) -> None: + if row.get("budget_limits"): + raise HTTPException( + status_code=400, + detail={"error": "Concurrent budget windows are currently supported only by customer hierarchies."}, + ) + + +async def validate_scalar_budget_attachment( + prisma_client: PrismaClient, budget_id: str | None, tx: MemberWriteTx | None = None +) -> _PrismaBudgetRecord | None: + if budget_id is None: + return + table: Final = ( + tx.litellm_budgettable + if tx is not None + else require_attributes( + attribute_or_none(prisma_client.writer_db, "litellm_budgettable"), _PrismaBudgetTable, "find_unique" + ) + ) + row: Final = await table.find_unique(where={"budget_id": budget_id}) + if row is not None: + reject_budget_windows(row.model_dump()) + return row + + +_UNSUPPORTED_BUDGET_REFERENCES: Final = """SELECT EXISTS ( + SELECT 1 FROM "LiteLLM_VerificationToken" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_EndUserTable" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_OrganizationTable" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_ProjectTable" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_TagTable" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_ModelAccessGroupBudgetTable" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_TeamMembership" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_OrganizationMembership" WHERE budget_id = $1 + UNION ALL SELECT 1 FROM "LiteLLM_TeamTable" WHERE metadata->>'team_member_budget_id' = $1 + UNION ALL SELECT 1 FROM "LiteLLM_VerificationToken" WHERE metadata->>'end_user_budget_id' = $1 +) AS referenced""" +_BUDGET_REFERENCE_ROWS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +class _BudgetReferenceReader(Protocol): + async def query_raw(self, query: LiteralString, *args: str) -> object: ... + + +async def validate_budget_windows_update(db: object, budget_id: str) -> None: + reader: Final = require_attributes(db, _BudgetReferenceReader, "query_raw") + rows: Final = _BUDGET_REFERENCE_ROWS.validate_python( + await reader.query_raw(_UNSUPPORTED_BUDGET_REFERENCES, budget_id) + ) + if litellm.max_end_user_budget_id == budget_id or any(row.get("referenced") is True for row in rows): + raise HTTPException( + status_code=400, + detail={"error": "Detach non-customer-hierarchy consumers before adding concurrent budget windows."}, + ) + + def _team_membership_table(prisma_client: PrismaClient, tx: MemberWriteTx | None) -> _PrismaTeamMembershipTable: return tx.litellm_teammembership if tx is not None else TeamMembershipRepository(prisma_client).table @@ -233,6 +292,7 @@ async def handle_budget_for_entity( # Check if budget_id is explicitly provided in the data data_budget_id: Final[str | None] = getattr(data, "budget_id", None) + await validate_scalar_budget_attachment(prisma_client, data_budget_id) # Case 1: Creating new entity - no existing budget_id if existing_budget_id is None: @@ -317,6 +377,7 @@ async def _clone_team_default_budget_for_member( return None default_budget_dict: Final = default_budget.model_dump() + reject_budget_windows(default_budget_dict) cloned_data: Final[dict] = { "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, @@ -369,6 +430,8 @@ async def _resolve_member_budget_id( default_budget: Final = await _budget_table(prisma_client, tx).find_unique( where={"budget_id": default_team_budget_id} ) + if default_budget is not None: + reject_budget_windows(default_budget.model_dump()) return default_team_budget_id if default_budget is not None else None if not has_explicit_limit and default_team_budget_id is not None: diff --git a/litellm/proxy/mcp_server/catalog.py b/litellm/proxy/mcp_server/catalog.py index e30f26abf41..a76f6c97cb5 100644 --- a/litellm/proxy/mcp_server/catalog.py +++ b/litellm/proxy/mcp_server/catalog.py @@ -15,11 +15,12 @@ from dataclasses import dataclass, replace from functools import partial, wraps from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Generic, ParamSpec, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Final, Generic, ParamSpec, Protocol, TypeAlias, TypeVar, cast from mcp.shared.exceptions import MCPError from mcp.types import CacheableResult from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from typing_extensions import Self from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -849,6 +850,7 @@ async def get_filtered_server_tools( params: PaginatedRequestParams | None = None, record_listing: bool = False, listing_updates: ExitStack | None = None, + use_tool_list_cache: bool = True, ) -> tuple[ListToolsResult, ServerOutcome]: from mcp.types import ListToolsResult @@ -893,6 +895,7 @@ async def get_filtered_server_tools( proxy_logging_obj=proxy_logging_obj, catalog_auth_header=catalog_auth_header, record_listing=False, + use_tool_list_cache=use_tool_list_cache, ) ) else: @@ -910,6 +913,7 @@ async def get_filtered_server_tools( catalog_auth_header=catalog_auth_header, record_listing=False, listing_updates=listing_updates, + use_tool_list_cache=use_tool_list_cache, ) tools: Final = page.tools filtered_tools = filter_tools_by_allowed_tools(tools, server) @@ -1001,6 +1005,7 @@ async def aggregate_gateway_tools( *, record_listing: bool = False, enforce_rate_limits: bool = True, + use_tool_list_cache: bool = True, ) -> AggregateToolListing: import time @@ -1050,6 +1055,7 @@ async def aggregate_gateway_tools( params=PaginatedRequestParams(cursor=cursor), record_listing=record_listing, listing_updates=listing_updates, + use_tool_list_cache=use_tool_list_cache, ) if cursor is not None and outcome.tag != "ok": from mcp.shared.exceptions import MCPError @@ -1337,29 +1343,55 @@ def combine_optional_catalog( ) -_DiscoveryPage = TypeVar("_DiscoveryPage", bound=CacheableResult) +class _Expiring(Protocol): + @property + def ttl_ms(self) -> int: ... + + def model_copy(self, *, update: Mapping[str, object] | None = None, deep: bool = False) -> Self: ... + + +_DiscoveryPage = TypeVar("_DiscoveryPage", bound=_Expiring) _DiscoveryKey: TypeAlias = tuple[str, str | None] _DISCOVERY_CACHE_LIMIT: Final = 1024 _DISCOVERY_ENTRY: Final = TypeAdapter(tuple[float, bytes]) class _DiscoveryCache(Generic[_DiscoveryPage]): - def __init__(self, ttl: float, clock: Callable[[], float], adapter: TypeAdapter[_DiscoveryPage]) -> None: + def __init__( + self, + ttl: float, + clock: Callable[[], float], + adapter: TypeAdapter[_DiscoveryPage], + max_entry_kb: int = 64, + storable: Callable[[_DiscoveryPage], bool] | None = None, + ) -> None: self._ttl = ttl self._clock = clock self._adapter = adapter - self._entries = InMemoryCache(max_size_in_memory=_DISCOVERY_CACHE_LIMIT, max_size_per_item=64, clock=clock) + self._storable = storable + self._entries = InMemoryCache( + max_size_in_memory=_DISCOVERY_CACHE_LIMIT, max_size_per_item=max_entry_kb, clock=clock + ) self._pending: dict[_DiscoveryKey, asyncio.Task[_DiscoveryPage]] = {} self._waiters: dict[asyncio.Task[_DiscoveryPage], int] = {} # mutable-ok: constant-time waiter accounting + self._oversized_server_ids: frozenset[str] = frozenset() + self.invalidations = 0 - def invalidate(self, server_id: str) -> None: + def _entry_keys(self, server_id: str) -> tuple[str, ...]: prefix: Final = f"[{json.dumps(server_id)}," keys: Final = cast( # cast-ok: private cache contains only JSON string keys "tuple[str, ...]", tuple(self._entries.cache_dict) ) - for entry_key in keys: - if entry_key.startswith(prefix): - self._entries.delete_cache(entry_key) + return tuple(entry_key for entry_key in keys if entry_key.startswith(prefix)) + + def holds(self, server_id: str) -> bool: + return bool(self._entry_keys(server_id)) + + def invalidate(self, server_id: str) -> None: + self.invalidations += 1 + self._oversized_server_ids = self._oversized_server_ids - {server_id} + for entry_key in self._entry_keys(server_id): + self._entries.delete_cache(entry_key) for key in tuple(self._pending): if key[0] == server_id: self._pending.pop(key) @@ -1369,7 +1401,9 @@ class _DiscoveryCache(Generic[_DiscoveryPage]): if not task.cancelled(): task.exception() - async def get(self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[_DiscoveryPage]]) -> _DiscoveryPage: + async def get( + self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[_DiscoveryPage]], *, capped: bool = True + ) -> _DiscoveryPage: if self._ttl <= 0: return await fetch() entry: Final[object] = self._entries.get_cache(json.dumps(key)) @@ -1384,7 +1418,7 @@ class _DiscoveryCache(Generic[_DiscoveryPage]): return await self._await_fetch(key, pending) if len(self._pending) >= _DISCOVERY_CACHE_LIMIT: return await fetch() - task: Final = asyncio.create_task(self._fetch(key, fetch)) + task: Final = asyncio.create_task(self._fetch(key, fetch, capped)) self._pending[key] = task task.add_done_callback(self._observe_completion) return await self._await_fetch(key, task) @@ -1404,17 +1438,34 @@ class _DiscoveryCache(Generic[_DiscoveryPage]): if not task.done(): task.cancel() - async def _fetch(self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[_DiscoveryPage]]) -> _DiscoveryPage: + async def _fetch( + self, key: _DiscoveryKey, fetch: Callable[[], Awaitable[_DiscoveryPage]], capped: bool + ) -> _DiscoveryPage: try: items: Final = await fetch() - ttl: Final = min(self._ttl, items.ttl_ms / 1000) - if ttl > 0 and self._pending.get(key) is asyncio.current_task(): - self._entries.set_cache( - json.dumps(key), - _DISCOVERY_ENTRY.dump_json((self._clock() + ttl, self._adapter.dump_json(items))), - ttl=ttl, - ) + if self._pending.get(key) is asyncio.current_task(): + self.keep(key, items, capped=capped) return items finally: if self._pending.get(key) is asyncio.current_task(): self._pending.pop(key) + + def keep(self, key: _DiscoveryKey, items: _DiscoveryPage, *, capped: bool = True) -> None: + declared: Final = items.ttl_ms / 1000 + ttl: Final = min(self._ttl, declared) if capped else declared + if ttl <= 0 or (self._storable is not None and not self._storable(items)): + return + entry: Final = _DISCOVERY_ENTRY.dump_json((self._clock() + ttl, self._adapter.dump_json(items))) + if self._entries.check_value_size(entry): + self._entries.set_cache(json.dumps(key), entry, ttl=ttl) + return + server_id: Final = key[0] + if server_id in self._oversized_server_ids: + return + self._oversized_server_ids = self._oversized_server_ids | {server_id} + verbose_logger.warning( + "MCP server %s returned a %s KB listing, above the %s KB cache entry limit. It is fetched on every request", + server_id, + len(entry) // 1024, + self._entries.max_size_per_item, + ) diff --git a/litellm/proxy/mcp_server/discoverable_endpoints.py b/litellm/proxy/mcp_server/discoverable_endpoints.py index f36609aa8b3..36a312d7c1f 100644 --- a/litellm/proxy/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/mcp_server/discoverable_endpoints.py @@ -110,6 +110,7 @@ from litellm.proxy.mcp_server.oauth_identity_binding import ( from litellm.proxy.mcp_server.oauth_utils import ( CIMD_METADATA_PATH, TOKEN_NO_CACHE_HEADERS, + asks_for_wildcard, build_upstream_authorize_url, build_upstream_oauth2_token_request, configured_proxy_base_url, @@ -1066,7 +1067,10 @@ async def _managed_authorization_scope( if scope_user_id and prisma_client is not None else None ) - merged_scope: Final = union_scopes(" ".join(prior_grant.get("scopes") or ()) if prior_grant else None, scope) + kept_scope: Final = ( + None if prior_grant is None or asks_for_wildcard(scope) else " ".join(prior_grant.get("scopes") or ()) + ) + merged_scope: Final = union_scopes(kept_scope, scope) selected_scope: Final = ( get_client_metadata_scopes( merged_scope, diff --git a/litellm/proxy/mcp_server/mcp_server_manager.py b/litellm/proxy/mcp_server/mcp_server_manager.py index 3d55f63c73d..d31e9246a38 100644 --- a/litellm/proxy/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/mcp_server/mcp_server_manager.py @@ -26,7 +26,7 @@ from collections.abc import ( ) from contextlib import AbstractAsyncContextManager, ExitStack, asynccontextmanager, nullcontext from dataclasses import dataclass, replace -from functools import lru_cache +from functools import lru_cache, partial from itertools import chain, groupby from types import EllipsisType, MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, cast @@ -58,8 +58,8 @@ from mcp.types import ( ResourceTemplate, ) from mcp.types import Tool as MCPTool -from pydantic import AnyUrl, Field, SecretStr, TypeAdapter, ValidationError -from typing_extensions import ReadOnly, assert_never +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, Self, assert_never import litellm from litellm._internal_context import with_service_target @@ -236,6 +236,7 @@ from litellm.types.mcp import ( without_header, ) from litellm.types.mcp_server.mcp_server_manager import ( + MAX_TOOLS_LIST_CACHE_TTL_SECONDS, MCPInfo, MCPOAuthMetadata, MCPServer, @@ -372,6 +373,12 @@ _UpstreamSessionKey: TypeAlias = tuple[ ] +@dataclass(frozen=True, slots=True) +class ResourceListingFailed: + server_name: str + error: Exception + + @dataclass(frozen=True, slots=True) class _OAuthDiscoveryResolved: server: MCPServer @@ -1620,13 +1627,14 @@ def _redacted_registry_dump(servers: dict[str, MCPServer]) -> dict[str, dict[str def caller_authorization_fans_out( server: MCPServer, - scope_servers: list[MCPServer] | None, + scope_servers: Sequence[MCPServer] | None, ) -> bool: """True when forwarding the caller's request-wide ``Authorization`` to ``server`` inside a listing fan-out would replay one credential against multiple upstreams: another server in the scope also consumes it (RFC 9700 cross-resource replay). ``scope_servers`` is None for - explicitly-addressed operations (tool call, get_prompt, read_resource, single-server routes), - where the client named the one target and the gateway is not choosing recipients.""" + explicitly-addressed operations (tool call, get_prompt, single-server routes), where the client + named the one target and the gateway is not choosing recipients. An aggregate read_resource passes + its scope because the gateway picks the owner from listings.""" if scope_servers is None: return False return any( @@ -1896,6 +1904,49 @@ def _mcp_discovery_cache_ttl() -> float: return ttl +_TOOL_LIST_ENTRY_LIMIT_KB: Final = 1024 + + +class _UpstreamToolListing(BaseModel): + model_config = ConfigDict(frozen=True) + + page: ListToolsResult + ttl_ms: int + upstream_instructions: str | None + + @classmethod + def fetched_by(cls, client: MCPClient, page: ListToolsResult) -> Self: + declared: Final[object] = getattr(client, "_last_initialize_instructions", None) + return cls(page=page, ttl_ms=page.ttl_ms, upstream_instructions=declared if isinstance(declared, str) else None) + + def is_whole(self) -> bool: + return not self.page.next_cursor + + def page_to_serve(self) -> ListToolsResult: + if self.ttl_ms == self.page.ttl_ms: + return self.page + return self.page.model_copy(update={"ttl_ms": self.ttl_ms}) + + +def _tool_page_cursor(params: PaginatedRequestParams | None) -> tuple[str | None] | None: + return None if params is None else (params.cursor,) + + +def _mcp_tools_list_cache_ttl() -> int | None: + raw: Final = os.environ.get("LITELLM_MCP_TOOLS_LIST_CACHE_TTL") + if raw is None: + return None + try: + ttl: Final = int(raw) + except ValueError: + verbose_logger.warning("Invalid LITELLM_MCP_TOOLS_LIST_CACHE_TTL; ignoring it") + return None + if not 0 <= ttl <= MAX_TOOLS_LIST_CACHE_TTL_SECONDS: + verbose_logger.warning("Invalid LITELLM_MCP_TOOLS_LIST_CACHE_TTL; ignoring it") + return None + return ttl + + def _listing_task_was_cancelled(error: asyncio.CancelledError) -> bool: """The MCP client raises its own CancelledError(SESSION_CLEANUP_TIMEOUT_MESSAGE) when a session's teardown outlives its budget while the listing task keeps running. That one is the server's fault, @@ -2045,6 +2096,15 @@ class MCPServerManager: self._template_discovery_cache = _DiscoveryCache[ListResourceTemplatesResult]( discovery_ttl, discovery_clock, TypeAdapter(ListResourceTemplatesResult) ) + self._tool_discovery_cache = _DiscoveryCache[_UpstreamToolListing]( + discovery_ttl, + discovery_clock, + TypeAdapter(_UpstreamToolListing), + max_entry_kb=_TOOL_LIST_ENTRY_LIMIT_KB, + storable=_UpstreamToolListing.is_whole, + ) + self._list_caching_enabled = discovery_ttl > 0 + self._tools_list_cache_ttl = _mcp_tools_list_cache_ttl() from litellm.proxy.mcp_server.catalog import CatalogSnapshots self.catalog = CatalogSnapshots(self) @@ -4440,6 +4500,7 @@ class MCPServerManager: *, catalog_auth_header: str | dict[str, str] | None | EllipsisType = ..., record_listing: bool = False, + use_tool_list_cache: bool = True, ) -> list[MCPTool]: result: Final = await self.get_tools_page( server=server, @@ -4453,6 +4514,7 @@ class MCPServerManager: proxy_logging_obj=proxy_logging_obj, catalog_auth_header=catalog_auth_header, record_listing=record_listing, + use_tool_list_cache=use_tool_list_cache, ) return result.tools @@ -4474,6 +4536,7 @@ class MCPServerManager: catalog_auth_header: str | dict[str, str] | None | EllipsisType = ..., record_listing: bool = False, listing_updates: ExitStack | None = None, + use_tool_list_cache: bool = True, ) -> ListToolsResult: from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, @@ -4521,6 +4584,7 @@ class MCPServerManager: # upstream API key or a user's OAuth token (MCPClient._get_auth_headers # applies extra_headers after writing Authorization from auth_value, so # an injected JWT would otherwise clobber the per-user token). + headers_before_signing: Final = extra_headers if user_api_key_auth is not None and not server.spec_path: from litellm.proxy.guardrails.guardrail_hooks.mcp_jwt_signer.mcp_jwt_signer import ( get_mcp_jwt_signer, @@ -4595,14 +4659,22 @@ class MCPServerManager: tools=[t.model_copy(update={"name": registered_names[t.name]}) for t in guarded_openapi] ) else: - page: Final = ( - await retry_upstream_rate_limit( - lambda: self._fetch_tools_page_with_timeout(client, server, params), - server.name, - deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, - ) - if params is not None - else ListToolsResult(tools=await self._fetch_tools_with_timeout(client, server.name)) + page: Final = await self._list_upstream_tools( + server, + client, + params, + partial( + self._discovery_key, + server, + user_api_key_auth, + mcp_auth_header, + extra_headers, + stdio_env, + subject_token, + raw_headers=raw_headers, + page_cursor=_tool_page_cursor(params), + ), + use_cache=use_tool_list_cache and extra_headers is headers_before_signing, ) tools = page.tools self._remember_upstream_initialize_instructions(server, client) @@ -4663,6 +4735,92 @@ class MCPServerManager: ) raise_classified_list_failure(e, server.name, suppress_challenge=server.is_dcr_bridge) + def _admin_tools_list_ttl(self, server: MCPServer) -> int | None: + if not self._list_caching_enabled: + return None + per_server: Final = server.tools_list_cache_ttl_seconds + return self._tools_list_cache_ttl if per_server is None else per_server + + def _tools_fetch( + self, client: MCPClient, server: MCPServer, params: PaginatedRequestParams | None + ) -> Callable[[], Awaitable[ListToolsResult]]: + if params is None: + return partial(self._fetch_tools_with_timeout, client, server.name) + return partial( + retry_upstream_rate_limit, + partial(self._fetch_tools_page_with_timeout, client, server, params), + server.name, + deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, + ) + + async def _list_upstream_tools( + self, + server: MCPServer, + client: MCPClient, + params: PaginatedRequestParams | None, + key_for_credential: Callable[[str], _DiscoveryKey], + *, + use_cache: bool, + ) -> ListToolsResult: + fetch_page: Final = self._tools_fetch(client, server, params) + + async def fetch() -> _UpstreamToolListing: + return _UpstreamToolListing.fetched_by(client, await fetch_page()) + + listing: Final = await self._cached_or_upstream_tools( + server, client, params, key_for_credential, fetch, use_cache + ) + self._remember_instructions_if_missing(server, listing.upstream_instructions) + return listing.page_to_serve() + + def _remember_instructions_if_missing(self, server: MCPServer, seen_by_this_caller: str | None) -> None: + if seen_by_this_caller and not self._upstream_initialize_instructions_by_server_id.get(server.server_id): + self._upstream_initialize_instructions_by_server_id[server.server_id] = seen_by_this_caller + + async def _cached_or_upstream_tools( + self, + server: MCPServer, + client: MCPClient, + params: PaginatedRequestParams | None, + key_for_credential: Callable[[str], _DiscoveryKey], + fetch: Callable[[], Awaitable[_UpstreamToolListing]], + use_cache: bool, + ) -> _UpstreamToolListing: + if not use_cache or (params is not None and params.cursor is not None): + return await fetch() + + async def cache_key() -> _DiscoveryKey: + return key_for_credential(await client.discovery_auth_fingerprint()) + + admin_ttl: Final = self._admin_tools_list_ttl(server) + if admin_ttl is None: + return await self._list_tools_for_upstream_ttl(server, cache_key, fetch) + + async def fetch_with_admin_ttl() -> _UpstreamToolListing: + listing: Final = await fetch() + return listing.model_copy(update={"ttl_ms": admin_ttl * 1000}) if listing.is_whole() else listing + + if admin_ttl <= 0: + return await fetch_with_admin_ttl() + return await self._tool_discovery_cache.get(await cache_key(), fetch_with_admin_ttl, capped=False) + + async def _list_tools_for_upstream_ttl( + self, + server: MCPServer, + cache_key: Callable[[], Awaitable[_DiscoveryKey]], + fetch: Callable[[], Awaitable[_UpstreamToolListing]], + ) -> _UpstreamToolListing: + if self._tool_discovery_cache.holds(server.server_id): + return await self._tool_discovery_cache.get(await cache_key(), fetch) + invalidations_before: Final = self._tool_discovery_cache.invalidations + listing: Final = await fetch() + if not self._list_caching_enabled or listing.ttl_ms <= 0 or not listing.is_whole(): + return listing + key: Final = await cache_key() + if self._tool_discovery_cache.invalidations == invalidations_before: + self._tool_discovery_cache.keep(key, listing) + return listing + def clear_initialize_instructions(self) -> None: self._upstream_initialize_instructions_by_server_id.clear() self._upstream_initialize_instructions_probed_at.clear() @@ -4673,6 +4831,7 @@ class MCPServerManager: self._prompt_discovery_cache.invalidate(server_id) self._resource_discovery_cache.invalidate(server_id) self._template_discovery_cache.invalidate(server_id) + self._tool_discovery_cache.invalidate(server_id) def invalidate_server_definition_caches(self, server_id: str) -> None: from litellm.proxy.mcp_server.discoverable_endpoints import ( # noqa: PLC0415 # lazy: discoverable_endpoints lazily imports this module's manager singleton @@ -4774,6 +4933,7 @@ class MCPServerManager: subject_token: str | None, credential_fingerprint: str | None = None, raw_headers: Mapping[str, str] | None = None, + page_cursor: tuple[str | None] | None = None, ) -> _DiscoveryKey: identity: Final = ( user_api_key_auth.model_dump( @@ -4800,6 +4960,7 @@ class MCPServerManager: stdio_env, subject_token, credential_fingerprint, + page_cursor, ), sort_keys=True, separators=(",", ":"), @@ -4945,8 +5106,33 @@ class MCPServerManager: raw_headers: dict[str, str] | None = None, client_ip: str | None = None, ) -> list[Resource]: - if self._skip_blocked_stdio_listing(server, "resource"): + listing: Final = await self.list_resources_from_server( + server=server, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + extra_headers=extra_headers, + add_prefix=add_prefix, + raw_headers=raw_headers, + client_ip=client_ip, + ) + if isinstance(listing, ResourceListingFailed): + verbose_logger.warning("Failed to get resources from server %s: %s", server.name, listing.error) return [] + return list(listing) + + async def list_resources_from_server( + self, + server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | Mapping[str, str] | None = None, + extra_headers: Mapping[str, str] | None = None, + add_prefix: bool = True, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, + ) -> tuple[Resource, ...] | ResourceListingFailed: + if self._skip_blocked_stdio_listing(server, "resource"): + return () + auth_header: Final = dict(mcp_auth_header) if isinstance(mcp_auth_header, Mapping) else mcp_auth_header try: headers: Final = ( dict( @@ -4961,7 +5147,7 @@ class MCPServerManager: subject_token: Final = self._obo_subject_token(server, raw_headers, user_api_key_auth) client: Final = await self.create_mcp_client( server=server, - mcp_auth_header=mcp_auth_header, + mcp_auth_header=auth_header, extra_headers=headers, stdio_env=stdio_env, subject_token=subject_token, @@ -4973,7 +5159,7 @@ class MCPServerManager: key: Final = self._discovery_key( server, user_api_key_auth, - mcp_auth_header, + auth_header, headers, stdio_env, subject_token, @@ -4985,10 +5171,9 @@ class MCPServerManager: return await client.list_resources_result(raise_on_error=True) items: Final = await self._resource_discovery_cache.get(key, fetch) - return self._create_prefixed_resources(items.resources, server, add_prefix=add_prefix) + return tuple(self._create_prefixed_resources(items.resources, server, add_prefix=add_prefix)) except Exception as error: - verbose_logger.warning("Failed to get resources from server %s: %s", server.name, error) - return [] + return ResourceListingFailed(server_name=server.name, error=error) async def get_resource_templates_from_server( self, @@ -5644,7 +5829,7 @@ class MCPServerManager: self, client: MCPClient, server_name: str, - ) -> list[MCPTool]: + ) -> ListToolsResult: """ Fetch tools from MCP client with timeout and error handling. @@ -5668,17 +5853,17 @@ class MCPServerManager: server_name: Name of the server for logging Returns: - List of tools from the server + The tools from the server with the upstream freshness of the complete walk """ try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): - tools: Final = await retry_upstream_rate_limit( - lambda: client.list_tools(raise_on_error=True), + listing: Final = await retry_upstream_rate_limit( + lambda: client.list_tools_result(raise_on_error=True), server_name, deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, ) - verbose_logger.debug("Tools from %s: %s", server_name, tools) - return tools + verbose_logger.debug("Tools from %s: %s", server_name, listing.tools) + return listing except TimeoutError as e: verbose_logger.warning("Timeout while listing tools from %s", server_name) raise MCPServerListError(ServerListFault(tag="timeout"), server_name) from e @@ -6131,6 +6316,7 @@ class MCPServerManager: dict(catalog_auth_header) if isinstance(catalog_auth_header, Mapping) else catalog_auth_header ), record_listing=False, + use_tool_list_cache=False, ) selected: Final = tuple(tool for tool in upstream if tool.name == name) scan: Final = await scan_tool_descriptions( diff --git a/litellm/proxy/mcp_server/oauth_utils.py b/litellm/proxy/mcp_server/oauth_utils.py index 3171c476ede..c4fbaa7b087 100644 --- a/litellm/proxy/mcp_server/oauth_utils.py +++ b/litellm/proxy/mcp_server/oauth_utils.py @@ -132,18 +132,27 @@ def build_upstream_authorize_url(authorization_url: str, params: Mapping[str, st return urlunparse(parsed._replace(query=urlencode(merged))) +MCP_WILDCARD_SCOPE: Final = "mcp_all" + + +def asks_for_wildcard(scope: str | None) -> bool: + return MCP_WILDCARD_SCOPE in (scope or "").split() + + def grant_covers_scopes( granted: tuple[str, ...], requested: tuple[str, ...], server_scopes: Callable[[], frozenset[str]] ) -> bool: - """Whether a stored grant holds every requested scope. A grant stored as one comma-joined token, the form - Slack and GitHub reply with, counts as its parts only when every part is a requested or server scope.""" - if all(scope in granted for scope in requested): + """Whether a stored grant holds every requested scope. A non-empty grant holds the wildcard scope, which a + provider grants as granular scopes in its place. A grant stored as one comma-joined token, the form Slack and + GitHub reply with, counts as its parts only when every part is a requested or server scope.""" + held: Final = frozenset((*granted, MCP_WILDCARD_SCOPE)) if granted else frozenset[str]() + if all(scope in held for scope in requested): return True if len(granted) != 1 or "," not in granted[0]: return False parts: Final = frozenset(granted[0].split(",")) known: Final = frozenset(requested) | server_scopes() - return parts <= known and all(scope in granted or scope in parts for scope in requested) + return parts <= known and all(scope in held or scope in parts for scope in requested) _PUBLISHED_SCOPES: Final = TypeAdapter[tuple[str, ...] | None](tuple[str, ...] | None) diff --git a/litellm/proxy/mcp_server/operations.py b/litellm/proxy/mcp_server/operations.py index afa3a373f3c..6c9b8184e1a 100644 --- a/litellm/proxy/mcp_server/operations.py +++ b/litellm/proxy/mcp_server/operations.py @@ -112,6 +112,7 @@ from litellm.proxy.mcp_server.interactions import ( from litellm.proxy.mcp_server.legacy_callbacks import get_scope_response from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: F401 # legacy module exports MCPServerManager, + ResourceListingFailed, _caller_authorization_fans_out, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _client_forwarded_authorization_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _resolve_openapi_tool_auth, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -142,6 +143,7 @@ from litellm.proxy.mcp_server.result_conversion import ( handler_outcome, to_call_tool_result, ) +from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_blocked from litellm.proxy.mcp_server.tool_registry import ( global_mcp_tool_registry, ) @@ -262,6 +264,61 @@ class _MCPServerRateLimitAdmission: rejected_servers: tuple[tuple[MCPServer, ProxyRateLimitError], ...] +@dataclass(frozen=True, slots=True) +class _ResourceOwnerRateLimited: + error: ProxyRateLimitError + + +@dataclass(frozen=True, slots=True) +class _ResourceOwnerMissing: + pass + + +@dataclass(frozen=True, slots=True) +class _ResourceOwnerAmbiguous: + server_names: tuple[str, ...] + + +_ResourceOwnerFailure: TypeAlias = ( + _ResourceOwnerRateLimited | ResourceListingFailed | _ResourceOwnerMissing | _ResourceOwnerAmbiguous +) + + +def _withhold_fanned_out_caller_authorization( + server: MCPServer, extra_headers: dict[str, str] | None, scope_servers: Sequence[MCPServer] +) -> dict[str, str] | None: + """Drop the caller's request-wide Authorization from a gateway-selected read when another server in + scope also consumes it. ``_prepare_mcp_server_headers`` already withholds it from client-forwarded + servers, but legacy oauth2 delegates and OAuth pass-through servers still carry it, and an aggregate + read must not replay one caller token to servers the gateway picked.""" + if extra_headers and caller_authorization_fans_out(server, scope_servers): + return without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER) + return extra_headers + + +def _raise_resource_owner_failure(failure: _ResourceOwnerFailure, url: AnyUrl) -> NoReturn: + match failure: + case _ResourceOwnerRateLimited(error=error): + raise error + case ResourceListingFailed(server_name=server_name): + raise HTTPException( + status_code=502, + detail=f"Unable to list resources from MCP server '{server_name}'; resource owner is unknown.", + ) + case _ResourceOwnerMissing(): + raise HTTPException(status_code=404, detail=f"Resource '{url}' not found on any allowed MCP server.") + case _ResourceOwnerAmbiguous(server_names=server_names): + raise HTTPException( + status_code=400, + detail=( + f"Resource URI '{url}' is ambiguous across MCP servers: {', '.join(server_names)}. " + "Use /{server_name}/mcp or x-mcp-servers to select one server." + ), + ) + case _: + assert_never(failure) + + _mcp_server_admission_memo: Final[ContextVar[dict[str, asyncio.Task[ProxyRateLimitError | None]] | None]] = ContextVar( "mcp_server_admission_memo", default=None ) @@ -1000,7 +1057,7 @@ def _prepare_mcp_server_headers( oauth2_headers: dict[str, str] | None, raw_headers: dict[str, str] | None, user_api_key_auth: UserAPIKeyAuth | None = None, - scope_servers: list[MCPServer] | None = None, + scope_servers: Sequence[MCPServer] | None = None, ) -> tuple[dict[str, str] | str | None, dict[str, str] | None]: """Build auth and extra headers for a server. @@ -1218,6 +1275,7 @@ async def find_mcp_proxy_tool( context=context, allowed_mcp_servers=allowed_mcp_servers, prefetched_oauth_creds=prefetched_oauth_creds, + use_tool_list_cache=False, ) return next((tool for tool in page.tools if mcp_proxy_tool_id(tool) == tool_id), None) @@ -1255,6 +1313,7 @@ async def _get_tools_from_mcp_servers( record_listing: bool = False, request_trace_id: str | None = None, enforce_rate_limits: bool = True, + use_tool_list_cache: bool = True, ) -> AggregateToolListing: """ Helper method to fetch tools from MCP servers based on server filtering criteria. @@ -1384,6 +1443,7 @@ async def _get_tools_from_mcp_servers( allowed_mcp_servers=allowed_mcp_servers, prefetched_oauth_creds=_prefetched_oauth_creds, record_listing=record_listing, + use_tool_list_cache=use_tool_list_cache, ) return page.tools, outcome @@ -1419,6 +1479,7 @@ async def _get_tools_from_mcp_servers( _prefetched_oauth_creds, record_listing=record_listing, enforce_rate_limits=enforce_rate_limits, + use_tool_list_cache=use_tool_list_cache, ) all_tools: Final = aggregated.tools server_outcomes: Final = aggregated.outcomes @@ -1710,6 +1771,7 @@ async def _list_mcp_tools( protocol_version: str | None = None, *, record_listing: bool = False, + use_tool_list_cache: bool = True, ) -> AggregateToolListing: """ List all available MCP tools. @@ -1745,6 +1807,7 @@ async def _list_mcp_tools( record_listing=record_listing, params=params, protocol_version=protocol_version, + use_tool_list_cache=use_tool_list_cache, ) verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools)) return listing @@ -2073,6 +2136,7 @@ async def _list_tools_before_first_call( client_ip=client_ip, record_listing=False, enforce_rate_limits=False, + use_tool_list_cache=False, ) except Exception as e: # noqa: BLE001 # best effort: resolution below answers as it did before verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e) @@ -3124,13 +3188,58 @@ async def mcp_read_resource( detail="User not allowed to read this resource.", ) - if len(allowed_mcp_servers) != 1: - raise HTTPException( - status_code=400, - detail=("Multiple MCP servers configured; read_resource currently supports exactly one allowed server."), - ) + candidates: Final = tuple(server for server in allowed_mcp_servers if not is_mcp_stdio_blocked(server.transport)) - server: Final = allowed_mcp_servers[0] + async def owner(candidate: MCPServer) -> MCPServer | ResourceListingFailed | None: + candidate_auth, candidate_headers = _prepare_mcp_server_headers( + server=candidate, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + scope_servers=candidates, + ) + listing: Final = await global_mcp_server_manager.list_resources_from_server( + server=candidate, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=candidate_auth, + extra_headers=_withhold_fanned_out_caller_authorization(candidate, candidate_headers, candidates), + add_prefix=False, + raw_headers=raw_headers, + client_ip=client_ip, + ) + if isinstance(listing, ResourceListingFailed): + return listing + return candidate if any(str(resource.uri) == str(url) for resource in listing) else None + + async def resolve_owner() -> MCPServer | _ResourceOwnerFailure: + admission: Final = await _admit_mcp_servers(candidates, user_api_key_auth) + if admission.rejected_servers: + return _ResourceOwnerRateLimited(error=admission.rejected_servers[0][1]) + lookups: Final = tuple(asyncio.create_task(owner(candidate)) for candidate in candidates) + try: + for lookup in asyncio.as_completed(lookups): + if isinstance(failure := await lookup, ResourceListingFailed): + return failure + finally: + for lookup in lookups: + lookup.cancel() + await asyncio.gather(*lookups, return_exceptions=True) + results: Final = tuple(lookup.result() for lookup in lookups) + owners: Final = tuple(result for result in results if isinstance(result, MCPServer)) + if len(owners) == 1: + return owners[0] + if not owners: + return _ResourceOwnerMissing() + return _ResourceOwnerAmbiguous(server_names=tuple(owner.name for owner in owners)) + + if len(allowed_mcp_servers) == 1: + await _enforce_mcp_server_rate_limit(user_api_key_auth, allowed_mcp_servers[0]) + resolved: Final = allowed_mcp_servers[0] if len(allowed_mcp_servers) == 1 else await resolve_owner() + if isinstance(resolved, _ResourceOwnerFailure): + _raise_resource_owner_failure(resolved, url) + server: Final = resolved server_auth_header, extra_headers = _prepare_mcp_server_headers( server=server, @@ -3139,15 +3248,15 @@ async def mcp_read_resource( oauth2_headers=oauth2_headers, raw_headers=raw_headers, user_api_key_auth=user_api_key_auth, + scope_servers=candidates, ) - await _enforce_mcp_server_rate_limit(user_api_key_auth, server) return await global_mcp_server_manager.read_resource_from_server( server=server, user_api_key_auth=user_api_key_auth, url=url, mcp_auth_header=server_auth_header, - extra_headers=extra_headers, + extra_headers=_withhold_fanned_out_caller_authorization(server, extra_headers, candidates), raw_headers=raw_headers, client_ip=client_ip, input_responses=input_responses, @@ -3683,21 +3792,27 @@ async def _execute_read_resource( _client_ip, ) = context.legacy_auth() - read_resource_result: Final = await mcp_read_resource( - url=params.uri, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - client_ip=_client_ip, - input_responses=params.input_responses, - request_state=params.request_state, - allow_input_required=context.wire_compat is WireCompat.MODERN, - ) + from mcp.shared.exceptions import MCPError + from mcp.types import INVALID_REQUEST - return read_resource_result + try: + return await mcp_read_resource( + url=params.uri, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + input_responses=params.input_responses, + request_state=params.request_state, + allow_input_required=context.wire_compat is WireCompat.MODERN, + ) + except MCPError: + raise + except HTTPException as error: + raise MCPError(code=INVALID_REQUEST, message=_http_detail_message(error.detail)) from error def _reject_mcp_proxy_operation() -> NoReturn: diff --git a/litellm/proxy/mcp_server/rest_endpoints.py b/litellm/proxy/mcp_server/rest_endpoints.py index 165fc23d754..a5cba312d64 100644 --- a/litellm/proxy/mcp_server/rest_endpoints.py +++ b/litellm/proxy/mcp_server/rest_endpoints.py @@ -748,6 +748,7 @@ if MCP_AVAILABLE: proxy_logging_obj: "ProxyLogging | None", *, record_listing: bool, + use_tool_list_cache: bool, ) -> list[MCPTool]: return list( await global_mcp_server_manager.get_tools_from_server( @@ -760,6 +761,7 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, proxy_logging_obj=proxy_logging_obj, record_listing=record_listing, + use_tool_list_cache=use_tool_list_cache, ) ) @@ -793,6 +795,7 @@ if MCP_AVAILABLE: client_ip, proxy_logging_obj, record_listing=False, + use_tool_list_cache=apply_tool_filters, ) server_filtered: Final = filter_tools_by_allowed_tools(tools, server) if apply_tool_filters else tools @@ -839,6 +842,7 @@ if MCP_AVAILABLE: IPAddressUtils.get_mcp_client_ip(request), None, record_listing=False, + use_tool_list_cache=False, ) scan: Final = await scan_tool_descriptions( apply_description_overrides(upstream, server), diff --git a/litellm/proxy/mcp_server/tool_search.py b/litellm/proxy/mcp_server/tool_search.py index e65fcf61f49..b90e5c79889 100644 --- a/litellm/proxy/mcp_server/tool_search.py +++ b/litellm/proxy/mcp_server/tool_search.py @@ -588,6 +588,7 @@ async def handle_mcp_tool_search( mcp_server_auth_headers=mcp_server_auth_headers, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + use_tool_list_cache=False, ) results: Final = await search_mcp_tools(query, mcp_listing.tools, top_k, settings, ranker) if isinstance(results, EmbeddingFailed): @@ -628,6 +629,7 @@ async def handle_mcp_proxy_tool( oauth2_headers=oauth2_headers, raw_headers=raw_headers, mcp_proxy_mode=True, + use_tool_list_cache=False, ) llm_router: Final = proxy_server.llm_router proxy_logging_obj: Final = proxy_server.proxy_logging_obj diff --git a/litellm/proxy/middleware/redis_request_batch_middleware.py b/litellm/proxy/middleware/redis_request_batch_middleware.py index bfb5f79a174..e1db6291d68 100644 --- a/litellm/proxy/middleware/redis_request_batch_middleware.py +++ b/litellm/proxy/middleware/redis_request_batch_middleware.py @@ -22,4 +22,4 @@ class RedisRequestBatchMiddleware: try: await self.app(scope, receive, send) finally: - await batches.flush_all() + await batches.flush_pre_call() diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index a81fced9021..222edfb7612 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -3,7 +3,7 @@ import itertools import logging import mimetypes import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime, timezone from types import MappingProxyType @@ -17,6 +17,7 @@ from typing import ( get_args, runtime_checkable, ) +from urllib.parse import unquote, urlparse from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -439,6 +440,50 @@ def get_original_file_id(encoded_id: str) -> str: return encoded_id +_GEMINI_FILES_HOST: Final = "generativelanguage.googleapis.com" + + +def canonical_provider_file_id(file_id: str) -> str: + """The one spelling of a provider file id that ownership rows store and look up, so ``files/abc``, + ``files%2Fabc``, ``abc/`` and the Gemini file URL all resolve to the bare ``abc``. Every other URL or URI + (``s3://``, ``gs://``, a storage blob URL) is returned unchanged, so an upload whose name happens to contain + ``files/`` never indexes someone else's ````.""" + decoded: Final = unquote(file_id).strip() + is_url: Final = "://" in decoded + if is_url and urlparse(decoded).hostname != _GEMINI_FILES_HOST: + return file_id + path: Final = urlparse(decoded).path if is_url else decoded + files_index: Final = path.find("files/") + bare: Final = (path[files_index:] if files_index != -1 else path).strip("/").removeprefix("files/").strip("/") + return bare or decoded + + +_GEMINI_FILES_URL: Final = "https://generativelanguage.googleapis.com/v1beta/files/" + + +def _gemini_files_tail(file_id: str) -> str: + """The id Gemini resolves ``file_id`` to, whatever scheme or case wraps it: the text after its ``files/``.""" + files_index: Final = file_id.find("files/") + return file_id[files_index:].strip("/").removeprefix("files/").strip("/") if files_index != -1 else "" + + +def _iter_stored_forms(bare_ids: Sequence[str]) -> Iterator[str]: + for bare_id in bare_ids: + yield bare_id + yield f"files/{bare_id}" + yield f"{_GEMINI_FILES_URL}{bare_id}" + + +def provider_file_id_spellings(file_id: str) -> tuple[str, ...]: + """Every spelling an ownership row may hold for ``file_id``: as sent, unwrapped from a model-encoded id and + canonicalized, reduced the way Gemini reduces it, plus the ``files/`` and Gemini URL forms that upload rows + stored before ids were canonicalized.""" + original: Final = get_original_file_id(file_id) + reductions: Final = (canonical_provider_file_id(original), _gemini_files_tail(unquote(original))) + bare_ids: Final = tuple(dict.fromkeys(bare for bare in reductions if bare)) + return tuple(dict.fromkeys((file_id, *_iter_stored_forms(bare_ids)))) + + def is_model_embedded_id(file_id: str) -> bool: """ Check if a file/batch ID has model routing information embedded. diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py index ec189b83195..d7dd2fb2921 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py @@ -13,7 +13,6 @@ from typing import IO, ClassVar, Final, Protocol, TypeAlias from urllib.parse import quote import httpx -import soundfile from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -522,6 +521,8 @@ async def write_media_within_limit(response: httpx.Response, media_file: IO[byte def media_file_seconds(path: Path) -> float | None: + import soundfile + try: with soundfile.SoundFile(str(path)) as audio: frames: Final = len(audio) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 0fcaccd340e..a6f2fe1c309 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -125,6 +125,11 @@ from litellm.proxy.common_request_processing import ( resolve_litellm_call_id, ) from litellm.proxy.common_utils.callback_utils import strip_callback_config +from litellm.proxy.common_utils.customer_usage import ( + CUSTOMER_USAGE_METADATA_KEY, + ResolvedCustomerUsage, + customer_usage_fields, +) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -149,8 +154,10 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy _strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs + check_customer_request_model_access, get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above redact_credential_headers, + resolve_customer_request_usage, strip_client_guardrail_information, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError @@ -716,6 +723,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): logging_obj: LiteLLMLoggingObj, _parsed_body: dict | None = None, litellm_call_id: str | None = None, + customer_usage: ResolvedCustomerUsage | None = None, ) -> dict: """ Filter out litellm params from the request body @@ -783,6 +791,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): _metadata[MODEL_ACCESS_GROUP_METADATA_KEY] = user_api_key_dict.matched_model_access_groups _metadata["user_api_key_metadata"] = _metadata["user_api_key_auth_metadata"] _metadata["user_api_key_team_metadata"] = strip_callback_config(user_api_key_dict.team_metadata) + if customer_usage is not None: + _metadata[CUSTOMER_USAGE_METADATA_KEY] = customer_usage.attribution.model_dump_json() # The per-model budget counters are keyed off these. get_sanitized_user_information_from_key # returns StandardLoggingUserAPIKeyMetadata, which carries no budget field, so without this # the post-call increment finds nothing and every passthrough request goes untracked and @@ -1285,6 +1295,49 @@ def _json_object(value: object) -> Mapping[str, object]: return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json +def _check_customer_passthrough_model_access( + request: Request, body: Mapping[str, object], auth: UserAPIKeyAuth, llm_router: litellm.Router | None +) -> None: + if not auth.customer_model_scopes: + return + model: Final = get_model_from_request( + request_data=dict(body), + route=get_request_route(request), + request_headers=request.headers, + request_query_params=request.query_params, + llm_router=llm_router, + request=request, + team_id=auth.team_id, + ) + check_customer_request_model_access({**body, "model": model}, auth, llm_router) + + +@dataclass(frozen=True, slots=True) +class _CustomerPassthroughAuth: + auth: UserAPIKeyAuth + usage: ResolvedCustomerUsage | None + + +async def _customer_passthrough_request_auth( + request: Request, body: Mapping[str, object], auth: UserAPIKeyAuth +) -> _CustomerPassthroughAuth: + from litellm.proxy.proxy_server import general_settings_view, llm_router + + if request_dispatched_to_pass_through_endpoint(request): + return _CustomerPassthroughAuth(auth, None) + usage: Final = await resolve_customer_request_usage( + { + **customer_usage_fields(body.get("litellm_metadata")), + **customer_usage_fields(body.get("metadata")), + }, + auth, + ledger_enabled=general_settings_view().get("use_spend_logs_ledger", True) is not False, + ) + request_auth: Final = auth.with_customer_model_scopes(usage.model_scopes) if usage is not None else auth + _check_customer_passthrough_model_access(request, body, request_auth, llm_router) + return _CustomerPassthroughAuth(request_auth, usage) + + def _anthropic_user_id_to_forward( body: Mapping[str, object], url: str, @@ -1321,6 +1374,27 @@ def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | Non return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)}) +@dataclass(frozen=True, slots=True) +class _PassthroughClientBody: + body: Mapping[str, object] + eu_jev_body: Mapping[str, object] | None + + +def _prepare_passthrough_client_body( + body: Mapping[str, object], auth: UserAPIKeyAuth, custom_llm_provider: str | None +) -> _PassthroughClientBody: + client_body: Final = dict(body) + strip_client_guardrail_information(client_body) + if not _key_or_team_allows_client_pricing_override(auth): + _strip_client_pricing_overrides(client_body) + eu_jev_body: Final = _eu_jev_client_body(custom_llm_provider, client_body) + scoped_body: Final = {**client_body, **_eu_jev_scoped_model_override(eu_jev_body)} + if custom_llm_provider in ("laya", "bespoke"): + checkpoint: Final = validate_oss_request(custom_llm_provider, scoped_body) + scoped_body["model"] = f"{custom_llm_provider}/{checkpoint}" + return _PassthroughClientBody(scoped_body, eu_jev_body) + + async def pass_through_request( request: Request, target: str, @@ -1369,7 +1443,7 @@ async def pass_through_request( from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( PassthroughGuardrailHandler, ) - from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj + from litellm.proxy.proxy_server import llm_router, proxy_config, proxy_logging_obj ######################################################### # Initialize variables @@ -1441,20 +1515,13 @@ async def pass_through_request( _parsed_body, ) - client_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) - strip_client_guardrail_information(client_body) - if not _key_or_team_allows_client_pricing_override(user_api_key_dict): - _strip_client_pricing_overrides(client_body) - eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body) - scoped_client_body: Final = { - **client_body, - **_eu_jev_scoped_model_override(eu_jev_client_body), - } - _parsed_body = scoped_client_body - if custom_llm_provider in ("laya", "bespoke"): - decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) - checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) - _parsed_body["model"] = f"{custom_llm_provider}/{checkpoint}" + client_body: Final = _prepare_passthrough_client_body( + TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), user_api_key_dict, custom_llm_provider + ) + _parsed_body = dict(client_body.body) + + customer_request: Final = await _customer_passthrough_request_auth(request, client_body.body, user_api_key_dict) + user_api_key_dict = customer_request.auth # rebind-ok: customer scopes stay request-local ### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ### # Passthrough endpoints are opt-in only for guardrails @@ -1521,6 +1588,9 @@ async def pass_through_request( call_type="pass_through_endpoint", endpoint_type=endpoint_type, ) + _check_customer_passthrough_model_access( + request, TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), user_api_key_dict, llm_router + ) if custom_llm_provider in ("laya", "bespoke"): hook_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) hook_model: Final = hook_body.get("model") @@ -1534,10 +1604,10 @@ async def pass_through_request( ) _ = validate_oss_request(custom_llm_provider, decision_body) _parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body) - if eu_jev_client_body is not None: + if client_body.eu_jev_body is not None: _parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage eu_jev_upstream_body( - TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body + TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), client_body.eu_jev_body ) ) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) @@ -1563,6 +1633,7 @@ async def pass_through_request( litellm_call_id=litellm_call_id, request=request, logging_obj=logging_obj, + customer_usage=customer_request.usage, ) # Store custom_llm_provider in kwargs and logging object if provided @@ -4753,7 +4824,7 @@ async def update_pass_through_endpoints( # Find the index for updating the list endpoint_index = None for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint + _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) if isinstance(endpoint, dict) else endpoint if _endpoint.id == endpoint_id: endpoint_index = idx break @@ -4986,7 +5057,7 @@ async def delete_pass_through_endpoints( # Find the index for deleting from the list endpoint_index = None for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint + _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) if isinstance(endpoint, dict) else endpoint if _endpoint.id == endpoint_id: endpoint_index = idx break @@ -5034,7 +5105,7 @@ def _find_endpoint_by_id( for endpoint in endpoints_data: _endpoint: PassThroughGenericEndpoint | None = None if isinstance(endpoint, dict): - _endpoint = PassThroughGenericEndpoint(**endpoint) + _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) elif isinstance(endpoint, PassThroughGenericEndpoint): _endpoint = endpoint diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index b9a60d9e37f..5d2477f77b4 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -20,7 +20,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, ModifyResponseException, ) -from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger import CustomLogger, subclass_overrides_hook from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, @@ -699,7 +699,10 @@ class PipelineExecutor: @staticmethod def supports_unified_execution(callback: CustomGuardrail) -> bool: """Whether this guardrail runs through the unified apply_guardrail path.""" - return "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks + return ( + subclass_overrides_hook(type(callback), "apply_guardrail", CustomGuardrail) + and not callback.use_native_lifecycle_hooks + ) @staticmethod def supports_streaming_execution(callback: CustomGuardrail) -> bool: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bd85089abc5..b6378d72297 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -46,6 +46,7 @@ from typing import ( Final, Literal, NamedTuple, + NoReturn, Optional, Protocol, TypeAlias, @@ -1001,6 +1002,8 @@ from litellm.proxy.spend_tracking.background_interaction_settlement import ( install_background_interaction_settlement, ) from litellm.proxy.spend_tracking.budget_reservation import ( + BudgetCounter, + customer_window_counters, get_budget_window_start, release_unbound_budget_reservation, stamp_budget_reservation_actual_cost, @@ -1123,6 +1126,7 @@ from litellm.types.llms.openai import ( ResponseInProgressEvent, ResponsesAPIResponse, ) +from litellm.types.proxy.auth.customer_hierarchy import CustomerHierarchyIds, CustomerScopeAuth from litellm.types.proxy.carried_budget_state import EntitySpendCounterKeys from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( @@ -1596,7 +1600,7 @@ async def _connect_to_count_stored_values(resources: AsyncExitStack) -> Supports if prisma_client is None: resources.push_async_callback(client.aclose) await client.connect() - return client.writer_db + return client.writer_db # pyright: ignore[reportReturnType] # PrismaWrapper forwards query_raw via __getattr__ async def _own_database(resources: AsyncExitStack, client: PrismaClient | None) -> database_runtime.Database | None: @@ -2994,6 +2998,7 @@ _ROUTE_MAX_REQUEST_SIZE_MB: Final[Mapping[tuple[str, str], float]] = MappingProx ("POST", "/claude-code/plugins/upload"): 11, ("POST", "/skill-hub/skills/upload"): 11, ("PUT", "/skill-hub/skills/{name}/upload"): 11, + ("PUT", "/skill-hub/submissions/{submission_id}/upload"): 11, } ) @@ -3418,7 +3423,7 @@ def fail_closed_budget_enforcement_enabled() -> bool: return _fail_closed_budget_enforcement() -def _raise_budget_unverifiable(counter_key: str) -> None: +def _raise_budget_unverifiable(counter_key: str) -> NoReturn: verbose_proxy_logger.warning( "fail_closed_budget_enforcement: rejecting request — spend for %s could " "not be verified against Redis or the database", @@ -3430,7 +3435,7 @@ def _raise_budget_unverifiable(counter_key: str) -> None: "error": ( "Budget enforcement unavailable: current spend could not be " "verified against Redis or the database, and " - "fail_closed_budget_enforcement is enabled, so the request was " + "an authoritative budget balance is required, so the request was " "rejected to avoid exceeding the configured budget. Retry shortly." ) }, @@ -3446,6 +3451,8 @@ async def get_current_spend( window_duration: str | None = None, window_start: datetime | None = None, fallback_authoritative: bool = False, + *, + require_authoritative_window: bool = False, ) -> float: """ Read current spend from the cross-pod spend counter. @@ -3476,7 +3483,16 @@ async def get_current_spend( and cached in-process for a few seconds, so a persistently stale counter drives at most one read per counter per window rather than one per request. """ - current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend) + current, verified = ( + ( + await _read_authoritative_window_spend( + counter_key, window_entity_type, window_entity_id, window_duration, window_start + ), + True, + ) + if require_authoritative_window + else await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend) + ) if fallback_authoritative: verified = True @@ -3644,6 +3660,30 @@ async def read_spend_counter_cache_value(counter_key: str) -> tuple[float | None return (float(value) if value is not None else None), authoritative +async def _read_authoritative_window_spend( + counter_key: str, + entity_type: str | None, + entity_id: str | None, + window_duration: str | None, + window_start: datetime | None, +) -> float: + if entity_type is None or entity_id is None or window_start is None: + raise ValueError("Authoritative window reads require the window entity and start") + window_spend: Final = await SpendCounterReseed.coalesced_window( + prisma_client=prisma_client, + spend_counter_store=spend_counter_store, + counter_key=counter_key, + entity_type=entity_type, + entity_id=entity_id, + window_duration=window_duration, + window_start=window_start, + require_authoritative=True, + ) + if window_spend is None: + _raise_budget_unverifiable(counter_key) + return window_spend + + async def _read_spend_counter_estimate(counter_key: str, fallback_spend: float) -> tuple[float, bool]: """Return (spend, authoritative). ``authoritative`` is True when the value came from Redis or a fresh DB read (cross-pod truth), False when it came @@ -3680,6 +3720,7 @@ async def increment_spend_counters( project_id: str | None = None, committed_spend_scope: EntitySpendCounterKeys | None = None, windows_from_ledger: bool = False, + customer_counters: Sequence[BudgetCounter] = (), ): """ Atomically increment spend counters for budget enforcement. @@ -3702,7 +3743,8 @@ async def increment_spend_counters( tags=tags, model_access_groups=model_access_groups, project_id=project_id, - ), + ) + | frozenset(counter.counter_key for counter in customer_counters), ): await _increment_spend_counters_batched( token=token, @@ -3718,6 +3760,7 @@ async def increment_spend_counters( project_id=project_id, committed_spend_scope=committed_spend_scope, windows_from_ledger=windows_from_ledger, + customer_counters=customer_counters, ) @@ -3747,7 +3790,44 @@ def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bo user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key) -async def accounting_window_keys(token: str | None, team_id: str | None) -> tuple[Mapping[str, object], ...]: +async def customer_accounting_counters( + ids: CustomerHierarchyIds | None, + admitted_scopes: tuple[CustomerScopeAuth, ...] | None = None, +) -> tuple[BudgetCounter, ...]: + if ids is None: + return () + from itertools import chain + + from litellm.proxy.auth.customer_hierarchy_auth import get_customer_scopes_for_accounting + + try: + scopes: Final = ( + admitted_scopes + if admitted_scopes is not None + else ( + await get_customer_scopes_for_accounting(ids, prisma_client, user_api_key_cache) + if prisma_client is not None + else () + ) + ) + except Exception: + logging.getLogger(verbose_proxy_logger.name).exception( + "Unable to load customer budget windows for spend accounting" + ) + return () + now: Final = datetime.now(timezone.utc) + return tuple( + chain.from_iterable( + customer_window_counters(scope.key, scope.entity_id, scope.budget_limits, now=now) for scope in scopes + ) + ) + + +async def accounting_window_keys( + token: str | None, + team_id: str | None, + customer_counters: Sequence[BudgetCounter] = (), +) -> tuple[Mapping[str, object], ...]: scopes: Final = tuple( (entity_type, entity_id, cache_key) for entity_type, entity_id, cache_key in ( @@ -3764,7 +3844,16 @@ async def accounting_window_keys(token: str | None, team_id: str | None) -> tupl ) from itertools import chain - return tuple(chain.from_iterable(snapshots)) + return tuple(chain.from_iterable(snapshots)) + tuple( + { + "entity_type": counter.entity_type, + "entity_id": counter.entity_id, + "window_duration": counter.window_duration, + "window_start": counter.window_start.isoformat(), + } + for counter in customer_counters + if counter.window_start is not None + ) async def _accounting_scope_windows( @@ -3834,6 +3923,7 @@ async def _increment_spend_counters_batched( project_id: str | None = None, committed_spend_scope: EntitySpendCounterKeys | None = None, windows_from_ledger: bool = False, + customer_counters: Sequence[BudgetCounter] = (), ): """Runs inside one spend counter batch: the reservation reconcile and the warm checks share a single MGET, and the reconcile adjustments go out in the same INCRBYFLOAT pipeline as the counter increments.""" @@ -3858,25 +3948,22 @@ async def _increment_spend_counters_batched( cost: Final[float] = response_cost - async def _window_increment( - entity_type: Litellm_EntityType, entity_id: str, window: object + async def _resolved_window_increment( + entity_type: Litellm_EntityType, + entity_id: str, + window_counter: str, + duration: str, + window_start: datetime | None, + window_reset_at: datetime | str | None, ) -> PendingSpendIncrement | None: - duration: Final = ( - window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) - ) - window_reset_at: Final = ( - window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) - ) - window_counter: Final = ( - key_window_counter(entity_id, str(duration)) - if entity_type is Litellm_EntityType.KEY - else team_window_counter(entity_id, str(duration)) - ) - window_start: Final = get_budget_window_start(window) pending_window: Final = ( await _prepare_window_spend_counter_increment( counter_key=window_counter, - entity_type=entity_type.value.capitalize(), + entity_type=( + entity_type.value.capitalize() + if entity_type in (Litellm_EntityType.KEY, Litellm_EntityType.TEAM) + else entity_type.value + ), entity_id=entity_id, window_duration=duration, window_start=window_start, @@ -3897,6 +3984,42 @@ async def _increment_spend_counters_batched( ) return pending_window + async def _window_increment( + entity_type: Litellm_EntityType, entity_id: str, window: object + ) -> PendingSpendIncrement | None: + duration: Final = ( + window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None) + ) + window_reset_at: Final = ( + window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None) + ) + window_counter: Final = ( + key_window_counter(entity_id, str(duration)) + if entity_type is Litellm_EntityType.KEY + else team_window_counter(entity_id, str(duration)) + ) + return await _resolved_window_increment( + entity_type, entity_id, window_counter, duration, get_budget_window_start(window), window_reset_at + ) + + async def _customer_windows() -> tuple[PendingSpendIncrement | BaseException, ...]: + pending: Final = await asyncio.gather( + *( + _resolved_window_increment( + Litellm_EntityType(counter.entity_type), + counter.entity_id, + counter.counter_key, + counter.window_duration, + counter.window_start, + counter.reset_at, + ) + for counter in customer_counters + if counter.window_duration is not None + ), + return_exceptions=True, + ) + return tuple(item for item in pending if item is not None) + async def _window_increments( entity_type: Litellm_EntityType, entity_id: str, budget_limits: object ) -> tuple[PendingSpendIncrement | BaseException, ...]: @@ -3996,6 +4119,7 @@ async def _increment_spend_counters_batched( scope_coros: Final = tuple( coro for coro in ( + _customer_windows() if customer_counters else None, _key_scope(token) if token is not None else None, _team_scope(team_id) if team_id is not None else None, _team_member_scope(user_id, team_id) if user_id is not None and team_id is not None else None, @@ -5858,6 +5982,16 @@ def _reload_settings_store(section: Section, store: SettingsStore, section_confi store["pass_through_endpoints"] = serving_pass_throughs +def _clear_db_settings_beside_pass_through_endpoints(section: Section, store: SettingsStore) -> None: + stored_pass_throughs: Final = store.db_value("pass_through_endpoints") + store.apply_db_row( + section, + MappingProxyType({"pass_through_endpoints": stored_pass_throughs}) + if isinstance(stored_pass_throughs, list) + else _EMPTY_SETTINGS_MAPPING, + ) + + def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool: return row is None or row.param_value is None or isinstance(row.param_value, Mapping) @@ -6708,7 +6842,7 @@ class ProxyConfig: ) else: for section, store in self._settings_stores.items(): - store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING) + _clear_db_settings_beside_pass_through_endpoints(section, store) config = self._config_with_resolved_settings(config) @@ -7081,7 +7215,7 @@ class ProxyConfig: elif key == "priority_reservation_settings": from litellm.types.utils import PriorityReservationSettings - litellm.priority_reservation_settings = PriorityReservationSettings(**value) + litellm.priority_reservation_settings = PriorityReservationSettings.model_validate(value) elif key == "callbacks": initialize_callbacks_on_proxy( value=value, @@ -11938,6 +12072,35 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) + async def send_performance_alert_summaries() -> None: + from litellm.integrations.performance_alert_notifications import PerformanceAlertNotifier + from litellm.repositories.alert_repository import AlertRepository + + slack: Final = proxy_logging_obj.slack_alerting_instance + email_connected: Final = bool(os.getenv("SMTP_HOST") and os.getenv("SMTP_SENDER_EMAIL")) + if not email_connected and not slack.performance_alert_webhooks(): + return + if _current_general_settings().get("performance_alerts_enabled", True) is not True: + return + if not ( + await PerformanceAlertRepository(prisma_client, enterprise=premium_user is True).monitoring() + ).enabled: + return + await PerformanceAlertNotifier( + ConfigRepository(prisma_client, use_writer=True), AlertRepository(prisma_client) + ).run(slack) + + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler does not provide typed job methods + send_performance_alert_summaries, + "interval", + minutes=5, + next_run_time=datetime.now(timezone.utc), + id="performance_alert_summaries", + replace_existing=True, + max_instances=1, + coalesce=True, + ) + ### PTU DAILY ROLLUP ### from litellm.proxy.spend_tracking.ptu_feature_flag import ( is_ptu_cost_attribution_enabled, @@ -19934,6 +20097,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str: response: Final = await generate_key_helper_fn( llm_router=llm_router, request_type="key", + actor=None, **{ "user_role": user_obj.user_role, "duration": LITELLM_UI_SESSION_DURATION, diff --git a/litellm/proxy/public_endpoints/model_hub_rows.py b/litellm/proxy/public_endpoints/model_hub_rows.py new file mode 100644 index 00000000000..9d1d48ad1b8 --- /dev/null +++ b/litellm/proxy/public_endpoints/model_hub_rows.py @@ -0,0 +1,86 @@ +"""The rows the public Model Hub lists: published model groups and opted-in pass-through endpoints.""" + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.proxy._types import PassThroughGenericEndpoint +from litellm.types.proxy.management_endpoints.model_management_endpoints import ModelGroupInfoProxy + +PASS_THROUGH_MODE: Final = "passthrough" +_SERVING_ENTRY: Final = TypeAdapter(dict[str, object]) + + +@dataclass(frozen=True, slots=True) +class ModelGroupRow: + info: ModelGroupInfoProxy + + +@dataclass(frozen=True, slots=True) +class PassThroughRow: + info: ModelGroupInfoProxy + + +HubRow: TypeAlias = ModelGroupRow | PassThroughRow + + +@dataclass(frozen=True, slots=True) +class NoModelsConfigured: + """Neither a router nor a published pass-through, so the hub has nothing to list.""" + + +def pass_through_row(endpoint: PassThroughGenericEndpoint) -> PassThroughRow: + return PassThroughRow( + info=ModelGroupInfoProxy( + model_group=endpoint.display_name or endpoint.path, + providers=[], + mode=PASS_THROUGH_MODE, + is_public_model_group=True, + pass_through_path=endpoint.path, + pass_through_methods=tuple(endpoint.methods) if endpoint.methods is not None else None, + ) + ) + + +def _serving_pass_through_endpoints() -> Iterator[PassThroughGenericEndpoint]: + """The endpoints this worker serves, from the in-memory general settings the config sync keeps current.""" + from litellm.proxy.proxy_server import proxy_config + + configured: Final = proxy_config.settings.get("pass_through_endpoints") + for entry in configured if isinstance(configured, list) else (): + try: + yield PassThroughGenericEndpoint.model_validate(_SERVING_ENTRY.validate_python(entry)) + except ValidationError: + continue + + +def _published_model_group_rows() -> tuple[ModelGroupRow, ...] | None: + """None when there is no router, which differs from a router that publishes nothing.""" + from litellm.proxy.proxy_server import ( + get_model_group_info, + llm_router, + ) + + if llm_router is None: + return None + if litellm.public_model_groups is None: + return () + return tuple( + ModelGroupRow(info=info) + for info in get_model_group_info( + llm_router=llm_router, all_models_str=litellm.public_model_groups, model_group=None + ) + ) + + +def published_hub_rows() -> tuple[HubRow, ...] | NoModelsConfigured: + pass_through_rows: Final = tuple( + pass_through_row(endpoint) for endpoint in _serving_pass_through_endpoints() if endpoint.show_in_model_hub + ) + model_group_rows: Final = _published_model_group_rows() + if model_group_rows is None and not pass_through_rows: + return NoModelsConfigured() + return (*(model_group_rows or ()), *pass_through_rows) diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 5afc3908820..ecb6df00da0 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3815,7 +3815,7 @@ "default_value": null } ], - "default_model_placeholder": "soniox/stt-async-v5" + "default_model_placeholder": "soniox/stt-async-v6" }, { "provider": "StrandsDecider", diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index cb1e51d0deb..48898afd8fa 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -5,6 +5,7 @@ import re from collections.abc import Awaitable, Callable, Mapping, Sequence from functools import cache from importlib.resources import files +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, HTTPException, Request @@ -16,7 +17,6 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_blog_posts import ( BlogPostsResponse, GetBlogPosts, - get_blog_posts, ) from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.offline_mode import is_offline_mode @@ -25,6 +25,13 @@ from litellm.proxy._types import ( ) from litellm.proxy.db.health_check_latest import latest_by_model_name from litellm.proxy.deps.repositories import PluginUploadRepositoryDep +from litellm.proxy.public_endpoints.model_hub_rows import ( + HubRow, + ModelGroupRow, + NoModelsConfigured, + PassThroughRow, + published_hub_rows, +) from litellm.proxy.utils import get_custom_url from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.repositories.table_repositories import ClaudeCodePluginRepository @@ -276,45 +283,52 @@ def _agent_create_fields() -> tuple[AgentCreateInfo, ...]: response_model=list[ModelGroupInfoProxy], ) async def public_model_hub(): - import litellm - from litellm.proxy.health_endpoints._health_endpoints import ( - convert_health_check_to_dict, - ) - from litellm.proxy.proxy_server import ( - get_model_group_info, - llm_router, - prisma_client, - ) - - if llm_router is None: + rows: Final = published_hub_rows() + if isinstance(rows, NoModelsConfigured): raise HTTPException(status_code=400, detail=CommonProxyErrors.no_llm_router.value) - model_groups: list[ModelGroupInfoProxy] = [] - if litellm.public_model_groups is not None: - model_groups = get_model_group_info( # rebind-ok: pre-existing rebinding on a rename-only line - llm_router=llm_router, - all_models_str=litellm.public_model_groups, - model_group=None, + health_checks_map: Final = await _legacy_health_checks() + return [_with_legacy_health(row, health_checks_map) for row in rows] + + +async def _legacy_health_checks() -> Mapping[str, Mapping[str, object]]: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return MappingProxyType({}) + try: + latest_checks: Final = await prisma_client.get_all_latest_health_checks() + return MappingProxyType( + { + model_name: MappingProxyType( + { + "status": check.status, + "response_time_ms": check.response_time_ms, + "checked_at": check.checked_at.isoformat() if check.checked_at else None, + } + ) + for model_name, check in latest_by_model_name(latest_checks).items() + } ) + except Exception: + return MappingProxyType({}) - # Fetch health check information if available - health_checks_map: Final = {} - if prisma_client is not None: - try: - latest_checks: Final = await prisma_client.get_all_latest_health_checks() - for model_name, check in latest_by_model_name(latest_checks).items(): - health_checks_map[model_name] = convert_health_check_to_dict(check) - except Exception: - pass - for model_group in model_groups: - health_info = health_checks_map.get(model_group.model_group) - if health_info: - model_group.health_status = health_info.get("status") - model_group.health_response_time = health_info.get("response_time_ms") - model_group.health_checked_at = health_info.get("checked_at") - - return model_groups +def _with_legacy_health(row: HubRow, health_checks_map: Mapping[str, Mapping[str, object]]) -> ModelGroupInfoProxy: + match row: + case PassThroughRow(): + return row.info + case ModelGroupRow(): + health_info: Final = health_checks_map.get(row.info.model_group) + if not health_info: + return row.info + return row.info.model_copy( + update={ + "health_status": health_info.get("status"), + "health_response_time": health_info.get("response_time_ms"), + "health_checked_at": health_info.get("checked_at"), + } + ) @router.get( @@ -546,7 +560,9 @@ async def get_litellm_blog_posts() -> BlogPostsResponse: Falls back to the bundled local backup on any failure. """ try: - return BlogPostsResponse(posts=get_blog_posts(url=litellm.blog_posts_url)) + return BlogPostsResponse( + posts=await GetBlogPosts.aget_blog_posts(url=litellm.blog_posts_url, fetch=_fetch_remote_bytes) + ) except Exception as e: verbose_logger.warning("LiteLLM: get_litellm_blog_posts endpoint fallback triggered: %s", str(e)) return BlogPostsResponse(posts=GetBlogPosts.load_local_blog_posts()) diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py index e58149fba0b..a00138c94c9 100644 --- a/litellm/proxy/public_endpoints/public_v1/model_hub.py +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -8,7 +8,6 @@ from typing import Annotated, Final, Literal, Protocol from fastapi import APIRouter, Depends, Request from typing_extensions import ReadOnly, TypedDict -import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -24,6 +23,13 @@ from litellm.proxy.list_api.list_framework import ( handle_facet, handle_list, ) +from litellm.proxy.public_endpoints.model_hub_rows import ( + HubRow, + ModelGroupRow, + NoModelsConfigured, + PassThroughRow, + published_hub_rows, +) from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, @@ -93,9 +99,19 @@ class HealthEnricher: lookup: HealthSnapshotLookup - async def __call__(self, rows: Sequence[ModelGroupInfoProxy]) -> Sequence[ModelGroupInfoProxy]: - health: Final = await self.lookup.latest_for(tuple(row.model_group for row in rows)) - return tuple(_with_health(row, health.get(row.model_group)) for row in rows) + async def __call__(self, rows: Sequence[HubRow]) -> Sequence[HubRow]: + health: Final = await self.lookup.latest_for( + tuple(row.info.model_group for row in rows if isinstance(row, ModelGroupRow)) + ) + return tuple(_row_with_health(row, health) for row in rows) + + +def _row_with_health(row: HubRow, health: Mapping[str, HealthSnapshot]) -> HubRow: + match row: + case PassThroughRow(): + return row + case ModelGroupRow(): + return ModelGroupRow(info=_with_health(row.info, health.get(row.info.model_group))) FEATURE_PREFIX: Final = "supports_" @@ -120,26 +136,26 @@ def _enabled_features(fields: Mapping[str, object]) -> tuple[str, ...]: ) -def _cells(row: ModelGroupInfoProxy) -> Cells: +def _cells(row: HubRow) -> Cells: + info: Final = row.info return MappingProxyType( { - "model_group": row.model_group, - "mode": row.mode, - "providers": tuple(row.providers), - "features": _features(row), - "max_input_tokens": row.max_input_tokens, - "max_output_tokens": row.max_output_tokens, - "input_cost_per_token": row.input_cost_per_token, - "output_cost_per_token": row.output_cost_per_token, - "rpm": row.rpm, - "tpm": row.tpm, + "model_group": info.model_group, + "mode": info.mode, + "providers": tuple(info.providers), + "features": _features(info), + "max_input_tokens": info.max_input_tokens, + "max_output_tokens": info.max_output_tokens, + "input_cost_per_token": info.input_cost_per_token, + "output_cost_per_token": info.output_cost_per_token, + "rpm": info.rpm, + "tpm": info.tpm, } ) -def _serialize(row: ModelGroupInfoProxy) -> ModelGroupInfoProxy: - """The row shape is the wire shape: the rows served are the router's own model group records.""" - return row +def _serialize(row: HubRow) -> ModelGroupInfoProxy: + return row.info def _scope(_caller: UserAPIKeyAuth) -> Scope: @@ -163,7 +179,7 @@ MODEL_HUB_FACETS: Final[Mapping[str, str]] = MappingProxyType( {"providers": "providers", "modes": "mode", "features": "features"} ) -MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec( +MODEL_HUB_LIST_SPEC: Final[ListSpec[HubRow, ModelGroupInfoProxy]] = ListSpec( resource="model groups", sortable=frozenset( ( @@ -189,13 +205,9 @@ MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ) -def _published_rows() -> Sequence[ModelGroupInfoProxy]: - from litellm.proxy.proxy_server import ( - get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way - llm_router, - ) - - if llm_router is None: +def _published_rows() -> Sequence[HubRow]: + rows: Final = published_hub_rows() + if isinstance(rows, NoModelsConfigured): raise ManagementProblem( ProblemDetail( type=f"{PROBLEM_TYPE_BASE}no-llm-router", @@ -204,21 +216,13 @@ def _published_rows() -> Sequence[ModelGroupInfoProxy]: detail=CommonProxyErrors.no_llm_router.value, ) ) - if litellm.public_model_groups is None: - return () - return tuple( - get_model_group_info( - llm_router=llm_router, - all_models_str=litellm.public_model_groups, - model_group=None, - ) - ) + return rows def _executor( - rows: Sequence[ModelGroupInfoProxy], + rows: Sequence[HubRow], prisma_client: PrismaClient | None, -) -> InMemoryListExecutor[ModelGroupInfoProxy]: +) -> InMemoryListExecutor[HubRow]: if prisma_client is None: return InMemoryListExecutor(rows=rows, cells=_cells) return InMemoryListExecutor( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index e5bc4f7d1d7..2963cf17b6c 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -2694,7 +2694,6 @@ async def _responses_websocket_session( user_request_timeout=user_request_timeout, user_max_tokens=user_max_tokens, user_api_base=user_api_base, - model=resolved_model, route_type="_aresponses_websocket", skip_guardrails=True, ) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 498f86cc40f..08c4b127de5 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -11,7 +11,14 @@ generator client { // Budget / Rate Limits for an org model LiteLLM_BudgetTable { + customer_organization_defaults LiteLLM_CustomerOrganizationTable[] @relation("CustomerOrganizationDefaultBudget") + customer_organizations LiteLLM_CustomerOrganizationTable[] @relation("CustomerOrganizationBudget") + customer_teams LiteLLM_CustomerTeamTable[] @relation("CustomerTeamBudget") + customer_members LiteLLM_CustomerMemberTable[] @relation("CustomerMemberBudget") + customer_member_overrides LiteLLM_CustomerMemberTable[] @relation("CustomerMemberBudgetOverride") + customer_team_overrides LiteLLM_CustomerTeamTable[] @relation("CustomerTeamBudgetOverride") budget_id String @id @default(uuid()) + name String? max_budget Float? soft_budget Float? max_parallel_requests Int? @@ -22,6 +29,8 @@ model LiteLLM_BudgetTable { model_tpm_limit Json? model_rpm_limit Json? budget_duration String? + budget_limits Json? + cap_policy String? budget_reset_at DateTime? allowed_models String[] @default([]) // per-member model scope; empty = inherit team models temp_budget_increase Float? @@ -40,6 +49,81 @@ model LiteLLM_BudgetTable { organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization } +model LiteLLM_CustomerOrganizationTable { + spend Float @default(0.0) + budget_template_ids String[] @default([]) + budget_id String? + budget LiteLLM_BudgetTable? @relation("CustomerOrganizationBudget", fields: [budget_id], references: [budget_id]) + customer_org_id String @id @default(uuid()) + alias String? + default_budget_template_id String? + default_budget_template LiteLLM_BudgetTable? @relation("CustomerOrganizationDefaultBudget", fields: [default_budget_template_id], references: [budget_id]) + teams LiteLLM_CustomerTeamTable[] + members LiteLLM_CustomerMemberTable[] + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String + @@index([spend]) +} + +model LiteLLM_CustomerTeamTable { + spend Float @default(0.0) + customer_team_id String @id @default(uuid()) + alias String? + parent_org_id String + parent_org LiteLLM_CustomerOrganizationTable @relation(fields: [parent_org_id], references: [customer_org_id]) + budget_id String? + budget LiteLLM_BudgetTable? @relation("CustomerTeamBudget", fields: [budget_id], references: [budget_id]) + budget_override_id String? + budget_override LiteLLM_BudgetTable? @relation("CustomerTeamBudgetOverride", fields: [budget_override_id], references: [budget_id], onDelete: Restrict) + budget_override_expires_at DateTime? + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String + @@index([parent_org_id]) + @@index([spend]) +} + +model LiteLLM_CustomerMemberTable { + spend Float @default(0.0) + customer_member_id String @id @default(uuid()) + alias String? + parent_org_id String + parent_org LiteLLM_CustomerOrganizationTable @relation(fields: [parent_org_id], references: [customer_org_id]) + parent_team_ids String[] @default([]) + budget_id String? + budget LiteLLM_BudgetTable? @relation("CustomerMemberBudget", fields: [budget_id], references: [budget_id], onDelete: Restrict) + budget_override_id String? + budget_override LiteLLM_BudgetTable? @relation("CustomerMemberBudgetOverride", fields: [budget_override_id], references: [budget_id], onDelete: Restrict) + budget_override_expires_at DateTime? + budget_increase_requests LiteLLM_CustomerBudgetIncreaseRequest[] + created_at DateTime @default(now()) + created_by String + updated_at DateTime @default(now()) @updatedAt + updated_by String + @@index([parent_org_id]) + @@index([spend]) +} + +model LiteLLM_CustomerBudgetIncreaseRequest { + request_id String @id @default(uuid()) + customer_member_id String + customer_member LiteLLM_CustomerMemberTable @relation(fields: [customer_member_id], references: [customer_member_id], onDelete: Cascade) + requested_max_budget Float + expires_at DateTime? + reason String? + status String @default("pending_review") + submitted_by String + submitted_at DateTime @default(now()) + reviewed_by String? + reviewed_at DateTime? + review_notes String? + @@index([status]) + @@index([customer_member_id]) +} + // Models on proxy model LiteLLM_CredentialsTable { credential_id String @id @default(uuid()) @@ -975,6 +1059,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? @@ -1630,6 +1753,7 @@ model LiteLLM_AccessGroupTable { mcp_tool_permissions Json? access_agent_ids String[] @default([]) access_skill_names String[] @default([]) + access_vector_store_ids String[] @default([]) assigned_team_ids String[] @default([]) assigned_key_ids String[] @default([]) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 096578264e3..69ba86706c4 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from itertools import chain from types import MappingProxyType -from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast +from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, TypeAlias, cast from fastapi import HTTPException, status from pydantic import ConfigDict, TypeAdapter @@ -18,8 +18,16 @@ import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route -from litellm.litellm_core_utils.duration_parser import duration_in_seconds +from litellm.litellm_core_utils.duration_parser import ( + duration_in_seconds, + get_annual_budget_window_start, + is_annual_budget_duration, +) +from litellm.litellm_core_utils.duration_parser import ( + get_budget_window_start as calendar_budget_window_start, +) from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate +from litellm.models.budget import BudgetLimit from litellm.proxy._types import ( Litellm_EntityType, LiteLLM_TeamMembership, @@ -30,6 +38,11 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import get_model_from_request from litellm.proxy.auth.budget_throttle import should_throttle_budget_exceeded from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.common_utils.timezone_utils import ( + BudgetResetSettings, + compute_budget_reset_at, + get_budget_reset_settings, +) from litellm.proxy.common_utils.user_api_key_cache import ( AUTH_OBJECTS_TARGET, UserApiKeyCache, @@ -59,14 +72,16 @@ from litellm.redis.stores.spend_counters import ( team_member_counter, user_counter, ) +from litellm.redis.stores.spend_counters.types import customer_window_counter from litellm.router import Router +from litellm.types.proxy.auth.customer_hierarchy import CustomerScopeKey from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict from litellm.types.utils import CallTypes @dataclass -class _BudgetCounter: +class BudgetCounter: counter_key: str max_budget: float fallback_spend: float @@ -76,6 +91,54 @@ class _BudgetCounter: spend_log_entity_id: str | None = None window_duration: str | None = None window_start: datetime | None = None + reset_at: datetime | None = None + + +_BudgetCounter: TypeAlias = BudgetCounter + + +def customer_window_counters( + scope_key: CustomerScopeKey, + entity_id: str, + budget_limits: Sequence[BudgetLimit], + *, + now: datetime | None = None, + reset_settings: BudgetResetSettings | None = None, +) -> tuple[BudgetCounter, ...]: + """Use the existing reset calendar with balances owned by the entity, not its shared budget row.""" + current: Final = now if now is not None else datetime.now(timezone.utc) + settings: Final = reset_settings or get_budget_reset_settings() + return tuple(_customer_window_counter(scope_key, entity_id, limit, current, settings) for limit in budget_limits) + + +def _customer_window_counter( + scope_key: CustomerScopeKey, + entity_id: str, + limit: BudgetLimit, + now: datetime, + settings: BudgetResetSettings, +) -> _BudgetCounter: + configured_reset: Final = ( + limit.reset_at.replace(tzinfo=timezone.utc) + if limit.reset_at is not None and limit.reset_at.tzinfo is None + else limit.reset_at + ) + standard_reset: Final = compute_budget_reset_at(limit.budget_duration, settings, now) + reset_at: Final = configured_reset if configured_reset is not None and configured_reset > now else standard_reset + start: Final = calendar_budget_window_start( + limit.budget_duration, reset_at.astimezone(standard_reset.tzinfo) + ).astimezone(timezone.utc) + return _BudgetCounter( + counter_key=customer_window_counter(scope_key, entity_id, limit.budget_duration, start), + max_budget=limit.max_budget, + fallback_spend=0.0, + entity_type=scope_key, + entity_id=entity_id, + spend_log_entity_id=entity_id, + window_duration=limit.budget_duration, + window_start=start, + reset_at=reset_at.astimezone(timezone.utc), + ) _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = { @@ -88,6 +151,9 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = { "Model access group": Litellm_EntityType.MODEL_ACCESS_GROUP.value, "Organization": Litellm_EntityType.ORGANIZATION.value, "Project": Litellm_EntityType.PROJECT.value, + "customer_org": Litellm_EntityType.CUSTOMER_ORG.value, + "customer_team": Litellm_EntityType.CUSTOMER_TEAM.value, + "customer_member": Litellm_EntityType.CUSTOMER_MEMBER.value, } _CACHED_VALUE: Final = TypeAdapter(object) @@ -386,11 +452,12 @@ def stamp_budget_reservation_actual_cost(budget_reservation: dict | None, actual return reserved_cost: Final = float(budget_reservation.get("reserved_cost") or 0.0) actual: Final = float(actual_cost or 0.0) + settlement_time: Final = datetime.now(timezone.utc) for entry in budget_reservation.get("entries") or []: if "counter_key" in entry: - entry["applied_adjustment"] = actual - _get_entry_reserved_cost( - entry=entry, default_reserved_cost=reserved_cost - ) + entry["applied_adjustment"] = _entry_settled_cost( + _WINDOW_OBJECT.validate_python(entry), actual, settlement_time + ) - _get_entry_reserved_cost(entry=entry, default_reserved_cost=reserved_cost) async def release_budget_reservation(budget_reservation: dict | None) -> None: @@ -499,6 +566,10 @@ async def _get_budget_counters( ) -> list[_BudgetCounter]: counters: Final[list[_BudgetCounter]] = [] + if valid_token.customer_hierarchy is not None: + for scope in valid_token.customer_hierarchy.scopes: + counters.extend(customer_window_counters(scope.key, scope.entity_id, scope.budget_limits)) + if valid_token.token is not None: if valid_token.max_budget is not None and valid_token.max_budget > 0: counters.append( @@ -1121,22 +1192,36 @@ class _EntryAdjustment: counter_key: str target_adjustment: float adjustment: float + actual_cost: float + + +def _entry_settled_cost(entry: Mapping[str, object], actual_cost: float, settlement_time: datetime | None) -> float: + reset_at: Final = _coerce_datetime(entry.get("window_reset_at")) + return 0.0 if settlement_time is not None and reset_at is not None and reset_at <= settlement_time else actual_cost def _entry_adjustment( - entry: dict[str, float | str], actual_cost: float, default_reserved_cost: float + entry: dict[str, float | str], + actual_cost: float, + default_reserved_cost: float, + settlement_time: datetime | None = None, ) -> _EntryAdjustment | None: counter_key: Final = entry.get("counter_key") if counter_key is None: return None - target_adjustment: Final = actual_cost - _get_entry_reserved_cost( + settled_cost: Final = _entry_settled_cost(entry, actual_cost, settlement_time) + target_adjustment: Final = settled_cost - _get_entry_reserved_cost( entry=entry, default_reserved_cost=default_reserved_cost ) adjustment: Final = target_adjustment - float(entry.get("applied_adjustment") or 0.0) if adjustment == 0: return None return _EntryAdjustment( - entry=entry, counter_key=str(counter_key), target_adjustment=target_adjustment, adjustment=adjustment + entry=entry, + counter_key=str(counter_key), + target_adjustment=target_adjustment, + adjustment=adjustment, + actual_cost=settled_cost, ) @@ -1152,11 +1237,12 @@ async def _set_reserved_entries_actual_cost( is settled on its own after the pipeline.""" from litellm.proxy.proxy_server import increment_spend_counters_pipeline + settlement_time: Final = datetime.now(timezone.utc) if reseed_on_inconsistent else None with _counters_batch_scope(frozenset(str(entry["counter_key"]) for entry in entries if "counter_key" in entry)): adjustments: Final = tuple( adjustment for entry in entries - if (adjustment := _entry_adjustment(entry, actual_cost, default_reserved_cost)) is not None + if (adjustment := _entry_adjustment(entry, actual_cost, default_reserved_cost, settlement_time)) is not None ) consistent: Final = tuple( await asyncio.gather( @@ -1182,7 +1268,7 @@ async def _set_reserved_entries_actual_cost( if apply_consistent: await increment_spend_counters_pipeline(pending=applicable_pending) for item in inconsistent: - await _reseed_reserved_entry(item=item, actual_cost=actual_cost) + await _reseed_reserved_entry(item=item, actual_cost=item.actual_cost) for item in adjustments if apply_consistent else inconsistent: item.entry["applied_adjustment"] = item.target_adjustment return () if apply_consistent else applicable_pending @@ -1268,6 +1354,7 @@ def _counter_to_reservation_entry( "entity_id": counter.entity_id, "reserved_cost": reserved_cost, "applied_adjustment": 0.0, + **({"window_reset_at": counter.reset_at.isoformat()} if counter.reset_at is not None else {}), } @@ -1287,12 +1374,18 @@ def fixed_budget_window(window: object) -> tuple[str, datetime] | None: return duration, start -def get_budget_window_start(window: object) -> datetime | None: +def get_budget_window_start(window: object, reset_settings: BudgetResetSettings | None = None) -> datetime | None: window_dict: Final = _coerce_window(window) budget_duration: Final = window_dict.get("budget_duration") if budget_duration is None: return None try: + if is_annual_budget_duration(str(budget_duration)): + annual_reset_at: Final = _coerce_datetime(window_dict.get("reset_at")) + if annual_reset_at is not None: + return get_annual_budget_window_start( + annual_reset_at, (reset_settings or get_budget_reset_settings()).timezone + ) duration_seconds: Final = duration_in_seconds(str(budget_duration)) except Exception: return None diff --git a/litellm/proxy/spend_tracking/customer_budget_alerts.py b/litellm/proxy/spend_tracking/customer_budget_alerts.py new file mode 100644 index 00000000000..d88ed4329e6 --- /dev/null +++ b/litellm/proxy/spend_tracking/customer_budget_alerts.py @@ -0,0 +1,187 @@ +import asyncio +import hashlib +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +from typing import Final, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, InstanceOf, TypeAdapter + +import litellm +from litellm._internal_context import with_service_target +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy.management_helpers.customer_hierarchy import ( + PrismaRecord, + customer_row_reset_at, + effective_member_budget, + related_budget, + utc_now, +) +from litellm.types.integrations.customer_budget import CustomerBudgetEvent, CustomerBudgetEventType, CustomerBudgetScope + + +class CustomerBudgetClaims(Protocol): + async def async_increment_cache(self, *, key: str, value: float, ttl: float) -> float | None: ... + + +@runtime_checkable +class _CustomerBudgetTable(Protocol): + async def find_unique( + self, *, where: Mapping[str, object], include: Mapping[str, object] | None = None + ) -> PrismaRecord | None: ... + + +class _CustomerRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + spend: float = 0.0 + alias: str | None = None + customer_org_id: str | None = None + customer_team_id: str | None = None + budget: Mapping[str, object] | None = None + budget_override: Mapping[str, object] | None = None + budget_override_expires_at: datetime | None = None + parent_org: "_CustomerRow | None" = None + + +@dataclass(frozen=True, slots=True) +class _Level: + scope: CustomerBudgetScope + transactions: str + id_column: str + table: str + include: Mapping[str, object] + + +_LEVELS: Final = ( + _Level( + "customer_org", + "customer_org_list_transactions", + "customer_org_id", + "litellm_customerorganizationtable", + {"budget": True}, + ), + _Level( + "customer_team", + "customer_team_list_transactions", + "customer_team_id", + "litellm_customerteamtable", + {"budget": True, "budget_override": True, "parent_org": True}, + ), + _Level( + "customer_member", + "customer_member_list_transactions", + "customer_member_id", + "litellm_customermembertable", + {"budget": True, "budget_override": True, "parent_org": True}, + ), +) +_CHARGES: Final = TypeAdapter(Mapping[str, float]) +_TABLE: Final = TypeAdapter(InstanceOf[_CustomerBudgetTable]) + + +def _events(level: _Level, entity_id: str, row: _CustomerRow, now: datetime) -> tuple[CustomerBudgetEvent, ...]: + template: Final = related_budget(row.budget, now) + override: Final = related_budget(row.budget_override, now, template.max_budget if template is not None else None) + budget: Final = ( + effective_member_budget(template, override, row.budget_override_expires_at, now) + if level.scope != "customer_org" + else template + ) + if budget is None: + return () + customer_team: Final = row if level.scope == "customer_team" else None + customer_org: Final = row if level.scope == "customer_org" else row.parent_org + reset_at: Final = customer_row_reset_at(row, now) + thresholds: Final[tuple[tuple[CustomerBudgetEventType, float | None], ...]] = ( + ("soft_budget_crossed", budget.soft_budget), + ("budget_crossed", budget.max_budget), + ) + return tuple( + CustomerBudgetEvent( + event=event, + scope=level.scope, + entity_id=entity_id, + customer_org_id=customer_org.customer_org_id if customer_org else None, + customer_org_alias=customer_org.alias if customer_org else None, + customer_team_id=( + customer_team.customer_team_id if level.scope != "customer_org" and customer_team else None + ), + customer_team_alias=customer_team.alias if level.scope != "customer_org" and customer_team else None, + customer_member_id=entity_id if level.scope == "customer_member" else None, + customer_member_alias=row.alias if level.scope == "customer_member" else None, + spend=row.spend, + budget=threshold, + period=reset_at.isoformat() if reset_at is not None else "lifetime", + timestamp=now, + ) + for event, threshold in thresholds + if threshold is not None and row.spend >= threshold + ) + + +async def _detect_customer_events( + level: _Level, entity_id: str, database: object, cache: CustomerBudgetClaims, now: datetime +) -> None: + try: + table: Final = _TABLE.validate_python(getattr(getattr(database, "writer_db", database), level.table)) + record: Final = await table.find_unique(where={level.id_column: entity_id}, include=level.include) + if record is None: + return + for event in _events(level, entity_id, _CustomerRow.model_validate(record.model_dump()), now): + await CustomerBudgetAlerting.emit(event, cache) + except Exception: # noqa: BLE001 # Notification failures must not replay committed charges. + verbose_proxy_logger.exception("Failed to send %s budget alert for %s", level.scope, entity_id) + + +class CustomerBudgetAlerting: + @staticmethod + def callbacks() -> tuple[CustomLogger, ...]: + return tuple( + callback + for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger) + if type(callback).async_on_customer_budget_event is not CustomLogger.async_on_customer_budget_event + ) + + @staticmethod + async def detect_committed_batch( + transactions: Mapping[str, object], + database: object, + cache: CustomerBudgetClaims, + *, + now: datetime | None = None, + ) -> None: + if not CustomerBudgetAlerting.callbacks(): + return + instant: Final = now if now is not None else utc_now() + for level in _LEVELS: + for entity_id, cost in _CHARGES.validate_python(transactions.get(level.transactions) or {}).items(): + if cost > 0: + await _detect_customer_events(level, entity_id, database, cache, instant) + + @staticmethod + @with_service_target("customer_budget_alerts") + async def emit(event: CustomerBudgetEvent, cache: CustomerBudgetClaims) -> None: + callbacks: Final = CustomerBudgetAlerting.callbacks() + if not callbacks: + return + identity: Final = json.dumps((event.event, event.scope, event.entity_id, event.period)) + key: Final = "customer_budget_alerts:" + hashlib.sha256(identity.encode()).hexdigest() + ttl: Final = ( + 100 * 365 * 24 * 60 * 60 + if event.period == "lifetime" + else max(1, math.ceil((datetime.fromisoformat(event.period) - event.timestamp).total_seconds())) + ) + if await cache.async_increment_cache(key=key, value=1, ttl=ttl) != 1: + return + await asyncio.gather(*(CustomerBudgetAlerting._deliver(callback, event) for callback in callbacks)) + + @staticmethod + async def _deliver(callback: CustomLogger, event: CustomerBudgetEvent) -> None: + try: + await callback.async_on_customer_budget_event(event) + except Exception: # noqa: BLE001 # A failed callback must not replay committed spend or hide other callbacks. + verbose_proxy_logger.exception("Failed to deliver %s customer budget event", event.event) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index f431763c049..a4c6d8a40c7 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,12 +1,13 @@ +import asyncio import ipaddress import json import os import re import secrets -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime, timezone from datetime import datetime as dt -from functools import reduce +from functools import partial, reduce from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable @@ -72,6 +73,7 @@ from litellm.litellm_core_utils.litellm_logging import ( from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.litellm_core_utils.ptu_pricing import azure_spillover from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes +from litellm.litellm_core_utils.secret_redaction import redact_tokens from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.llms.anthropic.common_utils import resolve_used_client_oauth_token from litellm.proxy._types import ( @@ -214,6 +216,12 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "is_silent_experiment", BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", + "user_api_key_customer_org_id", + "user_api_key_customer_org_alias", + "user_api_key_customer_team_id", + "user_api_key_customer_team_alias", + "user_api_key_customer_member_id", + "user_api_key_customer_member_alias", ) ) @@ -227,6 +235,30 @@ def _stored_user_email(metadata: Mapping[str, object]) -> str | None: return email if isinstance(email, str) and email.strip() else None +_CUSTOMER_METADATA_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) + + +class _CustomerIdentityMetadata(TypedDict, total=False): + user_api_key_customer_org_id: ReadOnly[str] + user_api_key_customer_org_alias: ReadOnly[str] + user_api_key_customer_team_id: ReadOnly[str] + user_api_key_customer_team_alias: ReadOnly[str] + user_api_key_customer_member_id: ReadOnly[str] + user_api_key_customer_member_alias: ReadOnly[str] + + +_CUSTOMER_IDENTITY_ADAPTER: Final[TypeAdapter[_CustomerIdentityMetadata]] = TypeAdapter(_CustomerIdentityMetadata) + + +def _stored_customer_identity(metadata: Mapping[object, object]) -> _CustomerIdentityMetadata: + values: Final = { + key: value + for key in _CustomerIdentityMetadata.__annotations__ + if isinstance(value := metadata.get(key), str) and value.strip() + } + return _CUSTOMER_IDENTITY_ADAPTER.validate_python(values) + + def _get_spend_logs_metadata( metadata: dict | None, applied_guardrails: list[str] | None = None, @@ -310,10 +342,12 @@ def _get_spend_logs_metadata( retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True # Filter the metadata dictionary to include only the specified keys + customer_metadata: Final = _CUSTOMER_METADATA_ADAPTER.validate_python(metadata) clean_metadata: Final = SpendLogsMetadata( **MappingProxyType( {key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key not in _STAMPED_METADATA_KEYS} ), + **_stored_customer_identity(customer_metadata), **upstream_provider_metadata, autorouter_savings=autorouter_savings, autorouter_savings_estimate=autorouter_savings_estimate, @@ -609,6 +643,45 @@ def _scrub_raw_model_from_error_information( ) +def _spend_log_worker_kwargs(kwargs: Mapping[str, object]) -> Mapping[str, object]: + params: Final = as_str_mapping(kwargs.get("litellm_params")) + if params is None: + return kwargs + metadata: Final = as_str_mapping(params.get("litellm_metadata")) + return { + **kwargs, + "litellm_params": { + **params, + **({"litellm_metadata": dict(metadata)} if metadata is not None else {}), + }, + } + + +async def get_logging_payload_async( + kwargs: Mapping[str, object] | None, + response_obj: object, + start_time: datetime, + end_time: datetime, + llm_router: "Router | None" = None, + run_in_thread: Callable[[Callable[[], SpendLogsPayload]], Awaitable[SpendLogsPayload]] = asyncio.to_thread, +) -> SpendLogsPayload: + from litellm.proxy.proxy_server import general_settings_view + + call_kwargs: Final[Mapping[str, object]] = kwargs if kwargs is not None else MappingProxyType({}) + offload: Final = general_settings_view().get("redact_tokens_in_spend_logs") is True + build: Final = partial( + get_logging_payload, + kwargs=dict(_spend_log_worker_kwargs(call_kwargs)) if offload else dict(call_kwargs), + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + llm_router=llm_router, + ) + if offload: + return await run_in_thread(build) + return build() + + def get_logging_payload( kwargs: dict | None, response_obj: object, @@ -1286,7 +1359,7 @@ def _get_messages_for_spend_logs_payload( messages: Final = standard_logging_payload.get("messages") if messages is not None: try: - return safe_dumps(messages, non_finite_as_null=True) + return _without_tokens(safe_dumps(messages, non_finite_as_null=True)) except Exception: return "{}" return "{}" @@ -1459,6 +1532,14 @@ def _is_spend_log_content(parent: Mapping[str, object], key: str) -> bool: ) +def _without_tokens(value: str) -> str: + from litellm.proxy.proxy_server import general_settings_view + + if general_settings_view().get("redact_tokens_in_spend_logs") is not True: + return value + return redact_tokens(strip_null_bytes(value), REDACTED_BY_LITELM_STRING) + + def _truncated_for_db(value: str, max_length: int) -> str: if len(value) <= max_length: return value @@ -1474,7 +1555,7 @@ def _truncated_for_db(value: str, max_length: int) -> str: def _compaction_summary_for_db(summary: str) -> str: return _truncated_for_db( - summary, max(_get_max_string_length_prompt_in_db(), MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB) + _without_tokens(summary), max(_get_max_string_length_prompt_in_db(), MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB) ) @@ -1503,7 +1584,7 @@ def _sanitize_request_body_for_spend_logs_payload( At every nesting level, also strips keys listed in _SENSITIVE_REQUEST_BODY_KEYS (e.g. secret_fields, which holds raw HTTP headers including Authorization tokens), and replaces string values under keys SensitiveDataMasker classifies as credentials with REDACTED_BY_LITELM_STRING, except inside tool payloads - and logprobs. + and logprobs. When enabled, token-shaped substrings (e.g. a GitHub token in a tool result) are masked everywhere. """ if visited is None: visited = set() @@ -1524,7 +1605,7 @@ def _sanitize_request_body_for_spend_logs_payload( elif isinstance(value, list): return [_sanitize_value(item, mask_credentials) for item in value] elif isinstance(value, str): - return _truncated_for_db(value, max_string_length_prompt_in_db) + return _truncated_for_db(_without_tokens(value), max_string_length_prompt_in_db) return value def _stored_value(key: str, value: object) -> object: diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 2de589564ac..199d57ce671 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -377,6 +377,16 @@ class UISettings(LiteLLMBaseModel): description="If true, shows the Chat page in the UI sidebar, letting users chat with an LLM and connect their own MCP server credentials via OAuth.", ) + disable_aria: bool = Field( + default=False, + description="Hides Aria for everyone on this deployment.", + ) + + aria_internal_user_access: Literal["none", "own_usage", "own_and_team_usage"] = Field( + default="own_usage", + description="Aria access for internal users. Proxy admins and proxy-wide spend viewers are unaffected.", + ) + moyai_url: str | None = Field( default=None, description="URL of a connected Moyai deployment. When set, the Moyai entry in the UI navigation opens this deployment instead of the Moyai landing page.", @@ -432,6 +442,8 @@ ALLOWED_UI_SETTINGS_FIELDS: Final = { "disable_custom_api_keys", "disable_key_generate_for_org_admin", "enable_chat_ui", + "disable_aria", + "aria_internal_user_access", "moyai_url", TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, } diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ee78db34d62..f93b606467f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -32,7 +32,6 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate from functools import partial, reduce -from itertools import takewhile from operator import attrgetter from types import MappingProxyType from typing import ( @@ -160,7 +159,28 @@ from litellm.integrations.custom_guardrail import ( ModifyResponseException, merged_guardrail_entries, ) -from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger import CustomLogger, subclass_overrides_hook +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.legacy import ( + PROXY_REQUEST_KEYS, + Admit, + HookSubscribers, + KwargsFields, + adapter_for, + hook_subscribers, + kwargs_after, + kwargs_patch, + run_subscribers, +) +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.outcomes import ( + MERGEABLE_FIELDS, + InvalidPatchError, + MergeField, + Patch, + PatchOp, +) +from litellm.integrations.custom_logger_v2.patches import LiveFields, apply_patch from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.utils import add_langfuse_trace_id_to_alert @@ -175,7 +195,11 @@ from litellm.litellm_core_utils.core_helpers import ( require_attributes, ) from litellm.litellm_core_utils.deployment_pricing import pricing_models -from litellm.litellm_core_utils.duration_parser import get_budget_window_start +from litellm.litellm_core_utils.duration_parser import ( + get_annual_budget_window_start, + get_budget_window_start, + is_annual_budget_duration, +) from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import ( BilledRoundsDispatch, @@ -208,6 +232,7 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolv from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_change +from litellm.proxy.common_utils.timezone_utils import get_budget_reset_settings from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db import tool_registry_writer from litellm.proxy.db.create_views import ( @@ -332,7 +357,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.models.team import LiteLLM_TeamTableCachedObj - from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction + from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction, UnwrittenTurns from litellm.proxy.db.baseline_accounting import BaselineAccountingRecord from litellm.proxy.db.model_usage_rollup import ModelUsageTransaction from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction @@ -1689,11 +1714,20 @@ def _deployment_attribution_for_model_group(model_group: object, team_id: str | ) -def _call_type_for_route(route: str | None) -> str | None: +def _call_type_for_route(route: str | None, *, method: str | None = None) -> str | None: """The route's call type when it maps to a single operation (its async and sync variants); - None for routes shared by several operations, since the method is not known here.""" + None for routes shared by several operations unless the method disambiguates them.""" if route is None: return None + normalized_route: Final = re.sub(r"^/v1(?:beta)?/", "/", route) + endpoint_call_type: Final = { + "/responses/compact": "acompact_responses", + "/interactions": "acreate_interaction", + }.get(normalized_route) + if endpoint_call_type is not None: + return endpoint_call_type + if normalized_route == "/videos" and method == "POST": + return "avideo_generation" call_types: Final = get_call_types_for_route(route) if not call_types: return None @@ -1769,7 +1803,14 @@ def _as_str_mapping(value: object) -> Mapping[str, object]: def _proxy_only_error_logged_metadata_keys(*, authenticated: bool, past_pre_call: bool) -> frozenset[str]: return frozenset( key - for key, logged in (("tags", authenticated), ("standard_logging_guardrail_information", past_pre_call)) + for key, logged in ( + ("tags", authenticated), + ("session_id", authenticated), + ("trace_user_id", authenticated), + ("trace_id", authenticated), + ("trace_name", authenticated), + ("standard_logging_guardrail_information", past_pre_call), + ) if logged ) @@ -1893,12 +1934,19 @@ class _CallbackCapabilities: # avoids the per-request ``get_custom_logger_compatible_class`` walk for # every string entry in ``litellm.callbacks``. resolved_callbacks: tuple[object, ...] = field(default_factory=tuple) + hook_callbacks: tuple[object, ...] = field(default_factory=tuple) + has_v2: bool = False listed_models_filters: tuple[CustomLogger, ...] = field(default_factory=tuple) -def _overrides_hook(callback: CustomLogger, hook_name: str) -> bool: - leaf_to_base: Final = takewhile(lambda klass: klass is not CustomLogger, type(callback).__mro__) - return any(hook_name in klass.__dict__ for klass in leaf_to_base) +def _overrides_hook(callback: object, hook_name: str) -> bool: + return subclass_overrides_hook(type(callback), hook_name) + + +def routes_through_apply_guardrail(callback: CustomGuardrail) -> bool: + return subclass_overrides_hook(type(callback), "apply_guardrail", CustomGuardrail) and not getattr( + callback, "use_native_lifecycle_hooks", False + ) def _overrides_moderation_hook(callback: CustomLogger) -> bool: @@ -1916,23 +1964,6 @@ class MalformedListingFilterReturn: tag: Literal["malformed_listing_filter_return"] = "malformed_listing_filter_return" -async def _names_kept_by_listing_callbacks( - callbacks: Sequence[CustomLogger], - user_api_key_dict: UserAPIKeyAuth, - model_names: tuple[str, ...], -) -> tuple[str, ...] | MalformedListingFilterReturn: - if not callbacks or not model_names: - return model_names - returned: Final = await callbacks[0].async_filter_listed_models(user_api_key_dict, model_names) - try: - kept: Final = frozenset(_LISTED_MODEL_NAMES.validate_python(returned)) - except ValidationError: - return MalformedListingFilterReturn(callback=type(callbacks[0]).__name__) - return await _names_kept_by_listing_callbacks( - callbacks[1:], user_api_key_dict, tuple(name for name in model_names if name in kept) - ) - - def _raise_malformed_listing_filter_return(error: MalformedListingFilterReturn) -> NoReturn: raise ProxyException( message=f"{error.callback}.async_filter_listed_models must return a sequence of model names", @@ -1946,6 +1977,174 @@ def _proxy_hook_factory(hook_name: str) -> Callable[..., CustomLogger]: return get_proxy_hook(hook_name) +def _request_dict(value: object) -> dict[str, object] | None: # mutable-ok: the live proxy request dict + return cast("dict[str, object] | None", value) # cast-ok: legacy hooks carry the untyped request dict + + +def _parallel_input( + callback: CustomGuardrail, + data: dict[str, object], # mutable-ok: the live proxy request dict + raw_request_snapshot: dict[str, object] | None, # mutable-ok: same request-payload shape as data +) -> dict[str, object]: # mutable-ok: same request-payload shape as data + if not callback.scan_raw_request or raw_request_snapshot is None: + return data + return independent_snapshot(raw_request_snapshot) + + +@dataclass(slots=True, eq=False) +class _PreCallRun: + user_api_key_dict: UserAPIKeyAuth + call_type: CallTypesLiteral + guardrails_only: bool + skip_guardrails: bool + endpoint_type: EndpointType + raw_request_snapshot: dict[str, object] | None # mutable-ok: same request-payload shape as latest + latest: dict[str, object] | None # mutable-ok: the request as the callbacks have left it so far + pipeline_managed: frozenset[str] = frozenset() + deferred: SensitiveDataRouteException | None = None + parallel: list[tuple[CustomGuardrail, list[object]]] = field( # mutable-ok: parallel results in list order + default_factory=list + ) + + def settle_parallel(self) -> None: + data: Final = self.latest + if data is None: + return + for callback, slot in self.parallel: + if not slot: + continue + result = slot[0] + if callback.scan_raw_request and not isinstance(result, BaseException) and result is not None: + callback.mark_pre_call_hook_ran(data) + if isinstance(result, BaseException) and not isinstance(result, SensitiveDataRouteException): + _record_raising_guardrail(data, callback) + + def raise_parallel_flow_change(self) -> None: + raised: Final = next( + (slot[0] for _, slot in self.parallel if slot and isinstance(slot[0], BaseException)), None + ) + if raised is not None: + raise raised + + +@dataclass(slots=True, eq=False) +class HookRun(Generic[_T]): + response: _T + flow_changes: list[list[BaseException]] = field( # mutable-ok: one slot per parallel guardrail in list order + default_factory=list + ) + transformed: HTTPException | None = None + + def raise_flow_change(self) -> None: + raised: Final = next((slot[0] for slot in self.flow_changes if slot), None) + if raised is not None: + raise raised + + def hold(self, response: _T) -> None: + self.response = response + + def adopt(self, reply: _T | None) -> None: + if reply is not None: + self.response = reply + + def keep_first(self, error: HTTPException) -> None: + if self.transformed is None: + self.transformed = error + + def parallel_slot(self) -> list[BaseException]: # mutable-ok: filled with one guardrail's flow change + slot: Final[list[BaseException]] = [] # mutable-ok: see above + self.flow_changes.append(slot) + return slot + + +@dataclass(slots=True, eq=False) +class _PostCallGuardrails: + data: dict[str, object] # mutable-ok: guardrail_to_apply is written into the live request + guardrail_data: dict[str, object] # mutable-ok: the request merged with model-level guardrails + request_view: Mapping[str, object] + pipeline_managed: frozenset[str] + user_api_key_dict: UserAPIKeyAuth + parallel_view: Mapping[str, object] | None = None + + def parallel_request_view(self) -> Mapping[str, object]: + if self.parallel_view is None: + self.parallel_view = _REQUEST_FIELDS.validate_python(self.data) + return self.parallel_view + + +class HookRunFields(LiveFields, Generic[_T]): + __slots__ = ("_kwargs", "_run", "_slot") + + def __init__(self, kwargs: KwargsFields, run: HookRun[_T], slot: Field) -> None: + self._kwargs: Final = kwargs + self._run: Final = run + self._slot: Final = slot + + def _held(self, field: Field) -> bool: + return field is self._slot or field is Field.deprecated_response_obj + + def __getitem__(self, field: Field) -> object: + return self._run.response if self._held(field) else self._kwargs[field] + + def __iter__(self) -> Iterator[Field]: + return iter((*self._kwargs, Field.deprecated_response_obj, self._slot)) + + def __len__(self) -> int: + return len(self._kwargs) + 2 + + def apply(self, patch: Patch) -> Mapping[Field, object]: + for op in patch.ops: + self._apply_op(op) + return self + + def _apply_op(self, op: PatchOp) -> None: + if not self._held(op.field): + if op.field is Field.deprecated_kwargs: + raise InvalidPatchError("this hook keeps the request it was given") + _ = self._kwargs.apply(Patch((op,))) + return + if isinstance(op, MergeField): + self._merge(op.keys) + return + if self._slot in MERGEABLE_FIELDS: + raise InvalidPatchError(f"{self._slot.value} is add-only here, so merge keys into it") + self._run.hold(op.value) # pyright: ignore[reportArgumentType] # a V2 patch carries an untyped value + + def _merge(self, keys: Mapping[str, object]) -> None: + current: Final = self._run.response + if not isinstance(current, dict): + raise InvalidPatchError(f"{self._slot.value} holds no mapping here, so keys cannot be merged into it") + cast("dict[str, object]", current).update(keys) # cast-ok: merged headers are str-keyed + + +_NO_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) +_NO_KEYS: Final[Mapping[Field, str]] = MappingProxyType({}) + + +def _resolved_logger(callback: object) -> CustomLogger | None: + resolved: Final = ( + litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class( + cast(_custom_logger_compatible_callbacks_literal, callback) # cast-ok: unknown names resolve to None + ) + if isinstance(callback, str) + else callback + ) + return resolved if isinstance(resolved, CustomLogger) else None + + +def hook_fields(kwargs: object, run: HookRun[_T], slot: Field) -> HookRunFields[_T]: + return HookRunFields(KwargsFields(kwargs, PROXY_REQUEST_KEYS), run, slot) + + +async def run_hook_subscribers( + subscribers: HookSubscribers, + fields: Mapping[Field, object], + binding: Callable[[object, object], Awaitable[None] | None], +) -> None: + if subscribers.entries: + _ = kwargs_after(await run_subscribers(subscribers, fields, binding)) + + class ProxyLogging: """ Logging/Custom Handlers for proxy. @@ -2639,9 +2838,7 @@ class ProxyLogging: Result from the guardrail execution """ # Use unified_guardrail if callback has apply_guardrail method - has_apply_guardrail: Final = "apply_guardrail" in type(callback).__dict__ and not getattr( - callback, "use_native_lifecycle_hooks", False - ) + has_apply_guardrail: Final = routes_through_apply_guardrail(callback) use_unified: Final = has_apply_guardrail and not ( hook_type == "during_call" and getattr(callback, "use_native_during_call_hook", False) ) @@ -3297,214 +3494,187 @@ class ProxyLogging: independent_snapshot(data) if needs_raw_request_snapshot else None ) - try: - if not skip_guardrails: - data, _ = await self._maybe_execute_pipelines( # rebind-ok: pipeline edits feed the callback loop below - data=data, - user_api_key_dict=user_api_key_dict, - call_type=call_type, - event_hook="pre_call", - raw_request_snapshot=raw_request_snapshot, - ) - - # Get pipeline-managed guardrails to skip in normal loop - pipeline_managed: Final[frozenset[str]] = ( - frozenset() if skip_guardrails else pipeline_managed_guardrail_names(data, "pre_call") + return await self._dispatch_pre_call( + _PreCallRun( + user_api_key_dict=user_api_key_dict, + call_type=call_type, + guardrails_only=guardrails_only, + skip_guardrails=skip_guardrails, + endpoint_type=endpoint_type, + raw_request_snapshot=raw_request_snapshot, + latest=data, ) + ) + + async def _dispatch_pre_call(self, run: "_PreCallRun") -> dict[str, object] | None: # mutable-ok: request dict + try: + if not run.skip_guardrails and run.latest is not None: + run.latest, _ = await self._maybe_execute_pipelines( + data=run.latest, + user_api_key_dict=run.user_api_key_dict, + call_type=run.call_type, + event_hook="pre_call", + raw_request_snapshot=run.raw_request_snapshot, + ) + if not run.skip_guardrails and run.latest is not None: + run.pipeline_managed = pipeline_managed_guardrail_names(run.latest, "pre_call") caps: Final = ProxyLogging.callback_capabilities() - # Skip the per-request callback walk entirely when nothing in - # ``litellm.callbacks`` overrides ``async_pre_call_hook`` and no - # CustomGuardrail is configured. Saves the loop overhead + - # ``time.time()`` x2 per registered callback for the common - # "callbacks=[]" case on small / dev deployments. if ( - (skip_guardrails or not caps.has_guardrail) + (run.skip_guardrails or not caps.has_guardrail) and not caps.has_content_enforcer - and (guardrails_only or not caps.has_pre_call_override) + and (run.guardrails_only or not caps.has_pre_call_override) + and not caps.has_v2 ): - if data is not None: - self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails) - return data + if run.latest is not None: + self._process_guardrail_metadata(run.latest, skip_guardrails=run.skip_guardrails) + return run.latest - parallel_guardrails: Final[tuple[CustomGuardrail, ...]] = ( - () - if skip_guardrails - else tuple( - cb - for cb in caps.resolved_callbacks - if isinstance(cb, CustomGuardrail) - and getattr(cb, "run_in_parallel", False) - and not (cb.guardrail_name and cb.guardrail_name in pipeline_managed) - ) + subscribers: Final = hook_subscribers( + Hook.pre_call, caps.hook_callbacks, include_v2=not run.guardrails_only ) - - deferred_route_exc: SensitiveDataRouteException | None = None - for _callback in caps.resolved_callbacks: - start_time = time.time() - try: - if isinstance(_callback, CustomGuardrail) and data is not None: - if skip_guardrails: - continue - - # Skip guardrails managed by a pipeline - if _callback.guardrail_name and _callback.guardrail_name in pipeline_managed: - continue - - if getattr(_callback, "run_in_parallel", False): - continue - - data = await self._run_sequential_guardrail_callback( - callback=_callback, - data=data, - raw_request_snapshot=raw_request_snapshot, - user_api_key_dict=user_api_key_dict, - call_type=call_type, - ) - - elif ( - _callback is not None - and isinstance(_callback, CustomLogger) - and (not guardrails_only or _callback.enforces_request_content) - and "async_pre_call_hook" in vars(_callback.__class__) - and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook - ): - if call_type in MCP_GUARDRAIL_CALL_TYPES and user_api_key_dict is None: - continue - - response: Exception | str | Mapping[str, object] | None = ( - await _callback.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=self.call_details["user_api_key_cache"], - data=data, - call_type=call_type, - endpoint_type=endpoint_type, - ) - if isinstance(_callback, PROXY_MaxParallelRequestsHandler_v3) - else await _callback.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=self.call_details["user_api_key_cache"], - data=data, - call_type=call_type, - ) - ) - if response is not None: - data = await self.process_pre_call_hook_response( - response=response, data=data, call_type=call_type - ) - except SensitiveDataRouteException as e: - # Defer the reroute until remaining guardrails have run so later - # security checks are not skipped; the first reroute wins and a - # later guardrail that blocks still propagates. Fall through to the - # service-span recording below so the triggering guardrail is still - # timed like every other callback. - if deferred_route_exc is None: - deferred_route_exc = e - - end_time = time.time() - duration = end_time - start_time - if ( - hasattr(self, "service_logging_obj") and duration > 0.01 - ): # only if duration is non-negligible - don't spam the logs - await self.service_logging_obj.async_service_success_hook( - service=ServiceTypes.PROXY_PRE_CALL, - duration=duration, - call_type=f"{_callback.__class__.__name__}", - parent_otel_span=getattr(user_api_key_dict, "parent_otel_span", None), - start_time=start_time, - end_time=end_time, - ) - - if deferred_route_exc is not None and data is not None: - data = await self._handle_sensitive_data_route_exception(deferred_route_exc, data, user_api_key_dict) - - if parallel_guardrails and data is not None: - await self._run_parallel_pre_call_guardrails( - guardrails=parallel_guardrails, - data=data, - raw_request_snapshot=raw_request_snapshot, - user_api_key_dict=user_api_key_dict, - call_type=call_type, + try: + result: Final = await run_subscribers( + subscribers, + KwargsFields(run.latest, PROXY_REQUEST_KEYS), + partial(self._pre_call_start, run), + between=partial(self._pre_call_deferred_reroute, run), ) + finally: + run.settle_parallel() + run.latest = _request_dict(kwargs_after(result)) + run.raise_parallel_flow_change() - if data is not None: - self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails) - - return data + if run.latest is not None: + self._process_guardrail_metadata(run.latest, skip_guardrails=run.skip_guardrails) + return run.latest except SensitiveDataRouteException as e: - data = await self._handle_sensitive_data_route_exception(e, data, user_api_key_dict) - if data is not None: - self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails) - return data + rerouted: Final = await self._handle_sensitive_data_route_exception(e, run.latest, run.user_api_key_dict) + if rerouted is not None: + self._process_guardrail_metadata(rerouted, skip_guardrails=run.skip_guardrails) + return rerouted except Exception: - if data is not None: - self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails) + if run.latest is not None: + self._process_guardrail_metadata(run.latest, skip_guardrails=run.skip_guardrails) raise - async def _run_parallel_pre_call_guardrails( + def _pre_call_start(self, run: _PreCallRun, callback: object, request: object) -> Awaitable[Patch | None] | None: + data: Final = _request_dict(request) + if isinstance(callback, CustomGuardrail) and adapter_for(callback).parallel: + return self._pre_call_start_parallel(run, callback, data) + return self._pre_call_sequential(run, callback, data) + + async def _pre_call_sequential( self, - guardrails: tuple[CustomGuardrail, ...], - data: dict, - raw_request_snapshot: dict | None, # mutable-ok: same request-payload shape as data - user_api_key_dict: UserAPIKeyAuth, - call_type: CallTypesLiteral, - ) -> None: - """ - Run opted-in pre_call guardrails concurrently against one shared payload - snapshot. These guardrails are declared block-only, so any modified data - they return is discarded; they run for their blocking side effect (raising - to reject the request before it reaches the LLM). Every guardrail is - awaited to completion (``return_exceptions=True``) so a raise by one never - leaves the others running as unobserved background tasks. A guardrail that - blocks (any exception other than a reroute or passthrough) takes precedence - over one that only changes the request flow, so a fast reroute can never - let a slower block be bypassed; the request is rejected before it reaches - the LLM, preserving the pre-call barrier that ``during_call`` guardrails - cannot provide. Per-guardrail latency is recorded by - ``_process_guardrail_callback``'s own metrics. - - A guardrail that also opted into ``scan_raw_request`` evaluates - ``raw_request_snapshot`` (taken before the sequential loop ran) instead - of ``data`` (the sequential loop's output), for the same reason the - sequential branch does: its block decision must not depend on what a - sequential guardrail already masked or rewrote. - """ - - def _input_for(callback: CustomGuardrail) -> dict: # mutable-ok: same request-payload shape as data - if not callback.scan_raw_request or raw_request_snapshot is None: - return data - return independent_snapshot(raw_request_snapshot) - - results: Final = await asyncio.gather( - *( - self._process_guardrail_callback( + run: _PreCallRun, + callback: object, + data: dict[str, object] | None, # mutable-ok: the live proxy request dict + ) -> Patch | None: + start_time: Final = time.time() + updated: dict[str, object] | None = data # rebind-ok: a hook may replace the request # mutable-ok: request + try: + if isinstance(callback, CustomGuardrail) and data is not None: + if run.skip_guardrails: + return None + if callback.guardrail_name and callback.guardrail_name in run.pipeline_managed: + return None + updated = await self._run_sequential_guardrail_callback( # rebind-ok: see above callback=callback, - data=_input_for(callback), - user_api_key_dict=user_api_key_dict, - call_type=call_type, - event_type=GuardrailEventHooks.pre_call, + data=data, + raw_request_snapshot=run.raw_request_snapshot, + user_api_key_dict=run.user_api_key_dict, + call_type=run.call_type, ) - for callback in guardrails - ), - return_exceptions=True, - ) - for callback, result in zip(guardrails, results, strict=True): - # _process_guardrail_callback stamped mark_pre_call_hook_ran on - # _input_for's throwaway snapshot copy for a scan_raw_request - # guardrail, never on the live, shared `data` -- without this, a - # deployment-level guardrail sharing this name would see no marker - # via _pre_call_hook_already_ran and re-run it a second time on - # live kwargs. - if callback.scan_raw_request and not isinstance(result, BaseException) and result is not None: - callback.mark_pre_call_hook_ran(data) - if isinstance(result, BaseException) and not isinstance(result, SensitiveDataRouteException): - _record_raising_guardrail(data, callback) - raised: Final = tuple(result for result in results if isinstance(result, BaseException)) - blocking: Final = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None) - if blocking is not None: - raise blocking - if raised: - raise raised[0] + elif ( + isinstance(callback, CustomLogger) + and (not run.guardrails_only or callback.enforces_request_content) + and _overrides_hook(callback, "async_pre_call_hook") + ): + if run.call_type in MCP_GUARDRAIL_CALL_TYPES and run.user_api_key_dict is None: + return None + response: Final[Exception | str | Mapping[str, object] | None] = ( + await callback.async_pre_call_hook( + user_api_key_dict=run.user_api_key_dict, + cache=self.call_details["user_api_key_cache"], + data=data, + call_type=run.call_type, + endpoint_type=run.endpoint_type, + ) + if isinstance(callback, PROXY_MaxParallelRequestsHandler_v3) + else await callback.async_pre_call_hook( + user_api_key_dict=run.user_api_key_dict, + cache=self.call_details["user_api_key_cache"], + data=data, + call_type=run.call_type, + ) + ) + if response is not None: + updated = await self.process_pre_call_hook_response( # rebind-ok: see above + response=response, data=data, call_type=run.call_type + ) + except SensitiveDataRouteException as e: + if run.deferred is None: + run.deferred = e + + end_time: Final = time.time() + duration: Final = end_time - start_time + if hasattr(self, "service_logging_obj") and duration > 0.01: + await self.service_logging_obj.async_service_success_hook( + service=ServiceTypes.PROXY_PRE_CALL, + duration=duration, + call_type=f"{callback.__class__.__name__}", + parent_otel_span=getattr(run.user_api_key_dict, "parent_otel_span", None), + start_time=start_time, + end_time=end_time, + ) + run.latest = updated + return kwargs_patch(data, updated) + + async def _pre_call_deferred_reroute( + self, run: _PreCallRun, fields: Mapping[Field, object] + ) -> Mapping[Field, object]: + data: Final = _request_dict(fields[Field.deprecated_kwargs]) + if run.deferred is None or data is None: + return fields + rerouted: Final = await self._handle_sensitive_data_route_exception(run.deferred, data, run.user_api_key_dict) + run.latest = rerouted + return apply_patch(fields, Patch().set(Field.deprecated_kwargs, rerouted)) + + def _pre_call_start_parallel( + self, + run: _PreCallRun, + callback: CustomGuardrail, + data: dict[str, object] | None, # mutable-ok: the live proxy request dict + ) -> Awaitable[None] | None: + if run.skip_guardrails or data is None: + return None + if callback.guardrail_name and callback.guardrail_name in run.pipeline_managed: + return None + slot: Final[list[object]] = [] # mutable-ok: filled with this guardrail's result or exception + run.parallel.append((callback, slot)) + return self._pre_call_parallel(run, callback, _parallel_input(callback, data, run.raw_request_snapshot), slot) + + async def _pre_call_parallel( + self, + run: _PreCallRun, + callback: CustomGuardrail, + data: dict[str, object], # mutable-ok: the live request or a raw-request snapshot + slot: list[object], # mutable-ok: see _pre_call_start_parallel + ) -> None: + try: + result: Final = await self._process_guardrail_callback( + callback=callback, + data=data, + user_api_key_dict=run.user_api_key_dict, + call_type=run.call_type, + event_type=GuardrailEventHooks.pre_call, + ) + except BaseException as e: + slot.append(e) + if _exception_changes_request_flow(e): + return + raise + slot.append(result) async def _handle_sensitive_data_route_exception( self, @@ -3676,8 +3846,12 @@ class ProxyLogging: has_moderation_override = False iterator_overrides: Final[list[tuple[CustomLogger, str]]] = [] # (callback, kind) resolved_callbacks: Final[list[CustomLogger]] = [] + hook_callbacks: Final[list[object]] = [] # mutable-ok: built once per callback-list signature for callback in callbacks: + if isinstance(callback, CustomLoggerV2): + hook_callbacks.append(callback) + continue if isinstance(callback, str): resolved = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class( cast(_custom_logger_compatible_callbacks_literal, callback) @@ -3687,6 +3861,7 @@ class ProxyLogging: if resolved is None or not isinstance(resolved, CustomLogger): continue resolved_callbacks.append(resolved) + hook_callbacks.append(resolved) cls = type(resolved) if cls is CustomLogger: continue @@ -3694,38 +3869,16 @@ class ProxyLogging: has_guardrail = True elif _overrides_moderation_hook(resolved): has_moderation_override = True - # Use the same leaf-class ``__dict__`` check as the other hook - # capabilities: only callbacks that actually override the hook - # contribute to the flag. Setting this for every ``CustomLogger`` - # instance (the prior behaviour) forced the full - # ``post_call_response_headers_hook`` body to run on every request - # even when no registered callback customized response headers. - cls_attrs = cls.__dict__ - if "async_post_call_response_headers_hook" in cls_attrs: + if _overrides_hook(resolved, "async_post_call_response_headers_hook"): has_post_call_response_headers = True - if "async_post_call_streaming_iterator_hook" in cls_attrs: + if _overrides_hook(resolved, "async_post_call_streaming_iterator_hook"): has_iterator_override = True iterator_overrides.append((resolved, "override")) - elif "apply_guardrail" in cls_attrs and not getattr(resolved, "use_native_lifecycle_hooks", False): + elif isinstance(resolved, CustomGuardrail) and routes_through_apply_guardrail(resolved): iterator_overrides.append((resolved, "apply_guardrail")) - # Walk the MRO for ``async_post_call_streaming_hook`` rather than - # using the leaf-class ``__dict__`` check used by the other flags: - # before this PR the hook was unconditionally invoked, so a - # callback that inherits an override from an intermediate parent - # (e.g. a vendor base class providing the override, with the - # registered class adding nothing else) MUST still be detected. - # A leaf-class miss here would silently drop the inherited hook. - base_streaming_hook = CustomLogger.async_post_call_streaming_hook - cls_streaming_hook = getattr( - cls, - "async_post_call_streaming_hook", - base_streaming_hook, - ) - if getattr(cls_streaming_hook, "__func__", cls_streaming_hook) is not getattr( - base_streaming_hook, "__func__", base_streaming_hook - ): + if _overrides_hook(resolved, "async_post_call_streaming_hook"): has_streaming_chunk_override = True - if "async_pre_call_hook" in cls_attrs: + if _overrides_hook(resolved, "async_pre_call_hook"): has_pre_call_override = True if resolved.enforces_request_content is True: has_content_enforcer = True @@ -3741,6 +3894,8 @@ class ProxyLogging: has_moderation_override=has_moderation_override, iterator_overrides=tuple(iterator_overrides), resolved_callbacks=tuple(resolved_callbacks), + hook_callbacks=tuple(hook_callbacks), + has_v2=len(hook_callbacks) > len(resolved_callbacks), listed_models_filters=tuple( callback for callback in resolved_callbacks if _overrides_hook(callback, "async_filter_listed_models") ), @@ -3807,70 +3962,63 @@ class ProxyLogging: guardrails_only: bool = False, ): caps: Final = ProxyLogging.callback_capabilities() - if not caps.has_guardrail and not caps.has_moderation_override: + if not caps.has_guardrail and not caps.has_moderation_override and not caps.has_v2: return data - # Step 1: Collect all guardrail tasks to run in parallel - guardrail_tasks: Final = [] + subscribers: Final = hook_subscribers(Hook.during_call, litellm.callbacks, include_v2=not guardrails_only) - for callback in litellm.callbacks: - if ( - isinstance(callback, CustomLogger) - and not isinstance(callback, CustomGuardrail) - and (not guardrails_only or callback.enforces_request_content) - and _overrides_moderation_hook(callback) - and user_api_key_dict is not None - ): - guardrail_tasks.append( - callback.async_moderation_hook( - data=data, - user_api_key_dict=user_api_key_dict, - call_type=call_type, - ) - ) - elif isinstance(callback, CustomGuardrail): - ################################################################ - # Check if guardrail should be run for GuardrailEventHooks.during_call hook - ################################################################ - - # V1 implementation - backwards compatibility - if callback.event_hook is None and hasattr(callback, "moderation_check"): - if callback.moderation_check == "pre_call": - continue - else: - # Main - V2 Guardrails implementation - from litellm.types.guardrails import GuardrailEventHooks - - event_type = GuardrailEventHooks.during_call - if call_type == CallTypes.call_mcp_tool.value: - event_type = GuardrailEventHooks.during_mcp_call - - if callback.should_run_guardrail(data=data, event_type=event_type) is not True: - continue - # Convert user_api_key_dict to proper format for async_moderation_hook - if call_type == CallTypes.call_mcp_tool.value: - user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict(user_api_key_dict) - else: - user_api_key_auth_dict = user_api_key_dict - guardrail_tasks.append( - self._run_during_call_guardrail( - callback=callback, - data=data, - user_api_key_dict=user_api_key_dict, - user_api_key_auth_dict=user_api_key_auth_dict, - call_type=call_type, - ) - ) - - # Step 2: Run all guardrail tasks in parallel - if guardrail_tasks: - try: - await asyncio.gather(*guardrail_tasks) - except Exception as e: - # If any guardrail raises an exception, it will propagate here - raise e + def start(callback: object, request: object) -> Awaitable[object] | None: + return self._start_during_call(callback, request, user_api_key_dict, call_type, guardrails_only) + _ = kwargs_after(await run_subscribers(subscribers, KwargsFields(data, PROXY_REQUEST_KEYS), start)) return data + def _start_during_call( + self, + callback: object, + request: object, + user_api_key_dict: UserAPIKeyAuth | None, + call_type: CallTypesLiteral, + guardrails_only: bool, + ) -> Awaitable[object] | None: + data: Final = _request_dict(request) + if data is None: + return None + if ( + isinstance(callback, CustomLogger) + and not isinstance(callback, CustomGuardrail) + and (not guardrails_only or callback.enforces_request_content) + and _overrides_moderation_hook(callback) + and user_api_key_dict is not None + ): + return callback.async_moderation_hook(data=data, user_api_key_dict=user_api_key_dict, call_type=call_type) + if not isinstance(callback, CustomGuardrail): + return None + if callback.event_hook is None and hasattr(callback, "moderation_check"): + if callback.moderation_check == "pre_call": + return None + elif ( + callback.should_run_guardrail( + data=data, + event_type=GuardrailEventHooks.during_mcp_call + if call_type == CallTypes.call_mcp_tool.value + else GuardrailEventHooks.during_call, + ) + is not True + ): + return None + user_api_key_auth_dict: Final = ( + self._convert_user_api_key_auth_to_dict(user_api_key_dict) + if call_type == CallTypes.call_mcp_tool.value + else user_api_key_dict + ) + return self._run_during_call_guardrail( + callback=callback, + data=data, + user_api_key_dict=user_api_key_dict, + user_api_key_auth_dict=user_api_key_auth_dict, + call_type=call_type, + ) + async def _run_during_call_guardrail( self, callback: CustomGuardrail, @@ -3880,8 +4028,7 @@ class ProxyLogging: call_type: CallTypesLiteral, ) -> None: if ( - "apply_guardrail" in type(callback).__dict__ - and not callback.use_native_lifecycle_hooks + routes_through_apply_guardrail(callback) and user_api_key_dict is not None and not callback.use_native_during_call_hook ): @@ -4172,42 +4319,49 @@ class ProxyLogging: redacted_traceback_str: Final = redact_string(traceback_str) if traceback_str is not None else None - # Track the first HTTPException returned or raised by any callback - transformed_exception: HTTPException | None = None + callbacks: Final = litellm.callbacks + run: Final[HookRun[Exception]] = HookRun(response=original_exception) + if not ProxyLogging.callback_capabilities().has_v2: + for callback in callbacks: + await self._post_call_failure_one( + run, request_data, user_api_key_dict, redacted_traceback_str, callback, request_data + ) + return run.transformed + await run_hook_subscribers( + hook_subscribers(Hook.post_call_failure, callbacks, admit=Admit.every_entry), + hook_fields(request_data, run, Field.error), + partial(self._post_call_failure_one, run, request_data, user_api_key_dict, redacted_traceback_str), + ) + return run.transformed - for callback in litellm.callbacks: + async def _post_call_failure_one( + self, + run: HookRun[Exception], + request_data: dict[str, object], # mutable-ok: same request-payload shape as post_call_failure_hook's + user_api_key_dict: UserAPIKeyAuth, + redacted_traceback_str: str | None, + callback: object, + request: object, + ) -> None: + try: + resolved: Final = _resolved_logger(callback) + if resolved is None: + return try: - _callback: CustomLogger | None = None - if isinstance(callback, str): - _callback = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class( - cast(_custom_logger_compatible_callbacks_literal, callback) - ) - else: - _callback = callback - if _callback is not None and isinstance(_callback, CustomLogger): - try: - hook_result = await _callback.async_post_call_failure_hook( - request_data=request_data, - user_api_key_dict=user_api_key_dict, - original_exception=original_exception, - traceback_str=redacted_traceback_str, - ) - # If callback returned an HTTPException, use it (first one wins) - if isinstance(hook_result, HTTPException) and transformed_exception is None: - transformed_exception = hook_result - except HTTPException as e: - # If callback raised an HTTPException, use it (first one wins) - if transformed_exception is None: - transformed_exception = e - except Exception as e: - # Log non-HTTPException errors from callbacks but don't break the flow - verbose_proxy_logger.exception( - "[Non-Blocking] Error in async_post_call_failure_hook callback: %s", e - ) + hook_result: Final = await resolved.async_post_call_failure_hook( + request_data=request_data, + user_api_key_dict=user_api_key_dict, + original_exception=run.response, + traceback_str=redacted_traceback_str, + ) + if isinstance(hook_result, HTTPException): + run.keep_first(hook_result) + except HTTPException as e: + run.keep_first(e) except Exception as e: - verbose_proxy_logger.exception("[Non-Blocking] Error setting up post_call_failure_hook callback: %s", e) - - return transformed_exception + verbose_proxy_logger.exception("[Non-Blocking] Error in async_post_call_failure_hook callback: %s", e) + except Exception as e: + verbose_proxy_logger.exception("[Non-Blocking] Error setting up post_call_failure_hook callback: %s", e) def _is_proxy_only_llm_api_error( self, @@ -4270,6 +4424,8 @@ class ProxyLogging: ) if litellm_logging_obj is None: from litellm._uuid import uuid + from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields + from litellm.proxy.proxy_server import proxy_config request_data.setdefault("litellm_call_id", str(uuid.uuid4())) @@ -4280,6 +4436,7 @@ class ProxyLogging: **_proxy_only_error_log_fields( # pyright: ignore[reportArgumentType] # the allowlist never carries is_async_call request_fields, logged_metadata_keys ), + **key_and_team_logging_fields(user_api_key_dict=user_api_key_dict, proxy_config=proxy_config), ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) @@ -4445,165 +4602,166 @@ class ProxyLogging: """ from litellm.proxy.proxy_server import llm_router - from litellm.types.guardrails import GuardrailEventHooks + run: Final[HookRun[LLMResponseTypes]] = HookRun(response=response) pipeline_response: Final = await self._run_post_call_pipelines( data=data, user_api_key_dict=user_api_key_dict, response=response, ) - if pipeline_response is not None: - response = pipeline_response # rebind-ok: adopt the pipeline's replacement response, same contract as the callback loops below + run.adopt(pipeline_response) pipeline_managed: Final = pipeline_managed_guardrail_names(data, "post_call") request_view: Final = _REQUEST_FIELDS.validate_python(data) guardrail_callbacks, other_callbacks = _partition_post_call_callbacks() - try: - # Merge model-level guardrails before checking which guardrails to run - guardrail_data: Final = check_and_merge_model_level_guardrails(data=data, llm_router=llm_router) - - parallel_guardrails: Final[tuple[CustomGuardrail, ...]] = tuple( - callback - for callback in guardrail_callbacks - if getattr(callback, "run_in_parallel", False) - and not (callback.guardrail_name and callback.guardrail_name in pipeline_managed) - ) - - for callback in guardrail_callbacks: - # Main - V2 Guardrails implementation - - if callback.guardrail_name and callback.guardrail_name in pipeline_managed: - continue - - if getattr(callback, "run_in_parallel", False): - continue - - if ( - callback.should_run_guardrail( - data=guardrail_data, - event_type=GuardrailEventHooks.post_call, - ) - is not True - ): - continue - - if callback.post_call_already_checked(request_view, response) is True: - continue - - guardrail_response: Any | None = None - - if "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks: - data["guardrail_to_apply"] = callback - guardrail_response = await self._run_guardrail_with_metrics( - callback, - unified_guardrail.async_post_call_success_hook( - user_api_key_dict=user_api_key_dict, - data=data, - response=response, - ), - "post_call", - request_data=data, - ) - else: - guardrail_response = await self._run_guardrail_with_metrics( - callback, - callback.async_post_call_success_hook( - user_api_key_dict=user_api_key_dict, - data=data, - response=response, - ), - "post_call", - request_data=data, - ) - - if guardrail_response is not None: - response = guardrail_response - - if parallel_guardrails: - await self._run_parallel_post_call_guardrails( - guardrails=parallel_guardrails, - data=data, - guardrail_data=guardrail_data, - response=response, - user_api_key_dict=user_api_key_dict, - ) - - ############ Handle CustomLogger ############################### - ################################################################# - - for callback in other_callbacks: - callback_response: LLMResponseTypes | None = await callback.async_post_call_success_hook( - user_api_key_dict=user_api_key_dict, data=data, response=response - ) - if callback_response is not None: - response = callback_response - except Exception as e: - raise e - return response - - async def _run_parallel_post_call_guardrails( - self, - guardrails: tuple[CustomGuardrail, ...], - data: dict, - guardrail_data: dict, - response: LLMResponseTypes, - user_api_key_dict: UserAPIKeyAuth, - ) -> None: - """ - Run opted-in post_call guardrails concurrently against the response - produced by the sequential guardrails. These guardrails are declared - block-only, so any modified response they return is discarded; they run - for their blocking side effect (raising to reject the response before it - reaches the client). Every guardrail is awaited to completion - (``return_exceptions=True``) so a raise by one never leaves the others - running as unobserved background tasks. A guardrail that blocks (any - exception other than a passthrough) takes precedence over one that only - changes the response flow, so a fast passthrough can never let a slower - block be bypassed. Each per-guardrail coroutine sets ``guardrail_to_apply`` - immediately before awaiting, and the unified hook pops it before its first - suspension point, so concurrent guardrails never race on that key. - """ - request_view: Final = _REQUEST_FIELDS.validate_python(data) - - async def _run_one(callback: CustomGuardrail) -> None: - if callback.should_run_guardrail(data=guardrail_data, event_type=GuardrailEventHooks.post_call) is not True: - return - if callback.post_call_already_checked(request_view, response) is True: - return - if "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks: - data["guardrail_to_apply"] = callback - await self._run_guardrail_with_metrics( - callback, - unified_guardrail.async_post_call_success_hook( - user_api_key_dict=user_api_key_dict, - data=data, - response=response, - ), - "post_call", - request_data=data, - ) - else: - await self._run_guardrail_with_metrics( - callback, - callback.async_post_call_success_hook( - user_api_key_dict=user_api_key_dict, - data=data, - response=response, - ), - "post_call", - request_data=data, - ) - - results: Final = await asyncio.gather( - *(_run_one(callback) for callback in guardrails), - return_exceptions=True, + guardrails: Final = _PostCallGuardrails( + data=data, + guardrail_data=check_and_merge_model_level_guardrails(data=data, llm_router=llm_router), + request_view=request_view, + pipeline_managed=pipeline_managed, + user_api_key_dict=user_api_key_dict, + ) + if not any(isinstance(callback, CustomLoggerV2) for callback in other_callbacks): + await self._post_call_without_v2(run, guardrails, guardrail_callbacks, other_callbacks) + return run.response + fields: Final = hook_fields(data, run, Field.response) + await run_hook_subscribers( + hook_subscribers(Hook.post_call_success, guardrail_callbacks), + fields, + partial(self._post_call_guardrail_start, run, guardrails), + ) + run.raise_flow_change() + await run_hook_subscribers( + hook_subscribers(Hook.post_call_success, other_callbacks), + fields, + partial(self._post_call_logger, run, data, user_api_key_dict), + ) + return run.response + + async def _post_call_without_v2( + self, + run: HookRun[LLMResponseTypes], + guardrails: "_PostCallGuardrails", + guardrail_callbacks: tuple[CustomGuardrail, ...], + other_callbacks: tuple[CustomLogger, ...], + ) -> None: + flagged: Final = tuple((callback, adapter_for(callback).parallel) for callback in guardrail_callbacks) + for callback, is_parallel in flagged: + if not is_parallel: + await self._post_call_guardrail_begin(run, guardrails, callback, parallel=False) + parallel: Final = tuple(callback for callback, is_parallel in flagged if is_parallel) + if parallel: + started: Final = tuple( + self._post_call_guardrail_begin(run, guardrails, callback, parallel=True) for callback in parallel + ) + results: Final = await asyncio.gather(*started, return_exceptions=True) + raised: Final = next((result for result in results if isinstance(result, BaseException)), None) + if raised is not None: + raise raised + run.raise_flow_change() + for callback in other_callbacks: + if _overrides_hook(callback, "async_post_call_success_hook"): + await self._post_call_logger(run, guardrails.data, guardrails.user_api_key_dict, callback, None) + + def _post_call_guardrail_start( + self, run: HookRun[LLMResponseTypes], guardrails: "_PostCallGuardrails", callback: object, request: object + ) -> Awaitable[None] | None: + if not isinstance(callback, CustomGuardrail): + return None + return self._post_call_guardrail_begin(run, guardrails, callback, parallel=adapter_for(callback).parallel) + + async def _post_call_guardrail_begin( + self, + run: HookRun[LLMResponseTypes], + guardrails: "_PostCallGuardrails", + callback: CustomGuardrail, + parallel: bool, + ) -> None: + if callback.guardrail_name and callback.guardrail_name in guardrails.pipeline_managed: + return + if not parallel: + await self._post_call_sequential_guardrail(run, guardrails, callback) + return + await self._post_call_parallel_guardrail( + run, guardrails, callback, guardrails.parallel_request_view(), run.parallel_slot() + ) + + async def _post_call_sequential_guardrail( + self, run: HookRun[LLMResponseTypes], guardrails: "_PostCallGuardrails", callback: CustomGuardrail + ) -> None: + if ( + callback.should_run_guardrail(data=guardrails.guardrail_data, event_type=GuardrailEventHooks.post_call) + is not True + ): + return + if callback.post_call_already_checked(guardrails.request_view, run.response) is True: + return + run.adopt(await self._post_call_guardrail(guardrails, callback, run.response)) + + async def _post_call_parallel_guardrail( + self, + run: HookRun[LLMResponseTypes], + guardrails: "_PostCallGuardrails", + callback: CustomGuardrail, + request_view: Mapping[str, object], + slot: list[BaseException], # mutable-ok: see _post_call_guardrail_start + ) -> None: + try: + if ( + callback.should_run_guardrail(data=guardrails.guardrail_data, event_type=GuardrailEventHooks.post_call) + is not True + ): + return + if callback.post_call_already_checked(request_view, run.response) is True: + return + _ = await self._post_call_guardrail(guardrails, callback, run.response) + except BaseException as e: + if not _exception_changes_request_flow(e): + raise + slot.append(e) + + async def _post_call_guardrail( + self, guardrails: "_PostCallGuardrails", callback: CustomGuardrail, response: LLMResponseTypes + ) -> LLMResponseTypes | None: + data: Final = guardrails.data + if routes_through_apply_guardrail(callback): + data["guardrail_to_apply"] = callback + return await self._run_guardrail_with_metrics( # pyright: ignore[reportReturnType] # unified hook is untyped + callback, + unified_guardrail.async_post_call_success_hook( + user_api_key_dict=guardrails.user_api_key_dict, + data=data, + response=response, + ), + "post_call", + request_data=data, + ) + return await self._run_guardrail_with_metrics( + callback, + callback.async_post_call_success_hook( + user_api_key_dict=guardrails.user_api_key_dict, + data=data, + response=response, + ), + "post_call", + request_data=data, + ) + + async def _post_call_logger( + self, + run: HookRun[LLMResponseTypes], + data: dict[str, object], # mutable-ok: same request-payload shape as post_call_success_hook's data + user_api_key_dict: UserAPIKeyAuth, + callback: object, + request: object, + ) -> None: + if not isinstance(callback, CustomLogger): + return + run.adopt( + await callback.async_post_call_success_hook( + user_api_key_dict=user_api_key_dict, data=data, response=run.response + ) ) - raised: Final = tuple(result for result in results if isinstance(result, BaseException)) - blocking: Final = next((exc for exc in raised if not _exception_changes_request_flow(exc)), None) - if blocking is not None: - raise blocking - if raised: - raise raised[0] async def post_mcp_call_hook( self, @@ -4627,7 +4785,7 @@ class ProxyLogging: swallowed into an unguarded result. """ caps: Final = ProxyLogging.callback_capabilities() - if not caps.has_guardrail: + if not caps.has_guardrail and not caps.has_v2: return response handler_cls: Final = load_guardrail_translation_mappings().get(CallTypes.call_mcp_tool) @@ -4635,20 +4793,34 @@ class ProxyLogging: verbose_proxy_logger.debug("MCP guardrail translation handler unavailable; skipping post_mcp_call hook") return response - for callback in caps.resolved_callbacks: - if not isinstance(callback, CustomGuardrail): - continue - if "apply_guardrail" not in type(callback).__dict__ or callback.use_native_lifecycle_hooks: - continue - if ( - callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_mcp_call) - is not True - ): - continue - response = await self._run_guardrail_with_metrics( + run: Final[HookRun[CallToolResult]] = HookRun(response=response) + await run_hook_subscribers( + hook_subscribers(Hook.mcp_tool_result, caps.hook_callbacks), + hook_fields(request_data, run, Field.tool_result), + partial(self._post_mcp_call_one, run, handler_cls, request_data, user_api_key_dict), + ) + return run.response + + async def _post_mcp_call_one( + self, + run: HookRun["CallToolResult"], + handler_cls: type["BaseTranslation"], + request_data: Mapping[str, Any], + user_api_key_dict: UserAPIKeyAuth | None, + callback: object, + request: object, + ) -> None: + if not isinstance(callback, CustomGuardrail): + return + if not routes_through_apply_guardrail(callback): + return + if callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_mcp_call) is not True: + return + run.hold( + await self._run_guardrail_with_metrics( callback, handler_cls().process_output_response( - response=response, + response=run.response, guardrail_to_apply=callback, litellm_logging_obj=request_data.get("litellm_logging_obj"), user_api_key_dict=user_api_key_dict, @@ -4657,7 +4829,7 @@ class ProxyLogging: "post_mcp_call", request_data=request_data, ) - return response + ) async def post_call_response_headers_hook( self, @@ -4673,62 +4845,91 @@ class ProxyLogging: Returns: Dict[str, str]: Merged headers from all callbacks. """ - merged_headers: Final[dict[str, str]] = {} - # Outer call sites in common_request_processing.py already gate this - # call with ``has_post_call_response_headers_callbacks()``. The - # cached detection makes the redundant interior guard cheap, but the - # guard would still iterate every code path through this function so - # keep it cheap and rely on the cached capability lookup. - if not ProxyLogging.callback_capabilities().has_post_call_response_headers: + merged_headers: Final[dict[str, str]] = {} # mutable-ok: callbacks merge their headers into it + caps: Final = ProxyLogging.callback_capabilities() + if not caps.has_post_call_response_headers and not caps.has_v2: return merged_headers try: - # Build litellm_call_info — normalized routing metadata for callbacks litellm_call_info: Final = self._build_litellm_call_info(data=data, response=response) - - for callback in litellm.callbacks: - _callback: CustomLogger | None = None - if isinstance(callback, str): - _callback = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class( - cast(_custom_logger_compatible_callbacks_literal, callback) - ) - else: - _callback = callback - - if _callback is not None and isinstance(_callback, CustomLogger): - if _accepts_litellm_call_info(_callback): - result = await _callback.async_post_call_response_headers_hook( - data=data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=request_headers, - litellm_call_info=litellm_call_info, - ) - else: - # Backwards compat: callback doesn't accept litellm_call_info - result = await _callback.async_post_call_response_headers_hook( - data=data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=request_headers, - ) - if result is not None: - merged_headers.update(result) + await run_hook_subscribers( + hook_subscribers(Hook.response_headers, litellm.callbacks, admit=Admit.every_entry), + hook_fields(data, HookRun(response=merged_headers), Field.response_headers), + partial( + self._response_headers_one, + merged_headers, + data, + user_api_key_dict, + response, + request_headers, + litellm_call_info, + ), + ) except Exception as e: verbose_proxy_logger.exception("Error in post_call_response_headers_hook: %s", str(e)) return merged_headers + async def _response_headers_one( + self, + merged_headers: dict[str, str], # mutable-ok: the merged headers this hook returns + data: dict[str, object], # mutable-ok: same request-payload shape as post_call_response_headers_hook's + user_api_key_dict: UserAPIKeyAuth, + response: object, + request_headers: dict[str, str] | None, # mutable-ok: same shape as post_call_response_headers_hook's + litellm_call_info: dict[str, object], # mutable-ok: same shape as _build_litellm_call_info's + callback: object, + request: object, + ) -> None: + resolved: Final = _resolved_logger(callback) + if resolved is None or not _overrides_hook(resolved, "async_post_call_response_headers_hook"): + return + result: Final = ( + await resolved.async_post_call_response_headers_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=request_headers, + litellm_call_info=litellm_call_info, + ) + if _accepts_litellm_call_info(resolved) + else await resolved.async_post_call_response_headers_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=request_headers, + ) + ) + if result is not None: + merged_headers.update(result) + async def hidden_by_listing_callbacks( self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] ) -> frozenset[str]: - filters: Final = ProxyLogging.callback_capabilities().listed_models_filters - if not filters: + caps: Final = ProxyLogging.callback_capabilities() + if not caps.listed_models_filters and not caps.has_v2: return frozenset() candidates: Final = tuple(model_names) - kept: Final = await _names_kept_by_listing_callbacks(filters, user_api_key_dict, candidates) - if isinstance(kept, MalformedListingFilterReturn): - _raise_malformed_listing_filter_return(kept) - return frozenset(candidates).difference(kept) + run: Final[HookRun[tuple[str, ...]]] = HookRun(response=candidates) + await run_hook_subscribers( + hook_subscribers(Hook.list_models, caps.hook_callbacks), + HookRunFields(KwargsFields(_NO_KWARGS, _NO_KEYS), run, Field.model_list), + partial(self._filter_listed_models, run, user_api_key_dict), + ) + return frozenset(candidates).difference(run.response) + + @staticmethod + async def _filter_listed_models( + run: HookRun[tuple[str, ...]], user_api_key_dict: UserAPIKeyAuth, callback: object, request: object + ) -> None: + names: Final = run.response + if not names or not isinstance(callback, CustomLogger): + return + returned: Final = await callback.async_filter_listed_models(user_api_key_dict, names) + try: + kept: Final = frozenset(_LISTED_MODEL_NAMES.validate_python(returned)) + except ValidationError: + _raise_malformed_listing_filter_return(MalformedListingFilterReturn(callback=type(callback).__name__)) + run.hold(tuple(name for name in names if name in kept)) @staticmethod def _build_litellm_call_info(data: dict, response: object) -> dict[str, object]: @@ -5681,7 +5882,6 @@ class PrismaClient: _tool_usage_transactions_lock = asyncio.Lock() model_usage_transactions: ClassVar[list["ModelUsageTransaction"]] = [] _model_usage_transactions_lock = asyncio.Lock() - autorouter_turn_transactions: ClassVar[list["AutoRouterTurnTransaction"]] = [] _autorouter_turn_transactions_lock = asyncio.Lock() # How long a health probe failure waits for an in-flight planned engine @@ -5700,6 +5900,7 @@ class PrismaClient: ): ## init logging object self.baseline_accounting_transactions: list[BaselineAccountingRecord] = [] + self.autorouter_turn_transactions: list[AutoRouterTurnTransaction] = [] self.baseline_accounting_lock: Final = asyncio.Lock() self.proxy_logging_obj = proxy_logging_obj self.token_auth: DatabaseTokenAuth | None = resolve_database_token_auth() @@ -9196,25 +9397,27 @@ async def _run_spend_logs_job( await flush_baseline_accounting(prisma_client) - async with prisma_client._autorouter_turn_transactions_lock: + autorouter_queue_lock: Final = prisma_client._autorouter_turn_transactions_lock + async with autorouter_queue_lock: autorouter_turns_to_process: Final = prisma_client.autorouter_turn_transactions[:MAX_LOGS_PER_INTERVAL] remaining_autorouter_turns: Final = prisma_client.autorouter_turn_transactions[ len(autorouter_turns_to_process) : ] prisma_client.autorouter_turn_transactions = remaining_autorouter_turns # rebind-ok: drain under lock - try: - from litellm.proxy.db.autorouter_session_rollup import flush_autorouter_turn_transactions - - await flush_autorouter_turn_transactions( - prisma_client=prisma_client, - transactions=autorouter_turns_to_process, - ) - except Exception as autorouter_tracking_err: # noqa: BLE001 # a drain bug must not abort the spend job - verbose_proxy_logger.error( - "Spend tracking - auto-router session rollup drain failed; %s turn transactions dropped: %s", - len(autorouter_turns_to_process), - autorouter_tracking_err, + unwritten_autorouter_turns: Final = await _flush_autorouter_turns(prisma_client, autorouter_turns_to_process) + if unwritten_autorouter_turns is not None: + async with autorouter_queue_lock: + prisma_client.autorouter_turn_transactions = [ # rebind-ok: requeue under lock + *unwritten_autorouter_turns.turns, + *prisma_client.autorouter_turn_transactions, + ] + verbose_proxy_logger.warning( + "Spend tracking - database unavailable during auto-router session rollup flush, " + "requeued %s unwritten turn transactions for the next flush: %s", + len(unwritten_autorouter_turns.turns), + unwritten_autorouter_turns.error, ) + raise unwritten_autorouter_turns.error try: from litellm.proxy.db.shadow_eval_funnel import flush_shadow_eval_funnel @@ -9224,6 +9427,22 @@ async def _run_spend_logs_job( verbose_proxy_logger.error("Spend tracking - shadow eval funnel drain failed: %s", funnel_err) +async def _flush_autorouter_turns( + prisma_client: PrismaClient, turns: "Sequence[AutoRouterTurnTransaction]" +) -> "UnwrittenTurns | None": + from litellm.proxy.db.autorouter_session_rollup import flush_autorouter_turn_transactions + + try: + return await flush_autorouter_turn_transactions(prisma_client=prisma_client, transactions=turns) + except Exception as autorouter_tracking_err: # noqa: BLE001 # a drain bug must not abort the spend job + verbose_proxy_logger.error( + "Spend tracking - auto-router session rollup drain failed; %s turn transactions dropped: %s", + len(turns), + autorouter_tracking_err, + ) + return None + + MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20 @@ -9516,7 +9735,11 @@ def _as_aware(moment: datetime) -> datetime: def _reset_window(budget_duration: str, budget_reset_at: datetime) -> tuple[datetime, datetime] | None: window_end: Final = _as_aware(budget_reset_at) - window_start: Final = get_budget_window_start(budget_duration, window_end) + window_start: Final = ( + get_annual_budget_window_start(window_end, get_budget_reset_settings().timezone) + if is_annual_budget_duration(budget_duration) + else get_budget_window_start(budget_duration, window_end) + ) return (window_start, window_end) if window_start < window_end else None @@ -10320,6 +10543,9 @@ async def get_available_models_for_user( prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + only_model_access_groups=only_model_access_groups, + team_object=requested_team_object, + return_wildcard_routes=return_wildcard_routes, ) ) if personal_key_team_model_access_applies(user_api_key_dict) diff --git a/litellm/proxy/vector_store_endpoints/utils.py b/litellm/proxy/vector_store_endpoints/utils.py index 4044e50f8c1..57dbca25f40 100644 --- a/litellm/proxy/vector_store_endpoints/utils.py +++ b/litellm/proxy/vector_store_endpoints/utils.py @@ -1,6 +1,7 @@ import json import re from collections.abc import Iterable, Mapping +from itertools import chain from types import MappingProxyType from typing import Final, Literal, Protocol @@ -13,6 +14,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, LitellmUserRoles, UserAPIKeyAuth, ) @@ -181,6 +183,8 @@ async def can_user_access_vector_store( 4. The caller's team-level object_permission.vector_stores explicitly lists this vector store id. 5. The caller's team_id matches the vector store's team_id. + 6. An access group of the caller's key or team lists this vector store id, + which counts exactly like rules 3 and 4. A dashboard session credential is evaluated against the same effective contexts as listing (its own grants plus each real team of the user). @@ -193,7 +197,9 @@ async def can_user_access_vector_store( return True auth_contexts: Final = await _vector_store_auth_contexts(user_api_key_dict) - return await _is_vector_store_granted_to_any(vector_store, auth_contexts) + return await _is_vector_store_granted_to_any(vector_store, auth_contexts) or ( + vector_store.get("vector_store_id") in await _access_group_vector_store_ids(auth_contexts) + ) async def _is_vector_store_granted( @@ -266,16 +272,54 @@ async def _is_vector_store_granted_to_any( return False +async def _team_or_none(auth_context: UserAPIKeyAuth) -> LiteLLM_TeamTable | None: + """A team that cannot be read has no access groups here, as it has no own list.""" + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if auth_context.team_id is None: + return None + try: + return await get_team_object( + team_id=auth_context.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=auth_context.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: # noqa: BLE001 # no groups, per the docstring + verbose_proxy_logger.debug("Could not read team id=%s for its access groups: %s", auth_context.team_id, e) + return None + + +async def _access_group_vector_store_ids(auth_contexts: tuple[UserAPIKeyAuth, ...]) -> frozenset[str]: + """Groups only add access here, so a group that cannot be read lists nothing.""" + from litellm.proxy.auth.auth_checks import access_group_vector_stores + + listed: Final = [ + await access_group_vector_stores(auth_context, await _team_or_none(auth_context)) + for auth_context in auth_contexts + ] + return frozenset(chain.from_iterable(chain.from_iterable(stores.values()) for stores in listed)) + + async def filter_listable_vector_stores( vector_stores: Iterable[LiteLLM_ManagedVectorStore], user_api_key_dict: UserAPIKeyAuth, ) -> tuple[LiteLLM_ManagedVectorStore, ...]: - """Non-admins only see stores their key, one of their teams' object_permission, or team ownership grants.""" + """Non-admins only see stores their key, a team's object_permission, team ownership or access groups grant.""" if _is_proxy_admin(user_api_key_dict): return tuple(vector_stores) auth_contexts: Final = await _vector_store_auth_contexts(user_api_key_dict) - return tuple([vs for vs in vector_stores if await _is_vector_store_granted_to_any(vs, auth_contexts)]) + group_store_ids: Final = await _access_group_vector_store_ids(auth_contexts) + return tuple( + [ + vs + for vs in vector_stores + if vs.get("vector_store_id") in group_store_ids or await _is_vector_store_granted_to_any(vs, auth_contexts) + ] + ) async def get_litellm_managed_vector_store( diff --git a/litellm/redis/stores/auth_cache/__init__.py b/litellm/redis/stores/auth_cache/__init__.py index 7891344022f..80d590ae7f8 100644 --- a/litellm/redis/stores/auth_cache/__init__.py +++ b/litellm/redis/stores/auth_cache/__init__.py @@ -61,6 +61,19 @@ class AuthCacheStore: self._forget_miss(key) self._handle.set_sync(key, value, _redis_ttl(ttl)) + @with_service_target(AUTH_CACHE_TARGET) + async def lease(self, key: str) -> str | None: + """The generation of ``key`` every worker shares, read on a cache miss before the database read that will + fill it. A delete of the key on any worker starts a new one.""" + return await self._handle.lease(key) + + @with_service_target(AUTH_CACHE_TARGET) + async def set_leased(self, key: str, lease: str | None, value: object, ttl: float | None) -> bool: + """The Redis SET alone, refused with False when ``key`` was deleted on any worker since ``lease`` was read. + Memory is the caller's to write.""" + self._forget_miss(key) + return await self._handle.set_leased(key, lease, value, _redis_ttl(ttl)) + @with_service_target(AUTH_CACHE_TARGET) def declare_set(self, key: str, value: object, ttl: float | None) -> Awaitable[None] | None: """Memory now and the SET on the request's pre-call batch, sent with its next flush; None outside a request, diff --git a/litellm/redis/stores/auth_cache/handle.py b/litellm/redis/stores/auth_cache/handle.py index 9a2d70079d1..f08d53d3ce8 100644 --- a/litellm/redis/stores/auth_cache/handle.py +++ b/litellm/redis/stores/auth_cache/handle.py @@ -13,6 +13,10 @@ class AuthCacheHandle(Protocol): def get_sync(self, key: str) -> object: ... def set(self, key: str, value: object, ttl_seconds: float | None = None) -> Awaitable[None]: ... def set_sync(self, key: str, value: object, ttl_seconds: float | None = None) -> None: ... + def lease(self, key: str) -> Awaitable[str | None]: ... + def set_leased( + self, key: str, lease: str | None, value: object, ttl_seconds: float | None = None + ) -> Awaitable[bool]: ... def declare_set( self, key: str, value: object, ttl_seconds: float | None, send: NativeSend ) -> NativePending[None]: ... diff --git a/litellm/redis/stores/connection.py b/litellm/redis/stores/connection.py index 21e2cc1f6f0..d2be3493e4d 100644 --- a/litellm/redis/stores/connection.py +++ b/litellm/redis/stores/connection.py @@ -5,9 +5,11 @@ from typing import TYPE_CHECKING, Final from litellm.rust_bridge._native import ( NativeAuthCache, + NativeDeploymentHealth, NativeRateLimits, NativeRedisClient, NativeRouterCooldowns, + NativeRouterSessionPins, NativeRouterUsage, NativeSensitiveRoutePins, NativeSpendCounters, @@ -44,6 +46,12 @@ class RedisStores: def sensitive_route_pins(self) -> NativeSensitiveRoutePins: return NativeSensitiveRoutePins(self.client) + def deployment_health(self) -> NativeDeploymentHealth: + return NativeDeploymentHealth(self.client) + + def router_session_pins(self) -> NativeRouterSessionPins: + return NativeRouterSessionPins(self.client) + def connect(redis_cache: RedisCache | None) -> RedisStores: settings: Final = None if redis_cache is None else redis_cache.store_settings diff --git a/litellm/redis/stores/deployment_health/__init__.py b/litellm/redis/stores/deployment_health/__init__.py new file mode 100644 index 00000000000..027cd3d5251 --- /dev/null +++ b/litellm/redis/stores/deployment_health/__init__.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from litellm._internal_context import with_service_target +from litellm.constants import HEALTH_CHECKS_TARGET +from litellm.redis.stores.deployment_health.handle import DeploymentHealthHandle +from litellm.redis.stores.deployment_health.types import DeploymentHealthStates, DeploymentHealthStateValue + +if TYPE_CHECKING: + from litellm.redis.stores.connection import RedisStores + +__all__ = ["DeploymentHealthStateValue", "DeploymentHealthStates", "DeploymentHealthStore"] + + +class DeploymentHealthStore: + """The background health checks' per-deployment states: memory per worker, Redis shared by every pod. + The store owns the key and its encoding; merging and staleness stay with ``DeploymentHealthCache``.""" + + def __init__(self, handle: DeploymentHealthHandle) -> None: + self._handle: Final = handle + + @property + def has_redis(self) -> bool: + return self._handle.has_redis + + def attach_redis(self, redis: RedisStores) -> None: + """Adds the Redis tier once, keeping memory, as ``DualCache.attach_redis_cache`` does.""" + if redis.client is not None: + self._handle.attach(redis.client) + + @with_service_target(HEALTH_CHECKS_TARGET) + async def get(self) -> DeploymentHealthStates | None: + return await self._handle.get() + + @with_service_target(HEALTH_CHECKS_TARGET) + def get_sync(self) -> DeploymentHealthStates | None: + return self._handle.get_sync() + + @with_service_target(HEALTH_CHECKS_TARGET) + def get_shared_sync(self) -> DeploymentHealthStates | None: + """What every pod merged so far, from Redis alone. No Redis, a failed read and an open breaker read as None.""" + return self._handle.get_shared_sync() + + @with_service_target(HEALTH_CHECKS_TARGET) + def set_sync(self, states: DeploymentHealthStates, ttl: float) -> None: + self._handle.set_sync(states, ttl) diff --git a/litellm/redis/stores/deployment_health/handle.py b/litellm/redis/stores/deployment_health/handle.py new file mode 100644 index 00000000000..680032c23db --- /dev/null +++ b/litellm/redis/stores/deployment_health/handle.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from collections.abc import Awaitable +from typing import Protocol + +from litellm.redis.stores.deployment_health.types import DeploymentHealthStates +from litellm.rust_bridge._native import NativeRedisClient + + +class DeploymentHealthHandle(Protocol): + @property + def has_redis(self) -> bool: ... + def attach(self, client: NativeRedisClient) -> bool: ... + def get(self) -> Awaitable[DeploymentHealthStates | None]: ... + def get_sync(self) -> DeploymentHealthStates | None: ... + def get_shared_sync(self) -> DeploymentHealthStates | None: ... + def set_sync(self, states: DeploymentHealthStates, ttl_seconds: float) -> None: ... diff --git a/litellm/redis/stores/deployment_health/types.py b/litellm/redis/stores/deployment_health/types.py new file mode 100644 index 00000000000..b43de912472 --- /dev/null +++ b/litellm/redis/stores/deployment_health/types.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TypeAlias + +from typing_extensions import ReadOnly, TypedDict + + +class DeploymentHealthStateValue(TypedDict): + is_healthy: ReadOnly[bool] + timestamp: ReadOnly[float] + reason: ReadOnly[str] + + +DeploymentHealthStates: TypeAlias = Mapping[str, DeploymentHealthStateValue] diff --git a/litellm/redis/stores/rate_limits/__init__.py b/litellm/redis/stores/rate_limits/__init__.py index 6964cf671bc..a49665dd634 100644 --- a/litellm/redis/stores/rate_limits/__init__.py +++ b/litellm/redis/stores/rate_limits/__init__.py @@ -99,6 +99,24 @@ class RateLimitStore: return local return await self._handle.get(entry, backfill_ttl_seconds=backfill_ttl) + @with_service_target(RATE_LIMITS_TARGET) + async def get_from_redis(self, counter: RateLimitCounter, backfill_ttl: float | None = None) -> LocalValue | None: + """Redis, never memory, for a count every worker must agree on: ``DualCache.async_get_cache`` answers from + memory first, and a worker's own copy carries just the usage it served itself. The count read is written + back to a memory that holds nothing for ``backfill_ttl``, as ``DualCache`` backfilled a miss; a copy the limiter + keeps for the whole window is left alone, so a Redis outage still falls back to it. Without Redis memory is + all there is.""" + if not self.has_redis: + return self._handle.get_local(counter) + _window, count = await self.read_counters([counter], batch=Batch.NONE) + if count is None: + return None + number: Final = float(count) + value: Final[LocalValue] = int(number) if number.is_integer() else number + if self._handle.get_local(counter) is None: + self._handle.set_local(counter, value, backfill_ttl) + return value + @with_service_target(RATE_LIMITS_TARGET) def sliding_window( self, counters: Sequence[RateLimitCounter], now: int, window_size: int, *, batch: Batch diff --git a/litellm/redis/stores/router_session_pins/__init__.py b/litellm/redis/stores/router_session_pins/__init__.py new file mode 100644 index 00000000000..d8ea1dd8bff --- /dev/null +++ b/litellm/redis/stores/router_session_pins/__init__.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Mapping, Sequence +from typing import TYPE_CHECKING, Final + +from litellm._internal_context import with_service_target +from litellm.caching.redis_batch import Batch +from litellm.redis.stores._route import ready, route +from litellm.redis.stores.router_session_pins.handle import RouterSessionPinsHandle +from litellm.redis.stores.router_session_pins.keys import ( + DeploymentPin, + DeploymentSessionPin, + PinKey, + TierModelPin, + TierSessionPin, +) + +if TYPE_CHECKING: + from litellm.redis.stores.connection import RedisStores + +__all__ = [ + "ROUTER_SESSION_PINS_TARGET", + "DeploymentPin", + "DeploymentSessionPin", + "PinKey", + "RouterSessionPinStore", + "TierModelPin", + "TierSessionPin", +] + +ROUTER_SESSION_PINS_TARGET: Final = "router_session_pins" + + +class RouterSessionPinStore: + """The router's session affinity pins (deployment affinity, the complexity router's session and tier pins): + memory per router, Redis shared by every pod. The store owns the claim rule, in Redis and in memory.""" + + def __init__(self, handle: RouterSessionPinsHandle) -> None: + self._handle: Final = handle + + @property + def has_redis(self) -> bool: + return self._handle.has_redis + + def attach_redis(self, redis: RedisStores) -> None: + """Adds the Redis tier once, keeping this router's memory pins, as ``DualCache.attach_redis_cache`` does.""" + if redis.client is not None: + self._handle.attach(redis.client) + + @with_service_target(ROUTER_SESSION_PINS_TARGET) + async def get(self, key: PinKey) -> object: + """``DualCache.async_get_cache``: memory, else Redis written back to memory.""" + if not self.has_redis: + return self._handle.get_local(key) + return await self._handle.get(key) + + def get_local(self, key: PinKey) -> object: + return self._handle.get_local(key) + + def set_local(self, key: PinKey, value: object, ttl: float) -> None: + """Replaces the memory entry, since ``InMemoryCache.set_cache`` keeps a live key's expiry.""" + self._handle.set_local(key, _plain(value), ttl) + + def delete_local(self, key: PinKey) -> None: + self._handle.delete_local(key) + + @with_service_target(ROUTER_SESSION_PINS_TARGET) + async def set(self, key: PinKey, value: object, ttl: float) -> None: + """``DualCache.async_set_cache``: memory, then a ``SET`` of ``json.dumps(value)``.""" + if not self.has_redis: + self.set_local(key, value, ttl) + return + await self._handle.set(key, _plain(value), ttl) + + @with_service_target(ROUTER_SESSION_PINS_TARGET) + async def get_shared(self, key: PinKey) -> object: + """``redis_cache.async_get_cache(raise_on_error=True)``: Redis alone, raising what it raises.""" + return await self._handle.get_shared(key) + + @with_service_target(ROUTER_SESSION_PINS_TARGET) + def read(self, key: PinKey, ttl: float, *, batch: Batch) -> Awaitable[object]: + """The read script on the batch ``batch`` picks: refreshes the pin's TTL, then memory takes what Redis holds + or drops a missing pin. A Redis failure raises, and the caller keeps its memory pin.""" + if not self.has_redis: + return ready(self._handle.get_local(key)) + target: Final = route(batch) + return target.result(self._handle.read(key, ttl, target.send)) + + @with_service_target(ROUTER_SESSION_PINS_TARGET) + async def claim( + self, + key: PinKey, + value: Mapping[str, str], + ttl: float, + *, + eligible_values: Sequence[Mapping[str, str]] | None, + preserve_legacy_model_ids: bool, + ) -> object: + """The claim script: the first writer keeps the key unless it is no longer eligible, and memory takes the + winner. None when Redis answered no pin. A Redis failure raises, and the caller settles with + ``claim_local``.""" + return await self._handle.claim( + key, dict(value), ttl, *_eligibility(eligible_values, preserve_legacy_model_ids) + ) + + def claim_local( + self, + key: PinKey, + value: Mapping[str, str], + ttl: float, + *, + eligible_values: Sequence[Mapping[str, str]] | None, + preserve_legacy_model_ids: bool, + ) -> object: + """``claim`` against this router's memory alone, for a router without Redis or one whose Redis claim + failed. Nothing is awaited between its read and its write, so claims on one loop agree.""" + return self._handle.claim_local( + key, dict(value), ttl, *_eligibility(eligible_values, preserve_legacy_model_ids) + ) + + +def _eligibility( + eligible_values: Sequence[Mapping[str, str]] | None, preserve_legacy_model_ids: bool +) -> tuple[Sequence[object] | None, bool]: + return ( + None if eligible_values is None else [dict(eligible) for eligible in eligible_values], + preserve_legacy_model_ids and eligible_values is not None, + ) + + +def _plain(value: object) -> object: + """A pin as the plain value the native store takes: a mapping becomes the dict it holds.""" + return dict(value) if isinstance(value, Mapping) else value diff --git a/litellm/redis/stores/router_session_pins/handle.py b/litellm/redis/stores/router_session_pins/handle.py new file mode 100644 index 00000000000..5ec2480aea7 --- /dev/null +++ b/litellm/redis/stores/router_session_pins/handle.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Sequence +from typing import Protocol + +from litellm.caching.redis_batch import NativePending +from litellm.redis.stores.router_session_pins.keys import PinKey +from litellm.rust_bridge._native import NativeRedisClient, NativeSend + + +class RouterSessionPinsHandle(Protocol): + @property + def has_redis(self) -> bool: ... + def attach(self, client: NativeRedisClient, /) -> bool: ... + def get(self, key: PinKey) -> Awaitable[object]: ... + def get_local(self, key: PinKey) -> object: ... + def set_local(self, key: PinKey, value: object, ttl_seconds: float) -> None: ... + def delete_local(self, key: PinKey) -> None: ... + def set(self, key: PinKey, value: object, ttl_seconds: float) -> Awaitable[None]: ... + def get_shared(self, key: PinKey) -> Awaitable[object]: ... + def read(self, key: PinKey, ttl_seconds: float, send: NativeSend) -> NativePending[object]: ... + def claim( + self, + key: PinKey, + value: object, + ttl_seconds: float, + eligible_values: Sequence[object] | None, + preserve_legacy_model_ids: bool, + ) -> Awaitable[object]: ... + def claim_local( + self, + key: PinKey, + value: object, + ttl_seconds: float, + eligible_values: Sequence[object] | None, + preserve_legacy_model_ids: bool, + ) -> object: ... diff --git a/litellm/redis/stores/router_session_pins/keys.py b/litellm/redis/stores/router_session_pins/keys.py new file mode 100644 index 00000000000..00d17b0e8c7 --- /dev/null +++ b/litellm/redis/stores/router_session_pins/keys.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TypeAlias + + +@dataclass(frozen=True, slots=True) +class DeploymentPin: + model_group: str + caller: str + + +@dataclass(frozen=True, slots=True) +class DeploymentSessionPin: + model_group: str + caller: str | None + session_id: str + + +@dataclass(frozen=True, slots=True) +class TierSessionPin: + router: str + caller: str | None + session_id: str + + +@dataclass(frozen=True, slots=True) +class TierModelPin: + router: str + tiers: str + caller: str | None + session_id: str + tier: str + replay_fallback: str | None + + +PinKey: TypeAlias = DeploymentPin | DeploymentSessionPin | TierSessionPin | TierModelPin diff --git a/litellm/redis/stores/router_usage/__init__.py b/litellm/redis/stores/router_usage/__init__.py index e9d46f8a68b..3fbae3d42b6 100644 --- a/litellm/redis/stores/router_usage/__init__.py +++ b/litellm/redis/stores/router_usage/__init__.py @@ -1,32 +1,52 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Protocol from litellm._internal_context import with_service_target +from litellm.caching.redis_batch import Batch +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.redis.stores._route import post_call_request, ready, route from litellm.redis.stores.router_usage.handle import RouterUsageHandle from litellm.redis.stores.router_usage.types import ( + MAX_INCREMENT, ROUTER_USAGE_TARGET, + CounterIncrement, CounterKind, + CounterScope, UsageCounter, ) +from litellm.rust_bridge._native import NativeRedisStoreError if TYPE_CHECKING: from litellm.redis.stores.connection import RedisStores __all__ = [ + "MAX_INCREMENT", "ROUTER_USAGE_TARGET", + "CounterIncrement", "CounterKind", + "CounterScope", "RouterUsage", "RouterUsageStore", "UsageCounter", + "UsageRead", ] +@dataclass(frozen=True, slots=True) +class UsageRead: + """A declared usage read, and how to give back the throttle refreshes it reserved.""" + + result: Awaitable[Sequence[float | None]] + release: Callable[[], None] + + class RouterUsage(Protocol): def attach_redis(self, redis: RedisStores) -> None: ... def flush_local_counters(self) -> None: ... - async def increment(self, counter: UsageCounter, value: float, ttl: int) -> float: ... + async def increment(self, counter: UsageCounter, value: float, ttl: int) -> float | None: ... async def increment_deployment( self, deployment_id: str, @@ -36,7 +56,7 @@ class RouterUsage(Protocol): requests: int, ) -> None: ... async def get_many(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: ... - def increment_sync(self, counter: UsageCounter, value: float, ttl: int) -> float: ... + def increment_sync(self, counter: UsageCounter, value: float, ttl: int) -> float | None: ... def increment_deployment_sync( self, deployment_id: str, @@ -46,6 +66,19 @@ class RouterUsage(Protocol): requests: int, ) -> None: ... def get_many_sync(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: ... + @property + def has_redis(self) -> bool: ... + def usage(self, counters: Sequence[UsageCounter], *, batch: Batch) -> UsageRead: ... + async def increment_pipeline(self, increments: Sequence[CounterIncrement]) -> Sequence[float] | None: ... + async def increment_post_call(self, increments: Sequence[CounterIncrement]) -> None: ... + async def increment_remote(self, increments: Sequence[CounterIncrement]) -> Sequence[float] | None: ... + def increment_local(self, increment: CounterIncrement) -> float: ... + def get_local(self, counter: UsageCounter) -> float | None: ... + def get_local_many(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: ... + def set_local(self, counter: UsageCounter, value: float) -> None: ... + async def current(self, counter: UsageCounter, limit: float) -> float | None: ... + def current_sync(self, counter: UsageCounter, limit: float) -> float | None: ... + async def shared_or_local(self, counter: UsageCounter) -> float | None: ... def snapshot(self) -> Mapping[str, float]: ... @@ -66,7 +99,7 @@ class RouterUsageStore: self._handle.flush_local_counters() @with_service_target(ROUTER_USAGE_TARGET) - async def increment(self, counter: UsageCounter, value: float, ttl: int) -> float: + async def increment(self, counter: UsageCounter, value: float, ttl: int) -> float | None: """Without Redis every async method answers from memory without suspending, as ``DualCache``'s did.""" if not self.has_redis: return self.increment_sync(counter, value, ttl) @@ -93,7 +126,7 @@ class RouterUsageStore: return await self._handle.get_many(counters) @with_service_target(ROUTER_USAGE_TARGET) - def increment_sync(self, counter: UsageCounter, value: float, ttl: int) -> float: + def increment_sync(self, counter: UsageCounter, value: float, ttl: int) -> float | None: """``increment`` for a sync call, which has no event loop to await on.""" return self._handle.increment_sync(counter, value, ttl) @@ -112,5 +145,79 @@ class RouterUsageStore: def get_many_sync(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: return self._handle.get_many_sync(counters) + @with_service_target(ROUTER_USAGE_TARGET) + def usage(self, counters: Sequence[UsageCounter], *, batch: Batch) -> UsageRead: + """``get_many`` declared now on the batch ``batch`` picks, so a pick's read can share the request's + pre-call trip. Without Redis it is answered from memory without suspending.""" + if not self.has_redis: + return UsageRead(ready(self.get_many_sync(counters)), lambda: None) + target: Final = route(batch) + declared: Final = self._handle.usage(counters, target.send) + return UsageRead(target.result(declared.read), declared.release) + + @with_service_target(ROUTER_USAGE_TARGET) + async def increment_pipeline(self, increments: Sequence[CounterIncrement]) -> Sequence[float] | None: + """``DualCache.async_increment_cache_pipeline``, on its own trip.""" + if not self.has_redis: + return [self._handle.increment_local(increment) for increment in increments] + return await self._handle.increment_pipeline(increments) + + @with_service_target(ROUTER_USAGE_TARGET) + async def increment_post_call(self, increments: Sequence[CounterIncrement]) -> None: + """``DualCache.async_increment_cache_pipeline_post_call``: memory now, Redis on the request's post-call + pipeline, or on a trip of its own outside a request.""" + request: Final = post_call_request() + if request is None or not self.has_redis: + await self.increment_pipeline(increments) + return + self._handle.increment_post_call(request, increments) + + @with_service_target(ROUTER_USAGE_TARGET) + async def increment_remote(self, increments: Sequence[CounterIncrement]) -> Sequence[float] | None: + """``redis_cache.async_increment_pipeline``: Redis alone, None without it, raising what it raises there, + so ``log_redis_failure`` keeps an open breaker at DEBUG and throttles timeouts.""" + try: + return await self._handle.increment_remote(increments) + except NativeRedisStoreError as e: + match e.args[:1]: + case ("breaker_open",): + raise RedisCircuitBreakerOpenError(*e.args[1:]) from e + case ("timeout",): + raise TimeoutError(*e.args[1:]) from e + case _: + raise + + def increment_local(self, increment: CounterIncrement) -> float: + return self._handle.increment_local(increment) + + def get_local(self, counter: UsageCounter) -> float | None: + return self._handle.get_local(counter) + + def get_local_many(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: + return self._handle.get_local_many(counters) + + def set_local(self, counter: UsageCounter, value: float) -> None: + self._handle.set_local(counter, value) + + @with_service_target(ROUTER_USAGE_TARGET) + async def current(self, counter: UsageCounter, limit: float) -> float | None: + """Memory alone once it reaches ``limit``, else Redis's count. No Redis and an open breaker serve memory, + and any other Redis failure reads as None, as ``RedisCache.async_get_cache`` swallows it.""" + if not self.has_redis: + return self._handle.get_local(counter) + return await self._handle.current(counter, limit) + + @with_service_target(ROUTER_USAGE_TARGET) + def current_sync(self, counter: UsageCounter, limit: float) -> float | None: + return self._handle.current_sync(counter, limit) + + @with_service_target(ROUTER_USAGE_TARGET) + async def shared_or_local(self, counter: UsageCounter) -> float | None: + """What every pod counted in Redis, else this router's memory count when Redis has none, is absent, refuses + or fails.""" + if not self.has_redis: + return self._handle.get_local(counter) + return await self._handle.shared_or_local(counter) + def snapshot(self) -> Mapping[str, float]: return self._handle.snapshot() diff --git a/litellm/redis/stores/router_usage/handle.py b/litellm/redis/stores/router_usage/handle.py index d3e138e4cd1..1b11f750079 100644 --- a/litellm/redis/stores/router_usage/handle.py +++ b/litellm/redis/stores/router_usage/handle.py @@ -3,15 +3,24 @@ from __future__ import annotations from collections.abc import Awaitable, Mapping, Sequence from typing import Protocol -from litellm.redis.stores.router_usage.types import UsageCounter -from litellm.rust_bridge._native import NativeRedisClient +from litellm.caching.redis_batch import NativePending +from litellm.redis.stores.router_usage.types import CounterIncrement, UsageCounter +from litellm.rust_bridge._native import NativeRedisClient, NativeRequestRedis, NativeSend + + +class DeclaredUsage(Protocol): + """A usage read declared on a batch, with the throttle refreshes it reserved.""" + + @property + def read(self) -> NativePending[Sequence[float | None]]: ... + def release(self) -> None: ... class RouterUsageHandle(Protocol): @property def has_redis(self) -> bool: ... def attach(self, client: NativeRedisClient, /) -> bool: ... - def increment(self, counter: UsageCounter, value: float, ttl_seconds: int) -> Awaitable[float]: ... + def increment(self, counter: UsageCounter, value: float, ttl_seconds: int) -> Awaitable[float | None]: ... def increment_deployment( self, deployment_id: str, @@ -21,7 +30,7 @@ class RouterUsageHandle(Protocol): requests: int, ) -> Awaitable[None]: ... def get_many(self, counters: Sequence[UsageCounter]) -> Awaitable[Sequence[float | None]]: ... - def increment_sync(self, counter: UsageCounter, value: float, ttl_seconds: int) -> float: ... + def increment_sync(self, counter: UsageCounter, value: float, ttl_seconds: int) -> float | None: ... def increment_deployment_sync( self, deployment_id: str, @@ -31,5 +40,16 @@ class RouterUsageHandle(Protocol): requests: int, ) -> None: ... def get_many_sync(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: ... + def usage(self, counters: Sequence[UsageCounter], send: NativeSend) -> DeclaredUsage: ... + def increment_pipeline(self, increments: Sequence[CounterIncrement]) -> Awaitable[Sequence[float] | None]: ... + def increment_post_call(self, request: NativeRequestRedis, increments: Sequence[CounterIncrement]) -> None: ... + def increment_remote(self, increments: Sequence[CounterIncrement]) -> Awaitable[Sequence[float] | None]: ... + def increment_local(self, increment: CounterIncrement) -> float: ... + def get_local(self, counter: UsageCounter) -> float | None: ... + def get_local_many(self, counters: Sequence[UsageCounter]) -> Sequence[float | None]: ... + def set_local(self, counter: UsageCounter, value: float) -> None: ... + def current(self, counter: UsageCounter, limit: float) -> Awaitable[float | None]: ... + def current_sync(self, counter: UsageCounter, limit: float) -> float | None: ... + def shared_or_local(self, counter: UsageCounter) -> Awaitable[float | None]: ... def flush_local_counters(self) -> None: ... def snapshot(self) -> Mapping[str, float]: ... diff --git a/litellm/redis/stores/router_usage/types.py b/litellm/redis/stores/router_usage/types.py index 062ee0a084e..a73c2ac38c0 100644 --- a/litellm/redis/stores/router_usage/types.py +++ b/litellm/redis/stores/router_usage/types.py @@ -4,12 +4,33 @@ from dataclasses import dataclass from typing import Final, Literal, TypeAlias ROUTER_USAGE_TARGET: Final = "router_usage" -CounterKind: TypeAlias = Literal["tpm", "rpm"] +CounterKind: TypeAlias = Literal["tpm", "rpm", "itpm", "otpm", "tpd", "rpd"] +CounterScope: TypeAlias = Literal["global_router", "deployment"] +# The native store counts in i64, so a larger amount never crosses into it. +MAX_INCREMENT: Final = 2**63 - 1 @dataclass(frozen=True, slots=True) class UsageCounter: + """One deployment's count in one window: ``HH-MM`` for a per-minute kind, ``YYYY-MM-DD`` for a per-day one. + ``global_router`` counters are the router core's, ``deployment`` ones those usage-based-routing-v2 and the + model rate limit check share.""" + + scope: CounterScope deployment_id: str model: str | None - minute: str + window: str kind: CounterKind + + @staticmethod + def for_deployment(model_id: str, model: str, kind: CounterKind, window: str) -> UsageCounter: + """A ``deployment`` counter. Callers spell the id and model with ``f"{value}"``, as main's + ``f"{model_id}:{model}:..."`` keys did, so a missing one is ``"None"``.""" + return UsageCounter("deployment", model_id, model, window, kind) + + +@dataclass(frozen=True, slots=True) +class CounterIncrement: + counter: UsageCounter + amount: float + ttl: float | None diff --git a/litellm/redis/stores/spend_counters/types.py b/litellm/redis/stores/spend_counters/types.py index 9be029b6953..946b290c2f2 100644 --- a/litellm/redis/stores/spend_counters/types.py +++ b/litellm/redis/stores/spend_counters/types.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import Final, TypeAlias +from datetime import datetime, timezone +from typing import Final, Literal, TypeAlias +from urllib.parse import quote, unquote from litellm.rust_bridge._native import NativeRedisStoreError @@ -13,6 +15,38 @@ SpendIncrement: TypeAlias = tuple[str, float] SpendReading: TypeAlias = tuple[float | None, bool] """(value, authoritative): authoritative when Redis answered, a clean miss included.""" +CustomerCounterScope: TypeAlias = Literal["customer_org", "customer_team", "customer_member"] + + +def customer_window_counter( + scope: CustomerCounterScope, entity_id: str, budget_duration: str, window_start: datetime +) -> str: + """Shared policies have independent balances for each scope, entity and current period.""" + return ( + f"spend:{scope}:{quote(entity_id, safe='')}:window:{budget_duration}" + f"@{window_start.astimezone(timezone.utc).isoformat()}" + ) + + +def parse_customer_window_counter(counter_key: str) -> tuple[CustomerCounterScope, str, str, datetime] | None: + prefix, separator, window = counter_key.rpartition(":window:") + parts: Final = prefix.split(":", 2) + if not separator or len(parts) != 3 or parts[0] != "spend": + return None + scope: Final = parts[1] + if scope not in ("customer_org", "customer_team", "customer_member"): + return None + duration, at, start = window.rpartition("@") + if not at or not duration: + return None + try: + parsed: Final = datetime.fromisoformat(start) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return scope, unquote(parts[2]), duration, parsed + def key_counter(token: str) -> str: return f"spend:key:{token}" diff --git a/litellm/repositories/accounting_delivery_repository.py b/litellm/repositories/accounting_delivery_repository.py index 624c2b8851a..e172e863b63 100644 --- a/litellm/repositories/accounting_delivery_repository.py +++ b/litellm/repositories/accounting_delivery_repository.py @@ -14,6 +14,18 @@ if TYPE_CHECKING: class LedgerTransaction(Protocol): + @property + def litellm_dailycustomermemberspend(self) -> TableActions[object]: ... + + @property + def litellm_customerorganizationtable(self) -> TableActions[object]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[object]: ... + + @property + def litellm_customermembertable(self) -> TableActions[object]: ... + @property def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index eccdf16263d..6b878fa6afc 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -21,8 +21,10 @@ from litellm.models.alert import ( AlertFilter, AlertHistoryPoint, AlertPage, + AlertRecipient, AlertResourceType, AlertStatus, + AlertSummary, AlertView, ) from litellm.repositories.prisma_protocols import PrismaRecord, TableActions @@ -64,6 +66,17 @@ class _HistoryRow(BaseModel): _HISTORY_ROWS: Final = TypeAdapter(tuple[_HistoryRow, ...]) +_SUMMARIES: Final = TypeAdapter(tuple[AlertSummary, ...]) +_RECIPIENTS: Final = TypeAdapter(tuple[AlertRecipient, ...]) + + +class _LatestAlert(BaseModel): + model_config = ConfigDict(frozen=True) + + id: int + + +_LATEST_ALERT: Final = TypeAdapter(tuple[_LatestAlert, ...]) _HISTORY_SOURCES: Final[ Mapping[AlertResourceType, tuple[LiteralString, LiteralString, LiteralString, LiteralString, LiteralString]] ] = { @@ -158,6 +171,45 @@ class AlertRepository: TableActions[PrismaRecord], prisma_client.writer_db.litellm_alert ) + async def latest_id(self) -> int: + async with self._database.tx(timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)") + rows: Final = _LATEST_ALERT.validate_python( + await transaction.query_raw('SELECT COALESCE(MAX(id), 0) AS id FROM "LiteLLM_Alert"') + ) + return rows[0].id + + async def recipients(self, after: str) -> tuple[AlertRecipient, ...]: + async with self._database.tx(timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)") + rows: Final = await transaction.query_raw( + """SELECT user_id, LOWER(TRIM(user_email)) AS email FROM "LiteLLM_UserTable" + WHERE user_role = 'proxy_admin' AND user_id > $1 AND LENGTH(TRIM(user_email)) BETWEEN 3 AND 320 + ORDER BY user_id LIMIT 100""", + after, + ) + return _RECIPIENTS.validate_python(rows) + + async def summaries(self, after: int, through: int) -> tuple[AlertSummary, ...]: + async with self._database.tx(timeout=timedelta(seconds=5)) as transaction: + await transaction.query_raw("SELECT set_config('statement_timeout', '2s', true)") + rows: Final = await transaction.query_raw( + """SELECT a.id::text, a.type, a.resource_type, a.created_at AT TIME ZONE 'UTC' AS created_at, + LEFT(COALESCE(k.key_alias, t.team_alias, u.user_alias, + CASE WHEN a.resource_type = 'model' THEN a.resource_id ELSE a.resource_type END), 120) AS resource_name, + jsonb_build_object('value', a.evidence->'value', 'baseline', a.evidence->'baseline', + 'unit', a.evidence->'unit', 'sample_size', a.evidence->'sample_size') AS evidence + FROM "LiteLLM_Alert" a + LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id + LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id + LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id + WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation' + ORDER BY a.id DESC LIMIT 5""", + after, + through, + ) + return _SUMMARIES.validate_python(rows) + async def record(self, detection: AlertDetection) -> Alert: from prisma import Json from prisma.errors import UniqueViolationError diff --git a/litellm/repositories/budget_repository.py b/litellm/repositories/budget_repository.py index af6c8cfe6f5..3e623c86c0e 100644 --- a/litellm/repositories/budget_repository.py +++ b/litellm/repositories/budget_repository.py @@ -3,7 +3,7 @@ Budget repository for database operations on LiteLLM_BudgetTable. """ from collections.abc import Mapping -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, TypeVar from litellm.models.budget import LiteLLM_BudgetTable from litellm.repositories.base_repository import BaseRepository @@ -14,6 +14,19 @@ if TYPE_CHECKING: from prisma import models as prisma_models +_Row: Final = TypeVar("_Row") + + +async def create_budget_row(table: TableActions[_Row], data: Mapping[str, object], actor: str) -> _Row: + return await table.create(data={**data, "created_by": actor, "updated_by": actor}) + + +async def update_budget_row( + table: TableActions[_Row], budget_id: str, data: Mapping[str, object], actor: str +) -> _Row | None: + return await table.update(where={"budget_id": budget_id}, data={**data, "updated_by": actor}) + + class _BudgetTable(PrismaTableRepository["prisma_models.LiteLLM_BudgetTable"]): table_name = "litellm_budgettable" @@ -66,7 +79,10 @@ class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): if allowed_models is not None: data["allowed_models"] = allowed_models - return await self.create(data) + record: Final = await create_budget_row(self.table, data, created_by) + model: Final = self._to_model(record) + assert model is not None + return model async def update_budget( self, @@ -100,7 +116,8 @@ class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): if allowed_models is not None: data["allowed_models"] = allowed_models - return await self.update(budget_id, data, id_field="budget_id") + record: Final = await update_budget_row(self.table, budget_id, data, updated_by) + return self._to_model(record) async def delete_budget(self, budget_id: str) -> LiteLLM_BudgetTable | None: """Delete a budget record.""" diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index e9973e1f674..744c33cd9f8 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -60,6 +60,7 @@ ROUTER_SETTINGS_PARAM: Final = "router_settings" _JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) _ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) +_BOOL: Final = TypeAdapter(bool) def _stored_router_settings(param_value: object) -> Mapping[str, JsonValue]: @@ -138,6 +139,13 @@ class ConfigRepository: ) return ConfigParam(param_name=param_name, param_value=param_value) + async def model_deprecations_enabled(self) -> bool: + record: Final = await self.get_param("model_deprecation_alerts_enabled") + return True if record is None else _BOOL.validate_python(record.param_value, strict=True) + + async def set_model_deprecations_enabled(self, enabled: bool) -> None: + await self.set_param("model_deprecation_alerts_enabled", enabled) + async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool: rows: Final = await self._writer.query_raw( """INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 9aa0950785b..d8b4d1eb8d5 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -96,6 +96,9 @@ class _DailyActivityDatabase(Protocol): @property def litellm_dailyenduserspend(self) -> TableActions[DailyActivityRow]: ... + @property + def litellm_dailycustomermemberspend(self) -> TableActions[DailyActivityRow]: ... + @property def litellm_dailyagentspend(self) -> TableActions[DailyActivityRow]: ... @@ -178,6 +181,8 @@ def _daily_rows_table( return prisma_client.db.litellm_dailyorganizationspend if table is DailyActivityTable.CUSTOMER: return prisma_client.db.litellm_dailyenduserspend + if table is DailyActivityTable.CUSTOMER_MEMBER: + return prisma_client.db.litellm_dailycustomermemberspend if table is DailyActivityTable.AGENT: return prisma_client.db.litellm_dailyagentspend assert_never(table) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index dc3ca19bb30..4b575761e19 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -43,6 +43,7 @@ PRISMA_TO_PG_TABLE: Final[Mapping[DailyActivityTable, str]] = MappingProxyType( DailyActivityTable.TAG: "LiteLLM_DailyTagSpend", DailyActivityTable.ORGANIZATION: "LiteLLM_DailyOrganizationSpend", DailyActivityTable.CUSTOMER: "LiteLLM_DailyEndUserSpend", + DailyActivityTable.CUSTOMER_MEMBER: "LiteLLM_DailyCustomerMemberSpend", DailyActivityTable.AGENT: "LiteLLM_DailyAgentSpend", } ) diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py index 6d55670e0b8..9eaaed77d3c 100644 --- a/litellm/repositories/performance_alert_repository.py +++ b/litellm/repositories/performance_alert_repository.py @@ -24,6 +24,7 @@ from litellm.models.performance_alert import ( ) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.repositories.alert_repository import record_alerts +from litellm.repositories.config_repository import ConfigRepository if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -161,6 +162,7 @@ class PerformanceAlertRepository: ) self._enterprise: Final = enterprise self._configured: Final = configured + self._config: Final = ConfigRepository(prisma_client, use_writer=True) async def _monitoring(self, transaction: _Transaction, *, lock: bool = False) -> AlertMonitoring: query: Final = 'SELECT enabled, alerts_used FROM "LiteLLM_PerformanceAlertState" WHERE id = 1' @@ -190,7 +192,14 @@ class PerformanceAlertRepository: async def monitoring(self) -> AlertMonitoring: async with self._transaction() as transaction: - return await self._monitoring(transaction) + monitoring: Final = await self._monitoring(transaction) + return monitoring.model_copy( + update={"model_deprecations_enabled": await self._config.model_deprecations_enabled()} + ) + + async def set_model_deprecations_enabled(self, enabled: bool) -> AlertMonitoring: + await self._config.set_model_deprecations_enabled(enabled) + return await self.monitoring() async def set_enabled(self, enabled: bool) -> AlertMonitoring: async with self._database.tx(timeout=timedelta(seconds=10)) as transaction: @@ -203,7 +212,7 @@ class PerformanceAlertRepository: updated_at = now() AT TIME ZONE 'UTC' WHERE id = 1""", enabled, ) - return await self._monitoring(transaction) + return await self.monitoring() @asynccontextmanager async def _transaction(self) -> AsyncGenerator[_Transaction]: diff --git a/litellm/repositories/prisma_protocols.py b/litellm/repositories/prisma_protocols.py index 63cfcbc7b45..2c80eca2749 100644 --- a/litellm/repositories/prisma_protocols.py +++ b/litellm/repositories/prisma_protocols.py @@ -141,6 +141,15 @@ class BatchTable(Protocol): class PrismaBatch(Protocol): + @property + def litellm_customermembertable(self) -> BatchTable: ... + + @property + def litellm_customerteamtable(self) -> BatchTable: ... + + @property + def litellm_customerorganizationtable(self) -> BatchTable: ... + @property def litellm_verificationtoken(self) -> BatchTable: ... diff --git a/litellm/repositories/skill_submission_repository.py b/litellm/repositories/skill_submission_repository.py index 03821da042a..673fd7cca8f 100644 --- a/litellm/repositories/skill_submission_repository.py +++ b/litellm/repositories/skill_submission_repository.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma's generated delegates are dynamic from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.models.plugin_upload import ( PluginPublication, @@ -45,6 +45,7 @@ class _StoredProposal(BaseModel): manifest: Mapping[str, object] version: str | None + revision: int = Field(default=0, ge=0) def proposal(self, name: str) -> SkillProposal: return SkillProposal( @@ -92,6 +93,7 @@ class _SubmissionRow(BaseModel): reviewed_by=self.reviewed_by, reviewed_at=self.reviewed_at, rejection_reason=self.rejection_reason, + revision=stored.revision, ) @@ -283,6 +285,71 @@ class SkillSubmissionRepository: ) return None if row is None or row.archive_content is None else row.archive_content.decode() + async def edit( + self, submission_id: str, owner: str, revision: int, upload: PluginPublication + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict | SubmissionLimitExceeded: + from prisma import Base64 + + async with self._database.tx() as transaction: + await transaction.query_raw( + "SELECT 1 AS locked FROM pg_advisory_xact_lock(hashtext('litellm.skill_submissions'), hashtext($1))", + owner, + ) + row: Final = await _owned_pending(transaction, submission_id, owner, revision) + if isinstance(row, (SubmissionMissing, SubmissionConflict)): + return row + if upload.name != row.plugin_name: + return SubmissionConflict("A submission's skill name cannot change.") + proposal_json: Final = _StoredProposal( + manifest=upload.manifest, version=upload.version, revision=revision + 1 + ).model_dump_json() + archive: Final = row.archive_content if upload.content is None else Base64.encode(upload.content) + usage: Final = _USAGE.validate_python( + await transaction.query_raw( + """SELECT COUNT(*) AS count, + COALESCE(SUM(octet_length(proposal_json) + + COALESCE(octet_length(archive_content), 0)), 0) AS storage_bytes + FROM "LiteLLM_SkillSubmission" WHERE submitted_by = $1 AND status = 'pending_review'""", + owner, + ) + )[0] + previous_bytes: Final = len(row.proposal_json.encode()) + ( + 0 if row.archive_content is None else len(row.archive_content.decode()) + ) + incoming_bytes: Final = len(proposal_json.encode()) + (0 if archive is None else len(archive.decode())) + if usage.storage_bytes - previous_bytes + incoming_bytes > _MAX_PENDING_BYTES: + return SubmissionLimitExceeded( + "Pending skill submissions cannot exceed 50 MiB of stored metadata and archives per user." + ) + updated: Final = await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, + data={"proposal_json": proposal_json, "archive_content": archive, "scan_json": None}, + ) + return SubmissionMissing() if updated is None else _submission(updated) + + async def withdraw( + self, submission_id: str, owner: str, revision: int + ) -> SkillSubmission | SubmissionMissing | SubmissionConflict: + async with self._database.tx() as transaction: + row: Final = await _owned_pending(transaction, submission_id, owner, revision) + if isinstance(row, (SubmissionMissing, SubmissionConflict)): + return row + receipt: Final = _StoredProposal( + manifest={"name": row.plugin_name, "source": {}}, + version=None, + revision=revision, + ) + updated: Final = await transaction.litellm_skillsubmission.update( + where={"id": submission_id}, + data={ + "status": "withdrawn", + "proposal_json": receipt.model_dump_json(), + "archive_content": None, + "scan_json": None, + }, + ) + return SubmissionMissing() if updated is None else _submission(updated) + async def review( self, submission_id: str, @@ -291,6 +358,7 @@ class SkillSubmissionRepository: reason: str | None, *, publication_base_url: str | None = None, + expected_revision: int | None = None, ) -> SubmissionReviewResult: from prisma.errors import UniqueViolationError @@ -304,7 +372,9 @@ class SkillSubmissionRepository: return SubmissionMissing() submission: Final = _submission(row) if submission.status != "pending_review": - return SubmissionConflict("This submission has already been reviewed.") + return SubmissionConflict("This submission is no longer pending review.") + if submission.revision != (expected_revision if expected_revision is not None else 0): + return SubmissionConflict("This submission changed. Reload it before reviewing.") content: Final = None if row.archive_content is None else row.archive_content.decode() if approve and content is not None and not publication_base_url: return SubmissionConflict("Uploaded skill approval requires a trusted publication URL.") @@ -344,6 +414,20 @@ class SkillSubmissionRepository: return SubmissionConflict("A skill with this name is already published. Submit a new update proposal.") +async def _owned_pending( + transaction: SubmissionTransaction, submission_id: str, owner: str, revision: int +) -> "prisma_models.LiteLLM_SkillSubmission | SubmissionMissing | SubmissionConflict": + await transaction.query_raw('SELECT id FROM "LiteLLM_SkillSubmission" WHERE id = $1 FOR UPDATE', submission_id) + row: Final = await transaction.litellm_skillsubmission.find_unique(where={"id": submission_id}) + if row is None or not owner or row.submitted_by != owner: + return SubmissionMissing() + if row.status != "pending_review": + return SubmissionConflict("Only pending submissions can be edited or withdrawn.") + if _StoredProposal.model_validate_json(row.proposal_json).revision != revision: + return SubmissionConflict("This submission changed. Reload it before trying again.") + return row + + async def _publish_submission( transaction: PluginPublicationTransaction, row: "prisma_models.LiteLLM_SkillSubmission", diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py index 99d2ef97634..9b8c4330bf4 100644 --- a/litellm/repositories/table_repositories.py +++ b/litellm/repositories/table_repositories.py @@ -77,6 +77,24 @@ class PrismaTableRepository(Generic[RowT_co]): return wrap_table_actions_for_config_sync(actions=actions, table_name=self.table_name) +class CustomerOrganizationRepository(PrismaTableRepository["prisma_models.LiteLLM_CustomerOrganizationTable"]): + table_name = "litellm_customerorganizationtable" + + +class CustomerMemberRepository(PrismaTableRepository["prisma_models.LiteLLM_CustomerMemberTable"]): + table_name = "litellm_customermembertable" + + +class CustomerTeamRepository(PrismaTableRepository["prisma_models.LiteLLM_CustomerTeamTable"]): + table_name = "litellm_customerteamtable" + + +class CustomerBudgetIncreaseRequestRepository( + PrismaTableRepository["prisma_models.LiteLLM_CustomerBudgetIncreaseRequest"] +): + table_name = "litellm_customerbudgetincreaserequest" + + class PolicyRepository(PrismaTableRepository["prisma_models.LiteLLM_PolicyTable"]): table_name = "litellm_policytable" diff --git a/litellm/repositories/unit_of_work.py b/litellm/repositories/unit_of_work.py index 950b98e7b64..0f58c75588e 100644 --- a/litellm/repositories/unit_of_work.py +++ b/litellm/repositories/unit_of_work.py @@ -20,7 +20,7 @@ from contextlib import asynccontextmanager from dataclasses import dataclass from datetime import datetime, timezone from math import isfinite, ulp -from typing import Final, Protocol, runtime_checkable +from typing import Final, Literal, Protocol, TypeAlias, runtime_checkable from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch @@ -236,6 +236,10 @@ _LOCK_BUDGET_ACCESS_GROUPS_SQL: Final = ( 'SELECT 1 AS locked FROM "LiteLLM_ModelAccessGroupBudgetTable" WHERE budget_id = ANY($1::text[]) ' 'ORDER BY access_group_name COLLATE "C" FOR UPDATE' ) +_LOCK_BUDGET_CUSTOMER_ORGS_SQL: Final = ( + 'SELECT 1 AS locked FROM "LiteLLM_CustomerOrganizationTable" WHERE budget_id = ANY($1::text[]) ' + 'ORDER BY customer_org_id COLLATE "C" FOR UPDATE' +) @dataclass(frozen=True, slots=True) @@ -250,6 +254,7 @@ class BudgetLinkedRowLocks: self.batch.execute_raw(_LOCK_BUDGET_PROJECTS_SQL, list(budget_ids)) self.batch.execute_raw(_LOCK_BUDGET_TAGS_SQL, list(budget_ids)) self.batch.execute_raw(_LOCK_BUDGET_ACCESS_GROUPS_SQL, list(budget_ids)) + self.batch.execute_raw(_LOCK_BUDGET_CUSTOMER_ORGS_SQL, list(budget_ids)) @dataclass(frozen=True, slots=True) @@ -287,6 +292,7 @@ class BudgetCascadeUnitOfWork: tags: LinkedSpendResetWrites model_access_groups: LinkedSpendResetWrites projects: LinkedSpendResetWrites + customer_organizations: LinkedSpendResetWrites budgets: BudgetWindowWrites @@ -316,11 +322,26 @@ async def budget_cascade_unit_of_work( tags=LinkedSpendResetWrites(table=batch.litellm_tagtable), model_access_groups=LinkedSpendResetWrites(table=batch.litellm_modelaccessgroupbudgettable), projects=LinkedSpendResetWrites(table=batch.litellm_projecttable), + customer_organizations=LinkedSpendResetWrites(table=batch.litellm_customerorganizationtable), budgets=BudgetWindowWrites(table=batch.litellm_budgettable), ) await batch.commit() +CustomerSpendTable: TypeAlias = Literal["litellm_customerteamtable", "litellm_customermembertable"] + + +@asynccontextmanager +async def customer_spend_reset_unit_of_work( + new_batch: Callable[[], PrismaBatch], table: CustomerSpendTable +) -> AsyncGenerator[LinkedSpendResetWrites, None]: + """One page of customer team or member spend resets, committed on its own like end users: an aliased request + can register a member, so the rows a template governs are unbounded.""" + batch: Final = new_batch() + yield LinkedSpendResetWrites(table=getattr(batch, table)) + await batch.commit() + + @asynccontextmanager async def enduser_spend_reset_unit_of_work( new_batch: Callable[[], PrismaBatch], diff --git a/litellm/router_api.py b/litellm/router_api.py index 38ed81570c9..e46337c37ee 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -58,6 +58,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.messages.dispatch import MessagesResult + from litellm.redis.stores.router_session_pins import RouterSessionPinStore from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.router import Router from litellm.router_rust.surface.model_list import HostedModelList @@ -1548,6 +1549,9 @@ class _HealthHost(Protocol): @property def _core_cooldown_cache(self) -> "CooldownCache": ... + @property + def _core_session_pins(self) -> "RouterSessionPinStore": ... + @property def _core_health_state_cache(self) -> "DeploymentHealthCache": ... @@ -1601,6 +1605,11 @@ class RouterHealth: """The cooldown store (was ``cooldown_cache``).""" return self._router._core_cooldown_cache + @property + def session_pins(self) -> "RouterSessionPinStore": + """The session affinity pins deployment affinity and the complexity router claim and read.""" + return self._router._core_session_pins + @property def health_checks(self) -> "DeploymentHealthCache": """Background health check results (was ``health_state_cache``).""" diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 8d5778b8a63..08afb2cbfb9 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -29,12 +29,13 @@ from functools import partial from time import perf_counter from time import time as unix_time from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, TypeVar, final, overload +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, final, overload -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator, model_validator +from typing_extensions import assert_never -from litellm._internal_context import service_target -from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, HEALTH_CHECKS_TARGET +from litellm._internal_context import request_parent_otel_span, service_target +from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS from litellm.integrations.custom_guardrail import is_guardrail_intervention from litellm.litellm_core_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, @@ -75,8 +76,10 @@ from litellm.litellm_core_utils.total_deadline import ( enforce_total_timeout_scope, ) from litellm.redis.stores.connection import connect -from litellm.redis.stores.router_cooldowns import ROUTER_COOLDOWNS_TARGET, RouterCooldownStore -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage +from litellm.redis.stores.deployment_health import DeploymentHealthStore +from litellm.redis.stores.router_cooldowns import RouterCooldownStore +from litellm.redis.stores.router_session_pins import RouterSessionPinStore +from litellm.redis.stores.router_usage import RouterUsage from litellm.router_api import ( ROUTER_INIT_ARGS, SETTING_FIELDS, @@ -127,8 +130,17 @@ from litellm.router_rust.invoke.streaming import ( ) from litellm.router_rust.invoke.walk import FallbackOutcomes, WalkMetadata, await_all, log_retry_into from litellm.router_rust.invoke.websocket import turns_counted_their_requests -from litellm.router_rust.routing_stores import RoutingStores -from litellm.router_rust.steps.access import caller_fields, selection_refusal +from litellm.router_rust.routing_stores import ( + CACHE_STEP, + CooldownsStep, + CountFailureStep, + CountSuccessStep, + LocalStep, + OutcomesStep, + RoutingStores, + service_target_of, +) +from litellm.router_rust.steps.access import caller_fields, pinned_deployment_wildcard_grant, selection_refusal from litellm.router_rust.steps.callbacks import OptionalChecks, RustOptionalChecks from litellm.router_rust.steps.plugins import SYNC_PATH_MESSAGE, candidate_models, run_plugins from litellm.router_rust.steps.precall import ( @@ -170,7 +182,6 @@ from litellm.router_rust.surface.health import ( logged_call, ) from litellm.router_rust.surface.health_routing import ( - HEALTH_STATE_KEY, HealthRouting, failover_document, fully_unhealthy_model_names, @@ -240,7 +251,13 @@ from litellm.router_utils.kubernetes_pod_discovery import ( ) from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import release_io_token_reservation -from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata +from litellm.router_utils.router_callbacks.track_deployment_metrics import ( + find_deployment_metadata, + get_deployment_failures_for_current_minute, + get_deployment_successes_for_current_minute, + increment_deployment_failures_for_current_minute, + increment_deployment_successes_for_current_minute, +) from litellm.router_utils.routing_read_batch import RoutingReadBatch from litellm.types.router import ( AllowedFailsPolicy, @@ -500,13 +517,6 @@ class RustCoreUnsupported(AttributeError): """A member or request the Rust core does not serve. It is never forwarded to a Python core.""" -def _cache_target(key: str) -> str: - """The service target a routing cache read or write is attributed to, as the Python core's.""" - if key == HEALTH_STATE_KEY: - return HEALTH_CHECKS_TARGET - return ROUTER_COOLDOWNS_TARGET if key.endswith(":cooldown") else ROUTER_USAGE_TARGET - - # --- failures ----------------------------------------------------------------------------------- @@ -1007,10 +1017,6 @@ def _listing_team_id(request: Mapping[str, object]) -> str | None: return team_id if isinstance(team_id, str) else None -# Request fallback settings, which replace the router's for the call. -_REQUEST_FALLBACK_KEYS: Final = ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks", "max_fallbacks") - - def _strings(value: object) -> "tuple[str, ...]": """Every string in ``value``, a name or a fallback list of names and dict entries.""" if isinstance(value, str): @@ -1018,7 +1024,7 @@ def _strings(value: object) -> "tuple[str, ...]": if isinstance(value, Mapping): entries: Final[Mapping[object, object]] = value # pyright: ignore[reportUnknownVariableType] # read as objects return tuple(itertools.chain.from_iterable(map(_strings, itertools.chain.from_iterable(entries.items())))) - if isinstance(value, list | tuple): + if isinstance(value, Sequence): items: Final[Sequence[object]] = value # pyright: ignore[reportUnknownVariableType] # see above return tuple(itertools.chain.from_iterable(map(_strings, items))) # pyright: ignore[reportUnknownArgumentType] # read as objects return () @@ -1030,31 +1036,82 @@ def _routed_model(request: Mapping[str, object]) -> object: return model if model is None or isinstance(model, str) else str(model) -def _start_fields(request: Mapping[str, object]) -> Mapping[str, object]: - """The request fields the Rust core routes on.""" - fields: Final = { - **request_retry_fields(request.get("num_retries")), - **({"cooldown_time": request["cooldown_time"]} if request.get("cooldown_time") is not None else {}), - } - # `_common_checks_available_deployment`: only True names a litellm model. - specific: Final = {"specific_deployment": True} if request.get("specific_deployment") is True else {} - # `record_disable_fallbacks(kwargs, disable_fallbacks is True)`: only True turns them off. - disabled: Final = {"disable_fallbacks": True} if request.get("disable_fallbacks") is True else {} - lists: Final = {key: _plain(request[key]) for key in _REQUEST_FALLBACK_KEYS if key in request} - weights: Final = request.get("_router_weights") - team_id: Final = get_request_team_id(request) - listing_team_id: Final = _listing_team_id(request) +class RequestStartFields(BaseModel): + model_config = ConfigDict(frozen=True) + + fallbacks: Sequence[str | Mapping[str, object]] | None = None + context_window_fallbacks: Sequence[str | Mapping[str, object]] | None = None + content_policy_fallbacks: Sequence[str | Mapping[str, object]] | None = None + max_fallbacks: Annotated[int, Field(ge=0, le=_U32_MAX)] | None = None + specific_deployment: bool = False + disable_fallbacks: bool = False + cooldown_time: object | None = None + router_weights: object | None = Field(default=None, validation_alias="_router_weights") + team_id: str | None = None + listing_team_id: str | None = None + routing_strategy: str | None = None + + @model_validator(mode="before") + @classmethod + def ignore_standalone_fallback_strings(cls, value: object) -> object: + request_fields: Final = as_str_mapping(value) + if request_fields is None: + return value + return { + key: item + for key, item in request_fields.items() + if key not in ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks") + or not isinstance(item, str) + } + + @field_validator("specific_deployment", "disable_fallbacks", mode="before") + @classmethod + def literal_true_only(cls, value: object) -> bool: + return value is True + + +def _request_start_fields(request: Mapping[str, object]) -> RequestStartFields: strategy: Final = request.get("routing_strategy") override: Final = normalize_strategy(strategy) if isinstance(strategy, str | RoutingStrategy) else None + fields: Final = { + **request, + "team_id": get_request_team_id(request), + "listing_team_id": _listing_team_id(request), + "routing_strategy": override, + } + try: + return RequestStartFields.model_validate(fields) + except ValidationError: + import litellm.exceptions as litellm_exceptions + + model: Final = request.get("model") + raise litellm_exceptions.BadRequestError( + message="Invalid router request fields", + model=model if isinstance(model, str) else None, + llm_provider="", + ) from None + + +def _start_fields(request: Mapping[str, object]) -> Mapping[str, object]: + """The request fields the Rust core routes on.""" + normalized: Final = _request_start_fields(request) + excluded: Final = { + name: True + for name, omit in ( + ("specific_deployment", not normalized.specific_deployment), + ("disable_fallbacks", not normalized.disable_fallbacks), + ("cooldown_time", normalized.cooldown_time is None), + ("router_weights", normalized.router_weights is None), + ("team_id", not normalized.team_id), + ("listing_team_id", normalized.listing_team_id is None), + ("routing_strategy", normalized.routing_strategy is None), + ) + if omit + } + request_fields: Final = normalized.model_dump(exclude_unset=True, exclude=excluded) return { - **fields, - **specific, - **({"routing_strategy": override} if override is not None else {}), - **disabled, - **({"router_weights": weights} if weights is not None else {}), - **lists, - **({"team_id": team_id} if team_id else {}), - **({"listing_team_id": listing_team_id} if listing_team_id is not None else {}), + **request_retry_fields(request.get("num_retries")), + **request_fields, **caller_fields(request), } @@ -1109,6 +1166,10 @@ class RustHealth(RouterHealth): def cooldowns(self) -> "CooldownCache": return self._core._cooldowns # pyright: ignore[reportPrivateUsage] # see above + @property + def session_pins(self) -> RouterSessionPinStore: + return self._core._session_pins # pyright: ignore[reportPrivateUsage] # see above + def failures_this_minute(self, deployment_id: str) -> int: count: Final[object] = self.store.get_cache(f"{deployment_id}:fails", local_only=True) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped return int(count) if isinstance(count, int | float) else 0 @@ -1212,6 +1273,7 @@ _REBOUND_STATE: Final = MappingProxyType( { "_core_cache": "_cache", "_core_cooldown_cache": "_cooldowns", + "_core_session_pins": "_session_pins", "_core_health_state_cache": "_health_checks", "_core_slack_alerting_logger": "_slack_alerting_logger", } @@ -1222,6 +1284,7 @@ _HEALTH_SERVED: Final = frozenset( { "store", "cooldowns", + "session_pins", "failures_this_minute", "record_failure", "usage", @@ -1316,6 +1379,10 @@ class _EndpointPort: def routing_cache(self) -> "DualCache": return self._core._cache # pyright: ignore[reportPrivateUsage] # see the class docstring + @property + def router_usage(self) -> RouterUsage: + return self._core._usage # pyright: ignore[reportPrivateUsage] # see the class docstring + @property def selectors(self) -> StrategySelectors: return self._core._selectors # pyright: ignore[reportPrivateUsage] # see the class docstring @@ -1431,7 +1498,10 @@ class RustCore( store=RouterCooldownStore(redis.router_cooldowns()), default_cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the constructor's value, as the Python core passes it ) - self._health_checks: DeploymentHealthCache = resolved.health_routing.health_checks(self._cache) + self._health_checks: DeploymentHealthCache = resolved.health_routing.health_checks( + DeploymentHealthStore(redis.deployment_health()) + ) + self._session_pins: Final = RouterSessionPinStore(redis.router_session_pins()) self._version: int = 1 self._settings: RustSettings | None = None self._tag_filtering_models: tuple[int, bool] | None = None @@ -1439,7 +1509,7 @@ class RustCore( # document router-core last received for them. self._index_written: frozenset[str] = frozenset() self._index_sent: Mapping[str, object] | None = None - self._selectors: Final = StrategySelectors(host, self._cache) + self._selectors: Final = StrategySelectors(host, self._cache, self._usage) self._selectors.init_default(resolved.routing_strategy, resolved.routing_strategy_args) self._selectors.init_groups(_group_selectors(resolved.routing_groups)) self._port: Final = _EndpointPort(self) @@ -1460,7 +1530,7 @@ class RustCore( ctx: Final = RoutingContext(self) # pyright: ignore[reportArgumentType] # the strategies read registry(), settings(), health() and select() self._ctx: Final = ctx strategies: Final = PreRoutingStrategies( - self._hosted, host, ctx, config.get("auto_router_capability_limit"), self._cache + self._hosted, host, ctx, config.get("auto_router_capability_limit"), self._session_pins ) self._hosted.strategies = strategies self._prerouter: Final = PreRouter(strategies, self._hosted, host, ctx, self._cache) @@ -1486,7 +1556,9 @@ class RustCore( self._hosted._core_routing_plugins = list(resolved.routing_plugins) # pyright: ignore[reportPrivateUsage] # the core fills its own host registry self._rebuild() self._join_callback_holders() - self._checks: Final = OptionalChecks(self, host, self._cache, self._hosted, ctx, config.arguments) + self._checks: Final = OptionalChecks( + self, host, self._cache, self._usage, self._session_pins, self._hosted, ctx, config.arguments + ) self._hosted.checks = self._checks strategies.ensure_affinity = self._checks.ensure_deployment_affinity_callback strategies.hand_over_affinity(self._checks.adopt) @@ -2072,6 +2144,8 @@ class RustCore( redis: Final = connect(changes.redis_cache) self._usage.attach_redis(redis) self._cooldowns.cooldown_store.attach_redis(redis) + self._health_checks.store.attach_redis(redis) + self._session_pins.attach_redis(redis) self._checks.prompt_caching_pin_cache.attach_redis_cache(changes.redis_cache) self._prerouter.attach_redis_cache(changes.redis_cache) return ApplyResult( @@ -2826,6 +2900,11 @@ class RustCore( ) fields: dict[str, object] = dict(_start_fields(request)) # mutable-ok: built up below # rebind-ok: same fields.update(self._group_policy_fields(request)) + pinned_grant: Final = pinned_deployment_wildcard_grant( + request.get("model"), fields.get("allowed_models"), get_request_team_id(request), self.registry() + ) + if pinned_grant is not None: + fields["allowed_models"] = list(pinned_grant) if filter_tags: fields["filter_tags"] = True if request_carries_image(messages, request): @@ -2849,11 +2928,14 @@ class RustCore( """The model names a retry policy lookup may run on (each hop looks up the name it calls): the requested one and every name in the fallback lists the walk may take.""" resolved: Final = self._resolved + normalized: Final = _request_start_fields(request) lists: Final = ( resolved.fallbacks, resolved.context_window_fallbacks, resolved.content_policy_fallbacks, - *(request.get(key) for key in _REQUEST_FALLBACK_KEYS), + normalized.fallbacks, + normalized.context_window_fallbacks, + normalized.content_policy_fallbacks, ) return frozenset((*_strings(request.get("model")), *_strings(lists))) @@ -2897,7 +2979,10 @@ class RustCore( overrides if passthrough is None else {**(overrides or {}), "passthrough_on_no_deployment": True} ) # `owning_picks`: least-busy frees every pick this request made but never started. - with owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))): + with ( + request_parent_otel_span(get_parent_otel_span_from_kwargs(request)), + owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))), + ): call, prepared = self._start(plan, request, routed, prepared_already=prepared_already) arm_compaction_state(self._port, plan, prepared) # pyright: ignore[reportArgumentType] # the call's own dict with ( @@ -2975,7 +3060,10 @@ class RustCore( self, plan: Plan, request: Mapping[str, object], overrides: Mapping[str, object] | None = None ) -> object: # The sync routed calls pick with `picks_reserve(True)` inside `owning_picks`. - with owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))): + with ( + request_parent_otel_span(get_parent_otel_span_from_kwargs(request)), + owning_picks(self._selectors.reservations, call_id_of(CALL_IDENTITY.validate_python(request))), + ): call, prepared = self._start(plan, request, overrides) with picks_reserve(plan.endpoint not in _UNRESERVED_OPS): driver, step = self._run_sync(plan, call, prepared, fresh=True) @@ -3514,80 +3602,43 @@ class RustCore( # -- cache steps ----------------------------------------------------------------------------- - async def _batch_read(self, keys: list[str]) -> Sequence[object]: # mutable-ok: the cache's key list - """One batch read: cooldowns and tpm/rpm usage from their stores, every other key from the routing cache.""" - stored: Final = await self._stored_with_usage(keys) - if stored is not None: - return stored - read: Final[object] = await self._cache.async_batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped - return read if isinstance(read, list) else [None] * len(keys) # pyright: ignore[reportUnknownVariableType] # see above - - async def _stored_with_usage(self, keys: Sequence[str]) -> tuple[object, ...] | None: + async def _cooldowns_with_usage(self, step: CooldownsStep) -> object: """A usage-based-routing-v2 pick reads its tpm/rpm counters alongside its cooldowns (``_usage_read_scope``).""" usage_read: Final = _USAGE_READ.get() - if usage_read is None or not keys or not keys[0].endswith(":cooldown"): - return await self._stores.batch_get(keys) + if usage_read is None: + return await self._stores.answer(step) batch, deployments = usage_read - stored, _ = await asyncio.gather(self._stores.batch_get(keys), batch.read_usage(deployments, None)) + stored, _ = await asyncio.gather(self._stores.answer(step), batch.read_usage(deployments, None)) return stored - def _batch_read_sync(self, keys: list[str]) -> Sequence[object]: # mutable-ok: the cache's key list - """``_batch_read`` for a sync pick.""" - stored: Final = self._stores.batch_get_sync(keys) - if stored is not None: - return stored - read: Final[object] = self._cache.batch_get_cache(keys=keys) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache's reads are untyped - return read if isinstance(read, list) else [None] * len(keys) # pyright: ignore[reportUnknownVariableType] # see above + async def _answer_cache(self, raw: Mapping[str, object]) -> object: + step: Final = CACHE_STEP.validate_python(raw) + with service_target(service_target_of(step)): + match step: + case CooldownsStep(): + return _plain(await self._cooldowns_with_usage(step)) + case OutcomesStep() | CountSuccessStep() | CountFailureStep(): + return self._answer_outcomes(step) + case _: + return _plain(await self._stores.answer(step)) - async def _increment_usage(self, keys: Sequence[object], amounts: Sequence[object]) -> None: - try: - counted: Final = await self._stores.increment_usage(keys, amounts) - except Exception as e: # noqa: BLE001 # usage that cannot be written never fails the call, as on the Python router - from litellm._logging import verbose_router_logger - - verbose_router_logger.debug("_increment_deployment_usage: usage was not recorded. Got - %s", e) - return - if not counted: - raise _not_usage(keys) - - async def _answer_cache(self, step: Mapping[str, object]) -> object: - with service_target(_cache_target(str(step.get("key")))): - return await self._answer_cache_in_target(step) - - async def _answer_cache_in_target(self, step: Mapping[str, object]) -> object: - key: Final = str(step.get("key")) - ttl: Final = _ttl(step) - local: Final = step.get("local") is True - stored: Final = None if local else await self._stores.answer(step.get("op"), key, step.get("value"), ttl) - if stored is not None: - return _plain(stored.value) - match step.get("op"): - case "get": - return _plain(await self._cache.async_get_cache(key, local_only=local)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # DualCache's reads are untyped - case "batch_get": - return _plain(await self._batch_read([str(k) for k in _as_list(step.get("keys"))])) - case "set": - if ttl is None: - await self._cache.async_set_cache(key, step.get("value")) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped - else: - await self._cache.async_set_cache(key, step.get("value"), ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above + def _answer_outcomes(self, step: LocalStep) -> object: + """This worker's per-minute outcomes, kept where the Python router's cooldown handlers keep them.""" + ctx: Final = self._core_ctx + match step: + case OutcomesStep(id=model_id): + return [ + get_deployment_successes_for_current_minute(ctx, model_id), + get_deployment_failures_for_current_minute(ctx, model_id), + ] + case CountSuccessStep(id=model_id): + _ = increment_deployment_successes_for_current_minute(ctx, model_id) return None - case "increment": - by: Final = _amount(step.get("value")) - total: Final = ( - await self._cache.async_increment_cache(key, by, local_only=local) # pyright: ignore[reportUnknownMemberType] # see above - if ttl is None - else await self._cache.async_increment_cache(key, by, local_only=local, ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above - ) - return by if total is None else total - case "increment_usage": - await self._increment_usage(_as_list(step.get("keys")), _as_list(step.get("value"))) + case CountFailureStep(id=model_id): + increment_deployment_failures_for_current_minute(ctx, model_id) return None - case "delete": - await self._cache.async_delete_cache(key) - return None - case other: - raise RuntimeError(f"router-core asked for an unknown cache op {other!r}") + case _: + assert_never(step) def run_routing_cache_sync(self, read_or_write: "Callable[[], Coroutine[object, object, object]]") -> object: """A routing-cache coroutine on a sync call, whose `make_call` runs on the one event loop @@ -3608,54 +3659,21 @@ class RustCore( @property def _stores(self) -> RoutingStores: """The stores router-core's cache steps reach, over the cooldowns as last assigned.""" - return RoutingStores(self._cooldowns, self._usage) + return RoutingStores(self._cooldowns, self._usage, self._health_checks) def io_token_usage_sync(self, model_group: str) -> tuple[int | None, int | None]: """``get_model_group_io_token_usage``, whose counters stay in the routing cache, on a sync call.""" read: Final = self.run_routing_cache_sync(lambda: self._hosted.get_model_group_io_token_usage(model_group)) return _IO_TOKEN_USAGE.validate_python(read) - def _answer_cache_sync(self, step: Mapping[str, object]) -> object: - with service_target(_cache_target(str(step.get("key")))): - return self._answer_cache_sync_in_target(step) - - def _answer_cache_sync_in_target(self, step: Mapping[str, object]) -> object: - key: Final = str(step.get("key")) - ttl: Final = _ttl(step) - local: Final = step.get("local") is True - op: Final = step.get("op") - stored: Final = None if local else self._stores.answer_sync(op, key, step.get("value"), ttl) - if stored is not None: - return _plain(stored.value) - match op: - case "get": - return _plain(self._cache.get_cache(key, local_only=local)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # DualCache's reads are untyped - case "batch_get": - return _plain(self._batch_read_sync([str(k) for k in _as_list(step.get("keys"))])) - case "set": - if ttl is None: - self._cache.set_cache(key, step.get("value")) # pyright: ignore[reportUnknownMemberType] # DualCache's writes are untyped - else: - self._cache.set_cache(key, step.get("value"), ttl=ttl) # pyright: ignore[reportUnknownMemberType] # see above - return None - case "increment": - by: Final = _amount(step.get("value")) - # DualCache.increment_cache is annotated int but adds floats as given. - return ( - self._cache.increment_cache(key, by, local_only=local) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # see above - if ttl is None - else self._cache.increment_cache(key, by, local_only=local, ttl=ttl) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # see above - ) - case "increment_usage": - keys: Final = _as_list(step.get("keys")) - if not self._stores.increment_usage_sync(keys, _as_list(step.get("value"))): - raise _not_usage(keys) - return None - case "delete": - self._cache.delete_cache(key) # pyright: ignore[reportUnknownMemberType] # DualCache's delete is untyped - return None - case other: - raise RuntimeError(f"router-core asked for an unknown cache op {other!r}") + def _answer_cache_sync(self, raw: Mapping[str, object]) -> object: + step: Final = CACHE_STEP.validate_python(raw) + with service_target(service_target_of(step)): + match step: + case OutcomesStep() | CountSuccessStep() | CountFailureStep(): + return self._answer_outcomes(step) + case _: + return _plain(self._stores.answer_sync(step)) # -- errors the core raises when no attempt failed ------------------------------------------ @@ -4024,19 +4042,6 @@ def _seconds(value: object, default: float) -> float | int: _IO_TOKEN_USAGE: Final = TypeAdapter(tuple[int | None, int | None]) -def _not_usage(keys: Sequence[object]) -> RuntimeError: - return RuntimeError(f"router-core asked to add usage to keys that are not one deployment's tpm/rpm: {keys!r}") - - -def _ttl(step: Mapping[str, object]) -> float | None: - ttl: Final = step.get("ttl") - return ttl if isinstance(ttl, int | float) else None - - -def _amount(value: object) -> float | int: - return value if isinstance(value, int | float) and not isinstance(value, bool) else 1 - - # The plans whose kwargs carry `original_generic_function`, which the fallback cooldown reads. _GENERIC_API_KINDS: Final = frozenset(("generic", "search")) @@ -5233,9 +5238,8 @@ class _Driver: releases: Final = ( release_started_attempt(core._selectors.reservations, core._cache, attempt.attempt_id), # pyright: ignore[reportPrivateUsage] # the driver serves its own core release_io_token_reservation( - core._cache, # pyright: ignore[reportPrivateUsage] # see above + core._usage, # pyright: ignore[reportPrivateUsage] # see above call.kwargs, # pyright: ignore[reportArgumentType] # the call's kwargs dict - parent_otel_span=get_parent_otel_span_from_kwargs(call.kwargs), ) if refund_io_tokens else None, diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 5b4a65a70b1..496f8093706 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -57,6 +57,7 @@ from litellm.vector_stores.search_query import store_search_params if TYPE_CHECKING: from litellm.caching.dual_cache import DualCache + from litellm.redis.stores.router_usage import RouterUsage from litellm.router_api import RouterRegistry from litellm.router_rust.core import RustCore from litellm.router_rust.steps.selection import StrategySelectors @@ -94,6 +95,9 @@ class EndpointCore(Protocol): @property def routing_cache(self) -> "DualCache": ... + @property + def router_usage(self) -> "RouterUsage": ... + @property def selectors(self) -> "StrategySelectors": ... @@ -893,7 +897,6 @@ def attempt_call( data: Final = _deployment_params(plan, deployment) model_name: Final = str(data.get("model")) _validate_decisions_connection(plan, data, request) - cache: Final = core.routing_cache caching: Final = shaping.cache_responses # A fallback hop routes the target group in a copy of the metadata (`run_async_fallback`). if fallback_level > 0: @@ -910,7 +913,9 @@ def attempt_call( start_silent_experiments(core, plan, silent_model, named.get("messages"), kwargs) if "messages" in named: kwargs.setdefault("messages", named["messages"]) - update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache, splits_stream_timeout=True) + update_kwargs_with_deployment( + shaping, deployment, kwargs, usage=core.router_usage, splits_stream_timeout=True + ) kwargs.pop("silent_model", None) final = without_router_only_kwargs( # rebind-ok: rebound as the Python router rebinds it { @@ -924,7 +929,7 @@ def attempt_call( return AttemptCall(plan.host_op, final, kwargs, str(params.get("model")), group) case "text" | "embedding" | "image" | "transcription" | "rerank" | "adapter": update_kwargs_with_deployment( - shaping, deployment, kwargs, cache=cache, splits_stream_timeout=plan.splits_stream_timeout + shaping, deployment, kwargs, usage=core.router_usage, splits_stream_timeout=plan.splits_stream_timeout ) refused: Final = embedding_batch_refusal(plan, deployment, named.get("input"), kwargs, requested) if refused is not None: @@ -938,7 +943,7 @@ def attempt_call( } return AttemptCall(plan.host_op, without_router_only_kwargs(final), kwargs, model_name, group) case "speech": - update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache) + update_kwargs_with_deployment(shaping, deployment, kwargs, usage=core.router_usage) voice = kwargs.pop("voice", None) # rebind-ok: rebound as the Python router rebinds it final = { # rebind-ok: rebound as the Python router rebinds it **data, @@ -952,7 +957,9 @@ def attempt_call( if plan.kind == "cancel_batch": data.update(_resolved_credentials(core, deployment, requested)) data.pop("litellm_credential_name", None) - update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache, function_name=plan.function_name) + update_kwargs_with_deployment( + shaping, deployment, kwargs, usage=core.router_usage, function_name=plan.function_name + ) final = { # rebind-ok: rebound as the Python router rebinds it **data, "custom_llm_provider": _provider(core, data), @@ -968,7 +975,7 @@ def attempt_call( shaping, deployment, kwargs, - cache=cache, + usage=core.router_usage, function_name="generic_api_call", splits_stream_timeout=invoked_handler_name(core, plan) in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) @@ -984,7 +991,7 @@ def attempt_call( } return AttemptCall(plan.host_op, final, kwargs, model_name, group) case "moderation": - update_kwargs_with_deployment(shaping, deployment, kwargs, cache=cache) + update_kwargs_with_deployment(shaping, deployment, kwargs, usage=core.router_usage) kwargs["model"] = model_name kwargs.update(data) return AttemptCall(plan.host_op, kwargs, kwargs, model_name, group) @@ -1015,7 +1022,7 @@ def _generic_attempt( core.shaping, deployment, kwargs, - cache=core.routing_cache, + usage=core.router_usage, function_name="_ageneric_api_call_with_fallbacks", splits_stream_timeout=handler in STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, opens_responses_socket=socket, @@ -2265,7 +2272,7 @@ class RustEndpoints: shaping, deployment, new_kwargs, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values - cache=self._core.routing_cache, + usage=self._core.router_usage, function_name="aretrieve_batch", ) bucket: Final = metadata_bucket(new_kwargs, metadata_variable_name) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values @@ -2350,7 +2357,7 @@ class RustEndpoints: self._core.shaping, deployment, kwargs_copy, # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values - cache=self._core.routing_cache, + usage=self._core.router_usage, function_name="acreate_file", ) data: Final = as_dict(deployment.get("litellm_params")) diff --git a/litellm/router_rust/invoke/prompt_management.py b/litellm/router_rust/invoke/prompt_management.py index 74a7991b06c..cc66fc16b6a 100644 --- a/litellm/router_rust/invoke/prompt_management.py +++ b/litellm/router_rust/invoke/prompt_management.py @@ -76,7 +76,7 @@ async def prompt_management_acompletion( core.shaping, deployment, kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs - cache=core.routing_cache, + usage=core.router_usage, splits_stream_timeout=True, ) params: Final = as_dict(deployment.get("litellm_params")) diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index bf6410d8242..5fa6fc2543c 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -52,7 +52,7 @@ from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Param from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN if TYPE_CHECKING: - from litellm.caching.dual_cache import DualCache + from litellm.redis.stores.router_usage import RouterUsage from litellm.router_python_host import PythonHost # The SDK-client and max_parallel_requests reads of the routing cache: process-local, so the family @@ -404,7 +404,7 @@ def update_kwargs_with_deployment( deployment: Mapping[str, object], kwargs: MutableMapping[str, object], # mutable-ok: the request kwargs the Python router mutates in place *, - cache: "DualCache | None", + usage: "RouterUsage | None", function_name: str | None = None, splits_stream_timeout: bool = False, opens_responses_socket: bool = False, @@ -467,8 +467,8 @@ def update_kwargs_with_deployment( if isinstance(other_bucket_metadata, Mapping) and KUBERNETES_POD_ROUTING_KEY in other_bucket_metadata: kwargs[other_bucket] = {**as_dict(other_bucket_metadata), KUBERNETES_POD_ROUTING_KEY: routing} # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them - if cache is not None: - refund_stale_reservation_before_retry(cache, kwargs) # pyright: ignore[reportArgumentType] # the request's kwargs dict + if usage is not None: + refund_stale_reservation_before_retry(usage, kwargs) # pyright: ignore[reportArgumentType] # the request's kwargs dict set_io_token_rate_limit_request_kwargs( kwargs, # pyright: ignore[reportArgumentType] # the request's kwargs dict store_in_context=deployment_has_io_token_limits(deployment), # pyright: ignore[reportArgumentType] # a deployment record diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py index b1de96fd91b..797f74e0b26 100644 --- a/litellm/router_rust/invoke/streaming.py +++ b/litellm/router_rust/invoke/streaming.py @@ -1763,11 +1763,11 @@ def _responses_continuation_request( responses_continuation_input, ) + if source_iterator.started_tool_call: + _raise_original_mid_stream_error(e) if _mid_stream_error_before_output(e): return None - if source_iterator.started_tool_call or isinstance( - e.original_exception, litellm_exceptions.ContentPolicyViolationError - ): + if isinstance(e.original_exception, litellm_exceptions.ContentPolicyViolationError): _raise_original_mid_stream_error(e) cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") if constrains_responses_continuation(_params_sent_to_deployment(attempt, cut_deployment_id)): diff --git a/litellm/router_rust/invoke/websocket.py b/litellm/router_rust/invoke/websocket.py index b38252503c7..a787c729641 100644 --- a/litellm/router_rust/invoke/websocket.py +++ b/litellm/router_rust/invoke/websocket.py @@ -14,6 +14,7 @@ from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Final +from litellm._internal_context import request_parent_otel_span from litellm._logging import verbose_router_logger from litellm.constants import USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, without @@ -115,18 +116,18 @@ async def _count_usage_based_routing_tokens( or not isinstance(deployment_name, str) ): return False - await count_before_callbacks( - deployment_metadata, - USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, - total_tokens, - partial( - selector.async_increment_tpm, - deployment_id=str(deployment_id), - deployment_name=deployment_name, - tokens=total_tokens, - parent_otel_span=parent_otel_span, - ), - ) + with request_parent_otel_span(parent_otel_span): + await count_before_callbacks( + deployment_metadata, + USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY, + total_tokens, + partial( + selector.async_increment_tpm, + deployment_id=str(deployment_id), + deployment_name=deployment_name, + tokens=total_tokens, + ), + ) return True @@ -167,7 +168,7 @@ async def admit_deployment_turn( io_token_admission: Final[Mapping[str, object]] = ( MappingProxyType({}) if rate_limit_check is None - else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime()) # pyright: ignore[reportArgumentType] # the configured deployment dict + else await async_admit_io_token_turn(rate_limit_check.usage, deployment, get_utc_datetime()) # pyright: ignore[reportArgumentType] # the configured deployment dict ) budget_limiter: Final[object] = _budget_limiter(core) if budget_limiter is not None: diff --git a/litellm/router_rust/routing_stores.py b/litellm/router_rust/routing_stores.py index af3ef7cd375..f3a6a485a55 100644 --- a/litellm/router_rust/routing_stores.py +++ b/litellm/router_rust/routing_stores.py @@ -2,24 +2,40 @@ from __future__ import annotations from collections.abc import Sequence from dataclasses import dataclass -from typing import Final, TypeAlias +from typing import Annotated, Final, Literal, Protocol, TypeAlias -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter +from typing_extensions import assert_never -from litellm._internal_context import service_target, with_service_target -from litellm.caching.redis_batch import Batch +from litellm._logging import verbose_router_logger +from litellm.constants import HEALTH_CHECKS_TARGET from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.redis.stores.deployment_health import DeploymentHealthStates from litellm.redis.stores.router_cooldowns import ROUTER_COOLDOWNS_TARGET from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage, UsageCounter +from litellm.redis.stores.router_usage.types import CounterKind from litellm.router_utils import routing_prefetch from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue +from litellm.router_utils.health_state_cache import health_states_of +from litellm.router_utils.pre_call_checks.model_rate_limit_check import RoutingArgs -__all__ = ("AllowedFailsKey", "CooldownKey", "Handled", "RoutingStores", "UsageKey", "parse_key") +__all__ = ( + "CACHE_STEP", + "CacheStep", + "CooldownsStep", + "CountFailureStep", + "CountSuccessStep", + "LocalStep", + "OutcomesStep", + "RoutingStores", + "StoreStep", + "service_target_of", +) _COOLDOWN_VALUE: Final = TypeAdapter(CooldownCacheValue) -def _cooldown_value(value: object) -> CooldownCacheValue: +def _cooldown_value(value: JsonValue) -> CooldownCacheValue: """The cooldown router-core wrote, its whole-second ``cooldown_time`` kept an ``int`` as it serialized it.""" validated: Final = _COOLDOWN_VALUE.validate_python(value) written: Final = as_str_mapping(value) @@ -29,82 +45,104 @@ def _cooldown_value(value: object) -> CooldownCacheValue: return validated -@dataclass(frozen=True, slots=True) -class CooldownKey: - model_id: str +class _Step(BaseModel): + model_config = ConfigDict(frozen=True) -@dataclass(frozen=True, slots=True) -class AllowedFailsKey: - model_id: str +class UsageWindow(_Step): + deployment_id: str + model: str + minute: str + + def counter(self, kind: CounterKind) -> UsageCounter: + return UsageCounter("global_router", self.deployment_id, self.model, self.minute, kind) + + +class CooldownsStep(_Step): + op: Literal["cooldowns"] + ids: tuple[str, ...] + + +class PutCooldownStep(_Step): + op: Literal["put_cooldown"] + id: str + cooldown: JsonValue + ttl: float + + +class IncrementAllowedFailsStep(_Step): + op: Literal["increment_allowed_fails"] + id: str suffix: str | None + ttl: float -@dataclass(frozen=True, slots=True) -class UsageKey: - counter: UsageCounter +class AddUsageStep(_Step): + op: Literal["add_usage"] + window: UsageWindow + tokens: int | float + requests: int | float -RoutingKey: TypeAlias = CooldownKey | AllowedFailsKey | UsageKey +class AddRequestStep(_Step): + op: Literal["add_request"] + window: UsageWindow -@dataclass(frozen=True, slots=True) -class Handled: - value: object +class UsageStep(_Step): + op: Literal["usage"] + window: UsageWindow -def _deployment_key(rest: str) -> RoutingKey | None: - if rest.endswith(":cooldown"): - return CooldownKey(rest.removesuffix(":cooldown")) - model_id, marker, suffix = rest.partition(":allowed_fails") - if not marker or (suffix and not suffix.startswith(":")): - return None - return AllowedFailsKey(model_id, suffix.removeprefix(":") or None) +class HealthStatesStep(_Step): + op: Literal["health_states"] -def _usage_key(rest: str) -> UsageKey | None: - parts: Final = rest.rsplit(":", 2) - if len(parts) != 3: - return None - head, kind, minute = parts - model_id, separator, model = head.partition(":") - if not separator: - return None - match kind: - case "tpm" | "rpm": - return UsageKey(UsageCounter(model_id, model, minute, kind)) +class OutcomesStep(_Step): + op: Literal["outcomes"] + id: str + + +class CountSuccessStep(_Step): + op: Literal["count_success"] + id: str + + +class CountFailureStep(_Step): + op: Literal["count_failure"] + id: str + + +StoreStep: TypeAlias = ( + CooldownsStep + | PutCooldownStep + | IncrementAllowedFailsStep + | AddUsageStep + | AddRequestStep + | UsageStep + | HealthStatesStep +) +LocalStep: TypeAlias = OutcomesStep | CountSuccessStep | CountFailureStep +CacheStep: TypeAlias = Annotated[StoreStep | LocalStep, Field(discriminator="op")] +CACHE_STEP: Final[TypeAdapter[CacheStep]] = TypeAdapter(CacheStep) + + +def service_target_of(step: StoreStep | LocalStep) -> str: + """The service target a cache step is attributed to, as the Python core's.""" + match step: + case CooldownsStep() | PutCooldownStep() | IncrementAllowedFailsStep(): + return ROUTER_COOLDOWNS_TARGET + case HealthStatesStep(): + return HEALTH_CHECKS_TARGET + case AddUsageStep() | AddRequestStep() | UsageStep() | OutcomesStep() | CountSuccessStep() | CountFailureStep(): + return ROUTER_USAGE_TARGET case _: - return None + assert_never(step) -def parse_key(key: str) -> RoutingKey | None: - """The store family a router-core key belongs to. Each store rebuilds the same key from what is parsed - here, so an id or model holding a ``:`` still names the key router-core asked for.""" - if key.startswith("deployment:"): - return _deployment_key(key.removeprefix("deployment:")) - if key.startswith("global_router:"): - return _usage_key(key.removeprefix("global_router:")) - return None - - -def _service_target(key: RoutingKey | None) -> str | None: - if isinstance(key, CooldownKey | AllowedFailsKey): - return ROUTER_COOLDOWNS_TARGET - if isinstance(key, UsageKey): - return ROUTER_USAGE_TARGET - return None - - -def _cooldown_ids(keys: Sequence[str]) -> tuple[str, ...] | None: - ids: Final = tuple(parsed.model_id for parsed in map(parse_key, keys) if isinstance(parsed, CooldownKey)) - return ids if keys and len(ids) == len(keys) else None - - -def _usage_counters(keys: Sequence[object]) -> tuple[UsageCounter, ...] | None: - counters: Final = tuple( - parsed.counter for parsed in (parse_key(str(key)) for key in keys) if isinstance(parsed, UsageKey) - ) - return counters if keys and len(counters) == len(keys) else None +class HealthStates(Protocol): + async def async_get_states(self) -> DeploymentHealthStates | None: ... + def get_states(self) -> DeploymentHealthStates | None: ... def _aligned(model_ids: Sequence[str], active: Sequence[tuple[str, CooldownCacheValue]]) -> tuple[object, ...]: @@ -112,119 +150,67 @@ def _aligned(model_ids: Sequence[str], active: Sequence[tuple[str, CooldownCache return tuple(found.get(model_id) for model_id in model_ids) -def _one_deployment(tpm: UsageCounter, rpm: UsageCounter) -> bool: - return (tpm.kind, rpm.kind) == ("tpm", "rpm") and (tpm.deployment_id, tpm.model, tpm.minute) == ( - rpm.deployment_id, - rpm.model, - rpm.minute, - ) - - -def _deployment_increment(keys: Sequence[object], amounts: Sequence[object]) -> tuple[UsageCounter, float, int] | None: - """``_increment_deployment_usage``'s tpm then rpm increment of one deployment.""" - counters: Final = _usage_counters(keys) - if counters is None or len(counters) != 2 or len(amounts) != 2: - return None - tpm, rpm = counters - tokens, requests = amounts - if not _one_deployment(tpm, rpm) or not isinstance(tokens, int | float) or not isinstance(requests, int | float): - return None - return tpm, tokens, int(requests) - - -def _by(value: object) -> float: - return value if isinstance(value, int | float) and not isinstance(value, bool) else 1 +def _usage_counters(window: UsageWindow) -> tuple[UsageCounter, UsageCounter]: + return window.counter("tpm"), window.counter("rpm") +@dataclass(frozen=True, slots=True) class RoutingStores: - """Router-core's cache steps for cooldowns, allowed fails and tpm/rpm usage, answered by the native stores - the Python core uses. ``None`` leaves the step to the routing cache.""" + """Router-core's cache steps answered by the stores the Python core uses. The stores own every key.""" - def __init__(self, cooldowns: CooldownCache, usage: RouterUsage) -> None: - self._cooldowns: Final = cooldowns - self._usage: Final = usage + cooldowns: CooldownCache + usage: RouterUsage + health: HealthStates - def answer_sync(self, op: object, key: str, value: object, ttl: float | None) -> Handled | None: - store: Final = self._cooldowns.cooldown_store - parsed: Final = parse_key(key) - target: Final = _service_target(parsed) - if target is None: - return None - with service_target(target): - match op, parsed: - case "get", CooldownKey(model_id): - return Handled(_aligned((model_id,), store.active_sync((model_id,)))[0]) - case "set", CooldownKey(model_id): - store.put(model_id, _cooldown_value(value), ttl or 0.0) - return Handled(None) - case "delete", CooldownKey(model_id): - store.evict_local(model_id) - return Handled(None) - case "increment", AllowedFailsKey(model_id, suffix): - return Handled(store.increment_allowed_fails(model_id, suffix, ttl or 0.0)) - case "increment", UsageKey(counter): - return Handled(self._usage.increment_sync(counter, _by(value), int(ttl or 0))) - case _: - return None + async def answer(self, step: StoreStep) -> object: + store: Final = self.cooldowns.cooldown_store + match step: + case CooldownsStep(ids=ids): + return _aligned(ids, await routing_prefetch.read_cooldowns(store, ids, None)) + case PutCooldownStep(id=model_id, cooldown=cooldown, ttl=ttl): + await store.async_put(model_id, _cooldown_value(cooldown), ttl) + return None + case IncrementAllowedFailsStep(id=model_id, suffix=suffix, ttl=ttl): + return await store.async_increment_allowed_fails(model_id, suffix, ttl) + case AddUsageStep(window=window, tokens=tokens, requests=requests): + try: + await self.usage.increment_deployment( + window.deployment_id, window.model, window.minute, tokens, int(requests) + ) + except Exception as e: # noqa: BLE001 # usage that cannot be written never fails the call, as on the Python router + verbose_router_logger.debug("_increment_deployment_usage: usage was not recorded. Got - %s", e) + return None + case AddRequestStep(window=window): + await self.usage.increment(window.counter("rpm"), 1, RoutingArgs.ttl) + return None + case UsageStep(window=window): + return tuple(await self.usage.get_many(_usage_counters(window))) + case HealthStatesStep(): + return health_states_of(await self.health.async_get_states()) + case _: + assert_never(step) - async def answer(self, op: object, key: str, value: object, ttl: float | None) -> Handled | None: - parsed: Final = parse_key(key) - target: Final = _service_target(parsed) - if target is None: - return None - store: Final = self._cooldowns.cooldown_store - with service_target(target): - match op, parsed: - case "get", CooldownKey(model_id): - return Handled(_aligned((model_id,), await store.active((model_id,), batch=Batch.NONE))[0]) - case "set", CooldownKey(model_id): - await store.async_put(model_id, _cooldown_value(value), ttl or 0.0) - return Handled(None) - case "increment", AllowedFailsKey(model_id, suffix): - return Handled(await store.async_increment_allowed_fails(model_id, suffix, ttl or 0.0)) - case "increment", UsageKey(counter): - by: Final = _by(value) - return Handled(await self._usage.increment(counter, by, int(ttl or 0))) - case _: - return self.answer_sync(op, key, value, ttl) - - def batch_get_sync(self, keys: Sequence[str]) -> tuple[object, ...] | None: - model_ids: Final = _cooldown_ids(keys) - if model_ids is not None: - with service_target(ROUTER_COOLDOWNS_TARGET): - return _aligned(model_ids, self._cooldowns.cooldown_store.active_sync(model_ids)) - counters: Final = _usage_counters(keys) - if counters is None: - return None - with service_target(ROUTER_USAGE_TARGET): - return tuple(self._usage.get_many_sync(counters)) - - async def batch_get(self, keys: Sequence[str]) -> tuple[object, ...] | None: - model_ids: Final = _cooldown_ids(keys) - if model_ids is not None: - store: Final = self._cooldowns.cooldown_store - with service_target(ROUTER_COOLDOWNS_TARGET): - return _aligned(model_ids, await routing_prefetch.read_cooldowns(store, model_ids, None)) - counters: Final = _usage_counters(keys) - if counters is None: - return None - with service_target(ROUTER_USAGE_TARGET): - return tuple(await self._usage.get_many(counters)) - - @with_service_target(ROUTER_USAGE_TARGET) - def increment_usage_sync(self, keys: Sequence[object], amounts: Sequence[object]) -> bool: - found: Final = _deployment_increment(keys, amounts) - if found is None: - return False - tpm, tokens, requests = found - self._usage.increment_deployment_sync(tpm.deployment_id, tpm.model, tpm.minute, tokens, requests) - return True - - @with_service_target(ROUTER_USAGE_TARGET) - async def increment_usage(self, keys: Sequence[object], amounts: Sequence[object]) -> bool: - found: Final = _deployment_increment(keys, amounts) - if found is None: - return False - tpm, tokens, requests = found - await self._usage.increment_deployment(tpm.deployment_id, tpm.model, tpm.minute, tokens, requests) - return True + def answer_sync(self, step: StoreStep) -> object: + store: Final = self.cooldowns.cooldown_store + match step: + case CooldownsStep(ids=ids): + return _aligned(ids, store.active_sync(ids)) + case PutCooldownStep(id=model_id, cooldown=cooldown, ttl=ttl): + store.put(model_id, _cooldown_value(cooldown), ttl) + return None + case IncrementAllowedFailsStep(id=model_id, suffix=suffix, ttl=ttl): + return store.increment_allowed_fails(model_id, suffix, ttl) + case AddUsageStep(window=window, tokens=tokens, requests=requests): + self.usage.increment_deployment_sync( + window.deployment_id, window.model, window.minute, tokens, int(requests) + ) + return None + case AddRequestStep(window=window): + self.usage.increment_sync(window.counter("rpm"), 1, RoutingArgs.ttl) + return None + case UsageStep(window=window): + return tuple(self.usage.get_many_sync(_usage_counters(window))) + case HealthStatesStep(): + return health_states_of(self.health.get_states()) + case _: + assert_never(step) diff --git a/litellm/router_rust/steps/access.py b/litellm/router_rust/steps/access.py index 087a87e2c6b..2f513b616b5 100644 --- a/litellm/router_rust/steps/access.py +++ b/litellm/router_rust/steps/access.py @@ -5,13 +5,16 @@ caller to its groups' deployments) and, for a proxy admin, the model the request from __future__ import annotations from collections.abc import Mapping -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request, request_user_api_key_auth +if TYPE_CHECKING: + from litellm.router_api import RouterRegistry + _NAMES: Final = TypeAdapter(tuple[object, ...]) @@ -42,6 +45,22 @@ def caller_fields(request: Mapping[str, object]) -> Mapping[str, object]: } +def pinned_deployment_wildcard_grant( + name: object, allowed_models: object, team_id: str | None, registry: RouterRegistry +) -> tuple[str, ...] | None: + granted: Final = _names(allowed_models) + if not isinstance(name, str) or not granted or name in granted or name in registry.index("model_name"): + return None + deployment: Final = registry.deployment(name) + if deployment is None: + return None + from litellm.proxy.auth.auth_checks import model_matches_any_wildcard_pattern_in_list + + access_groups: Final = registry.access_groups(model_name=deployment.model_name, team_id=team_id) + patterns: Final = tuple(model for model in granted if model not in access_groups) + return (*granted, name) if model_matches_any_wildcard_pattern_in_list(deployment.model_name, patterns) else None + + def _router_weights_error(raw: object) -> BaseException: """The error ``validate_router_weights`` raises for weights the Rust core refused.""" from pydantic import ValidationError diff --git a/litellm/router_rust/steps/callbacks.py b/litellm/router_rust/steps/callbacks.py index 48cb998df9b..5cda8675612 100644 --- a/litellm/router_rust/steps/callbacks.py +++ b/litellm/router_rust/steps/callbacks.py @@ -17,7 +17,8 @@ which outcome callbacks the Python router would still have running for a request router core. """ -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Awaitable, Iterable, Mapping, Sequence +from functools import partial from itertools import chain from typing import TYPE_CHECKING, Final from weakref import WeakSet @@ -27,6 +28,17 @@ from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import PROMPT_CACHE_PIN_CACHE_MAX_SIZE from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.enums import Hook +from litellm.integrations.custom_logger_v2.hook_point_fields import CandidateFields +from litellm.integrations.custom_logger_v2.legacy import ( + Admit, + hook_subscribers, + kwargs_after, + kwargs_patch, + run_subscribers, +) +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 +from litellm.integrations.custom_logger_v2.outcomes import Patch from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.routing_common_utils import get_request_team_id from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy @@ -51,6 +63,8 @@ from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.redis.stores.router_session_pins import RouterSessionPinStore + from litellm.redis.stores.router_usage import RouterUsage from litellm.router_python_host import PythonHost from litellm.router_rust.steps.selection import StrategySelectors from litellm.router_rust.surface.registry import HostRegistry @@ -109,6 +123,8 @@ class OptionalChecks: owner: "GlobalCallbackHolder", host: "PythonHost", cache: DualCache, + usage: "RouterUsage", + pins: "RouterSessionPinStore", hosted: "HostRegistry", ctx: object, arguments: Mapping[str, object], @@ -116,6 +132,8 @@ class OptionalChecks: self._owner: Final = owner self._host: Final = host self._cache: Final = cache + self._usage: Final = usage + self._pins: Final = pins self._hosted: Final = hosted self._ctx: Final = ctx self.optional_callbacks: list[object] | None = None # mutable-ok: the router's live list, as Python keeps it @@ -209,7 +227,7 @@ class OptionalChecks: def _affinity_check(self, user_key: bool, responses_api: bool, session_id: bool) -> DeploymentAffinityCheck: return DeploymentAffinityCheck( - cache=self._cache, + pins=self._pins, ttl_seconds=self.deployment_affinity_ttl_seconds, # pyright: ignore[reportArgumentType] # the argument as passed enable_user_key_affinity=user_key, enable_responses_api_affinity=responses_api, @@ -313,7 +331,7 @@ class OptionalChecks: ) self.router_budget_logger = callback elif name == "enforce_model_rate_limits": - callback = ModelRateLimitingCheck(dual_cache=self._cache) + callback = ModelRateLimitingCheck(usage=self._usage) if callback is None: continue if self.optional_callbacks is not None and any( @@ -454,7 +472,11 @@ class OptionalChecks: def has_filters(self) -> bool: """Whether ``filter_deployments`` would run any callback: without one it keeps every deployment.""" - return any(filters_deployments(callback) for callback in self.routing_callbacks()) + callbacks: Final = tuple(self.routing_callbacks()) + return any(filters_deployments(callback) for callback in callbacks) or ( + Hook.filter_deployments in CustomLoggerV2.declared_hooks + and hook_subscribers(Hook.filter_deployments, callbacks, admit=Admit.every_logger).has_v2 + ) async def filter_deployments( self, @@ -467,25 +489,25 @@ class OptionalChecks: turn, then the prompt caching check names the deployment it prefers among the rest, which the Rust core takes only if its later filters keep it; a filter's error is raised for the Rust core to decide on.""" - remaining: list[object] = list(deployments) # mutable-ok: filter contract # rebind-ok: narrowed + remaining: object = list(deployments) # rebind-ok: narrowed by each filter request_kwargs: Final = dict(request) callbacks: Final = tuple(self.routing_callbacks()) - for callback in callbacks: - if not isinstance(callback, CustomLogger) or not filters_deployments(callback): - continue - if isinstance(callback, PromptCachingDeploymentCheck): - continue - remaining = await callback.async_filter_deployments( # pyright: ignore[reportUnknownMemberType, reportAssignmentType, reportUnknownVariableType] # see above - model=model, - healthy_deployments=remaining, - messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given - request_kwargs=request_kwargs, - parent_otel_span=None, - ) + if ( + Hook.filter_deployments in CustomLoggerV2.declared_hooks + and (subscribers := hook_subscribers(Hook.filter_deployments, callbacks, admit=Admit.every_logger)).has_v2 + ): + filters: Final = partial(_start_filter, model, messages, request_kwargs) + narrowed: Final = await run_subscribers(subscribers, CandidateFields(remaining, model), filters) + remaining = kwargs_after(narrowed) # rebind-ok: narrowed by the hook's subscribers + else: + for callback in callbacks: + started = _filter_call(callback, model, messages, request_kwargs, remaining) + if started is not None: + remaining = await started prompt_caching: Final = next((cb for cb in callbacks if isinstance(cb, PromptCachingDeploymentCheck)), None) if prompt_caching is None: - return remaining, None - return remaining, await prompt_caching.async_preferred_deployment_id( + return remaining, None # pyright: ignore[reportReturnType] # whatever the last filter returned + return remaining, await prompt_caching.async_preferred_deployment_id( # pyright: ignore[reportReturnType] # see above model, remaining, # pyright: ignore[reportArgumentType] # RustCore's deployment dicts messages, # pyright: ignore[reportArgumentType] # the request's messages, as given @@ -574,6 +596,41 @@ def _ids_of(value: object) -> Sequence[object]: return value if isinstance(value, list | tuple) else () # pyright: ignore[reportUnknownVariableType] # the step's id list +def _filter_call( + callback: object, + model: str, + messages: object, + request_kwargs: dict[str, object], # mutable-ok: the filters take the request dict + remaining: object, +) -> Awaitable[object] | None: + if not isinstance(callback, CustomLogger) or not filters_deployments(callback): + return None + if isinstance(callback, PromptCachingDeploymentCheck): + return None + return callback.async_filter_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped deployment dicts + model=model, + healthy_deployments=remaining, # pyright: ignore[reportArgumentType] # whatever the previous filter returned + messages=messages, # pyright: ignore[reportArgumentType] # the request's messages, as given + request_kwargs=request_kwargs, + parent_otel_span=None, + ) + + +def _start_filter( + model: str, + messages: object, + request_kwargs: dict[str, object], # mutable-ok: the filters take the request dict + callback: object, + remaining: object, +) -> Awaitable[Patch | None] | None: + started: Final = _filter_call(callback, model, messages, request_kwargs, remaining) + return None if started is None else _narrowed_by(remaining, started) + + +async def _narrowed_by(remaining: object, started: Awaitable[object]) -> Patch | None: + return kwargs_patch(remaining, await started) + + def filters_deployments(callback: object) -> bool: """A routing callback that overrides ``async_filter_deployments``: the base filter returns its input unchanged, so only overriding filters are run.""" diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py index 2953a87ed92..44a4800af8d 100644 --- a/litellm/router_rust/steps/prerouting.py +++ b/litellm/router_rust/steps/prerouting.py @@ -66,6 +66,7 @@ from litellm.utils import provider_rejectable_params if TYPE_CHECKING: from litellm.caching.redis_cache import RedisCache + from litellm.redis.stores.router_session_pins import RouterSessionPinStore from litellm.router_python_host import PythonHost from litellm.router_rust.surface.registry import HostRegistry from litellm.router_strategy.auto_router.auto_router import PreRoutingHookResponse @@ -172,13 +173,18 @@ class PreRoutingStrategies: reads ``_core_auto_routers`` and its siblings there).""" def __init__( - self, hosted: "HostRegistry", host: "PythonHost", ctx: object, capability_limit: object, cache: DualCache + self, + hosted: "HostRegistry", + host: "PythonHost", + ctx: object, + capability_limit: object, + pins: "RouterSessionPinStore", ) -> None: self._hosted: Final = hosted self._host: Final = host self._ctx: Final = ctx self._capability_limit: object = capability_limit - self._cache: Final = cache + self._pins: Final = pins self._affinity: object = None # The router's own `_ensure_deployment_affinity_callback` (its optional checks), when set. self.ensure_affinity: Callable[[], None] | None = None @@ -454,7 +460,7 @@ class PreRoutingStrategies: return group is not None and group.routing_strategy == "priority" affinity: Final = DeploymentAffinityCheck( - cache=self._cache, + pins=self._pins, ttl_seconds=3600, enable_user_key_affinity=False, enable_responses_api_affinity=False, diff --git a/litellm/router_rust/steps/selection.py b/litellm/router_rust/steps/selection.py index 183350bc05f..f7a982356b6 100644 --- a/litellm/router_rust/steps/selection.py +++ b/litellm/router_rust/steps/selection.py @@ -35,6 +35,7 @@ from litellm.types.router import RoutingGroup, RoutingStrategy if TYPE_CHECKING: from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache + from litellm.redis.stores.router_usage import RouterUsage from litellm.router_python_host import PythonHost Selector: TypeAlias = ( @@ -68,6 +69,7 @@ def build_selector( strategy: str | None, args: Mapping[str, object], cache: "DualCache", + usage: "RouterUsage", reservations: "InMemoryCache | None" = None, ) -> Selector | None: """``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every @@ -80,7 +82,7 @@ def build_selector( case "usage-based-routing": return LowestTPMLoggingHandler(router_cache=cache, routing_args=routing_args) case "usage-based-routing-v2": - return LowestTPMLoggingHandler_v2(router_cache=cache, routing_args=routing_args) + return LowestTPMLoggingHandler_v2(usage=usage, routing_args=routing_args) case "latency-based-routing": return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args) case "cost-based-routing": @@ -98,11 +100,12 @@ class StrategySelectors: """The selectors of one router: the default group's, each routing group's, and the lazily built per-request override ones.""" - def __init__(self, host: "PythonHost", cache: "DualCache") -> None: + def __init__(self, host: "PythonHost", cache: "DualCache", usage: "RouterUsage") -> None: from litellm.caching.in_memory_cache import InMemoryCache self._host: Final = host self._cache: Final = cache + self._usage: Final = usage # `_core_least_busy_reservations`: the picks least-busy reserved, shared by its selectors. self.reservations: Final = InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS) self._strategy: str | None = None @@ -144,7 +147,7 @@ class StrategySelectors: self._strategy = normalize_strategy(strategy) if configured: self._configured = self._strategy - self._default = build_selector(self._strategy, args, self._cache, self.reservations) + self._default = build_selector(self._strategy, args, self._cache, self._usage, self.reservations) if self._default is not None: self._register(self._default) @@ -158,7 +161,7 @@ class StrategySelectors: if self._default is None or self._strategy != self._configured: return try: - rebuilt: Final = build_selector(self._strategy, args, self._cache, self.reservations) + rebuilt: Final = build_selector(self._strategy, args, self._cache, self._usage, self.reservations) except (TypeError, ValidationError): verbose_router_logger.exception( "Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy @@ -177,7 +180,7 @@ class StrategySelectors: for name, strategy, args in groups if ( selector := build_selector( - None if strategy == "priority" else strategy, args, self._cache, self.reservations + None if strategy == "priority" else strategy, args, self._cache, self._usage, self.reservations ) ) is not None @@ -239,7 +242,7 @@ class StrategySelectors: return self.default with self._lock: if strategy not in self._overrides: - self._overrides[strategy] = build_selector(strategy, {}, self._cache, self.reservations) + self._overrides[strategy] = build_selector(strategy, {}, self._cache, self._usage, self.reservations) selector: Final = self._overrides[strategy] if selector is not None and request is not None and strategy not in self._registered_strategies(): from litellm.litellm_core_utils.litellm_logging import Logging diff --git a/litellm/router_rust/surface/health.py b/litellm/router_rust/surface/health.py index f2ac72a262a..211ead61aeb 100644 --- a/litellm/router_rust/surface/health.py +++ b/litellm/router_rust/surface/health.py @@ -505,14 +505,15 @@ class RustHealthCallbacks: ) -> str | None: """``sync_deployment_callback_on_success``, for a sync stream that ended.""" from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type + from litellm.router_utils.router_callbacks.track_deployment_metrics import deployment_successes_key if is_batch_retrieve_call_type(kwargs.get("call_type")): return None target: Final = _success_target(kwargs) if target is None: return None - key: Final = self._drive_sync_steps(self._handle.count_success(target[0], callbacks)).get("key") - return key if isinstance(key, str) else None + counted: Final = self._drive_sync_steps(self._handle.count_success(target[0], callbacks)).get("counted") + return deployment_successes_key(target[0]) if counted is True else None async def repin_after_success(self, kwargs: Mapping[str, object], completion_response: object) -> None: """The session re-pin ``deployment_callback_on_success`` runs for a success logged against a diff --git a/litellm/router_rust/surface/health_routing.py b/litellm/router_rust/surface/health_routing.py index 46542bf9af2..6ffdaec508e 100644 --- a/litellm/router_rust/surface/health_routing.py +++ b/litellm/router_rust/surface/health_routing.py @@ -17,11 +17,10 @@ from litellm.router_utils.health_state_cache import DeploymentHealthCache from litellm.types.router import AllowedFailsPolicy if TYPE_CHECKING: - from litellm.caching.caching import DualCache + from litellm.redis.stores.deployment_health import DeploymentHealthStore from litellm.router_config import RouterConfig from litellm.types.router import Deployment, DeploymentTypedDict -HEALTH_STATE_KEY: Final = DeploymentHealthCache.CACHE_KEY # The failover flags the Python core keeps as plain router attributes, with their defaults. FAILOVER_FLAGS: Final = ("enable_weighted_failover", "fallback_on_unsupported_params") @@ -75,8 +74,8 @@ class HealthRouting: else None, } - def health_checks(self, cache: "DualCache") -> DeploymentHealthCache: - return DeploymentHealthCache(cache=cache, staleness_threshold=self.health_check_staleness) + def health_checks(self, store: "DeploymentHealthStore") -> DeploymentHealthCache: + return DeploymentHealthCache(store=store, staleness_threshold=self.health_check_staleness) def failover_document(plain: Mapping[str, object]) -> Mapping[str, object]: @@ -149,7 +148,6 @@ async def fully_unhealthy_model_names( __all__ = [ "FAILOVER_FLAGS", - "HEALTH_STATE_KEY", "HealthRouting", "all_blocked", "failover_document", diff --git a/litellm/router_rust/surface/legacy_members.py b/litellm/router_rust/surface/legacy_members.py index 640fdf739c0..9b436d6ce3e 100644 --- a/litellm/router_rust/surface/legacy_members.py +++ b/litellm/router_rust/surface/legacy_members.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Final from typing_extensions import NotRequired, ReadOnly, TypedDict -from litellm._internal_context import with_service_target +from litellm._internal_context import request_parent_otel_span, with_service_target from litellm._logging import verbose_router_logger from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger @@ -423,10 +423,11 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): awaitable: Final[Awaitable[object]] = response # pyright: ignore[reportAssignmentType] # the endpoint's awaitable response = await awaitable # rebind-ok: awaited here except Exception: - with contextlib.suppress(Exception): - await async_io_token_refund_failure( - self._cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) - ) + with ( + contextlib.suppress(Exception), + request_parent_otel_span(get_parent_otel_span_from_kwargs(kwargs)), + ): + await async_io_token_refund_failure(self._usage, kwargs) raise await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) return await self.set_response_headers( @@ -513,7 +514,9 @@ class RustLegacyMembers(RustRegistryMembers, RustOptionalChecks): if model_group is None or deployment_id is None: return model: Final = None if deployment_name is None else str(deployment_name) - counter: Final = UsageCounter(str(deployment_id), model, get_utc_datetime().strftime("%H-%M"), "rpm") + counter: Final = UsageCounter( + "global_router", str(deployment_id), model, get_utc_datetime().strftime("%H-%M"), "rpm" + ) await self._usage.increment(counter=counter, value=1, ttl=_USAGE_TTL) # -- the plugin context ------------------------------------------------------------------------ diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index dd67e252370..7cf4d46edc3 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -7,8 +7,8 @@ host registry. router-core holds the rules: a deployment's model info (``get_router_model_info``, ``get_deployment_model_info``: ``router-core/src/model_info.rs``), a model group's info and the configured-value reads (``_set_model_group_info``, ``get_model_listing_info``, the configured token -limits, mode, display name and service tiers: ``router-core/src/group_info.rs``), and the usage -counters' keys (``RouterCoreHandle.usage_key``). What lives in this process is answered here as a +limits, mode, display name and service tiers: ``router-core/src/group_info.rs``); the usage +counters' keys are the ``router_usage`` store's. What lives in this process is answered here as a query asks for it, at query time: the cost map (``get_model_info``, ``litellm.model_cost``), provider resolution, the wildcard pattern routes, what the background refresh discovered, and a group deployment's validated params, supported params, endpoints, fast mode and reasoning efforts. @@ -33,7 +33,6 @@ from pydantic import BaseModel, TypeAdapter from typing_extensions import overload import litellm -from litellm._internal_context import service_target from litellm._logging import verbose_router_logger from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.litellm_core_utils.add_retry_fallback_headers import ( @@ -56,7 +55,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import ( from litellm.litellm_core_utils.routing_common_utils import team_may_use_deployment from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.bedrock.common_utils import get_bedrock_base_model -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage, UsageCounter +from litellm.redis.stores.router_usage import RouterUsage, UsageCounter from litellm.router_python_host import python_host from litellm.router_utils.model_group_pricing import deployment_pricing from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import build_io_token_rate_limit_headers @@ -811,7 +810,10 @@ class HostModelInfo: return cached[2] deployments: Final = tuple(pair for pair in members if pair[0] is not None and pair[1] is not None) counters: Final = tuple( - tuple(UsageCounter(id, litellm_model, current_minute, kind) for id, litellm_model in deployments) # pyright: ignore[reportArgumentType] # what the listing names, as the Python core passed it + tuple( + UsageCounter("global_router", id, litellm_model, current_minute, kind) # pyright: ignore[reportArgumentType] # what the listing names, as the Python core passed it + for id, litellm_model in deployments + ) for kind in ("tpm", "rpm") ) if len(self._group_counters_cache) >= DEFAULT_MAX_LRU_CACHE_SIZE: @@ -839,30 +841,31 @@ class HostModelInfo: """ Returns current ITPM/OTPM usage for a model group (sum across deployments). """ - dt: Final = get_utc_datetime() - current_minute: Final = dt.strftime("%H-%M") - itpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged - otpm_keys: Final[list[str]] = [] # mutable-ok: ported from the Python core unchanged - + current_minute: Final = get_utc_datetime().strftime("%H-%M") model_list: Final = self._core_get_model_list(model_name=model_group) if model_list is None: return None, None - for model in model_list: - model_id: str | None = model.get("model_info", {}).get("id") - litellm_model: str | None = model["litellm_params"].get("model") - if model_id is None or litellm_model is None: - continue - itpm_keys.append(self._core_handle().usage_key(model_id, litellm_model, "itpm", current_minute)) - otpm_keys.append(self._core_handle().usage_key(model_id, litellm_model, "otpm", current_minute)) + named: Final[tuple[tuple[object, object], ...]] = tuple( + (model.get("model_info", {}).get("id"), model["litellm_params"].get("model")) for model in model_list + ) + deployments: Final = tuple( + (f"{model_id}", f"{litellm_model}") + for model_id, litellm_model in named + if model_id is not None and litellm_model is not None + ) + itpm_counters: Final = tuple( + UsageCounter("global_router", model_id, litellm_model, current_minute, "itpm") + for model_id, litellm_model in deployments + ) + otpm_counters: Final = tuple( + UsageCounter("global_router", model_id, litellm_model, current_minute, "otpm") + for model_id, litellm_model in deployments + ) + combined_values: Final = await self._core_usage_store.get_many((*itpm_counters, *otpm_counters)) - with service_target(ROUTER_USAGE_TARGET): - combined_values: Final = await self._core_cache.async_batch_get_cache(keys=itpm_keys + otpm_keys) - if combined_values is None: - return None, None - - itpm_values: Final = combined_values[: len(itpm_keys)] - otpm_values: Final = combined_values[len(itpm_keys) :] + itpm_values: Final = combined_values[: len(itpm_counters)] + otpm_values: Final = combined_values[len(itpm_counters) :] total_itpm: int | None = None # rebind-ok: ported from the Python core unchanged for value in itpm_values: diff --git a/litellm/router_strategy/base_routing_strategy.py b/litellm/router_strategy/base_routing_strategy.py index 43acae1f62f..f1d0373ef25 100644 --- a/litellm/router_strategy/base_routing_strategy.py +++ b/litellm/router_strategy/base_routing_strategy.py @@ -9,28 +9,27 @@ from typing import Final from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger -from litellm.caching.caching import DualCache -from litellm.caching.redis_cache import RedisPipelineIncrementOperation, log_redis_failure +from litellm.caching.redis_cache import log_redis_failure from litellm.constants import DEFAULT_REDIS_SYNC_INTERVAL -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET +from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, CounterIncrement, RouterUsage, UsageCounter class BaseRoutingStrategy(ABC): def __init__( self, - dual_cache: DualCache, + usage: RouterUsage, should_batch_redis_writes: bool, default_sync_interval: float | None, ): - self.dual_cache = dual_cache - self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = [] + self.usage = usage + self.redis_increment_operation_queue: list[CounterIncrement] = [] self._sync_task: asyncio.Task[None] | None = None self._retired = False self._flush_after_retire: asyncio.Task[None] | None = None if should_batch_redis_writes: self.setup_sync_task(default_sync_interval) - self.in_memory_keys_to_update: set[str] = set() # Set with max size of 1000 keys + self.in_memory_keys_to_update: set[UsageCounter] = set() # Set with max size of 1000 keys def setup_sync_task(self, default_sync_interval: float | None): """Setup the sync task in a way that's compatible with FastAPI""" @@ -56,7 +55,7 @@ class BaseRoutingStrategy(ABC): def _schedule_flush_after_retire(self) -> None: if self._flush_after_retire is not None and not self._flush_after_retire.done(): return - if not self.redis_increment_operation_queue or self.dual_cache.redis_cache is None: + if not self.redis_increment_operation_queue or not self.usage.has_redis: return try: loop: Final = asyncio.get_running_loop() @@ -79,19 +78,7 @@ class BaseRoutingStrategy(ABC): except asyncio.CancelledError: pass - async def _increment_value_list_in_current_window( - self, increment_list: list[tuple[str, int]], ttl: int - ) -> list[float]: - """ - Increment a list of values in the current window - """ - results: Final = [] - for key, value in increment_list: - result = await self._increment_value_in_current_window(key=key, value=value, ttl=ttl) - results.append(result) - return results - - async def _increment_value_in_current_window(self, key: str, value: float, ttl: int): + async def _increment_value_in_current_window(self, counter: UsageCounter, value: float, ttl: int): """ Increment spend within existing budget window @@ -100,19 +87,11 @@ class BaseRoutingStrategy(ABC): - Increments the spend in memory cache (so spend instantly updated in memory) - Queues the increment operation to Redis Pipeline (using batched pipeline to optimize performance. Using Redis for multi instance environment of LiteLLM) """ - result: Final = await self.dual_cache.in_memory_cache.async_increment( - key=key, - value=value, - ttl=ttl, - ) - increment_op: Final = RedisPipelineIncrementOperation( - key=key, - increment_value=value, - ttl=ttl, - ) + increment_op: Final = CounterIncrement(counter=counter, amount=value, ttl=ttl) + result: Final = self.usage.increment_local(increment_op) self.redis_increment_operation_queue.append(increment_op) - self.add_to_in_memory_keys_to_update(key=key) + self.add_to_in_memory_keys_to_update(counter) if self._retired: self._schedule_flush_after_retire() return result @@ -148,35 +127,26 @@ class BaseRoutingStrategy(ABC): Only runs if Redis is initialized """ try: - if not self.dual_cache.redis_cache: + if not self.usage.has_redis: return # Redis is not initialized if len(self.redis_increment_operation_queue) > 0: - # Compress operations for the same key - compressed_ops: Final[dict[str, RedisPipelineIncrementOperation]] = {} - ops_to_remove: Final = [] - for idx, op in enumerate(self.redis_increment_operation_queue): - if op["key"] in compressed_ops: - # Add to existing increment - compressed_ops[op["key"]]["increment_value"] += op["increment_value"] - else: - compressed_ops[op["key"]] = op - - ops_to_remove.append(idx) - - # Convert back to list - compressed_queue: Final = list(compressed_ops.values()) - - increment_result: Final = await self.dual_cache.redis_cache.async_increment_pipeline( - increment_list=compressed_queue, + queued: Final = tuple(self.redis_increment_operation_queue) + compressed_queue: Final = tuple( + CounterIncrement( + counter=counter, + amount=sum(op.amount for op in queued if op.counter == counter), + ttl=next(op.ttl for op in queued if op.counter == counter), + ) + for counter in dict.fromkeys(op.counter for op in queued) ) - self.redis_increment_operation_queue = [ - op for idx, op in enumerate(self.redis_increment_operation_queue) if idx not in ops_to_remove - ] + increment_result: Final = await self.usage.increment_remote(compressed_queue) + + self.redis_increment_operation_queue = self.redis_increment_operation_queue[len(queued) :] if increment_result is not None: - return_result = {key["key"]: op for key, op in zip(compressed_queue, increment_result)} + return_result = {op.counter: total for op, total in zip(compressed_queue, increment_result)} else: return_result = {} return return_result @@ -185,19 +155,13 @@ class BaseRoutingStrategy(ABC): log_redis_failure(verbose_router_logger, logging.ERROR, "Error syncing in-memory cache with Redis", e) self.redis_increment_operation_queue = [] - def add_to_in_memory_keys_to_update(self, key: str): - self.in_memory_keys_to_update.add(key) + def add_to_in_memory_keys_to_update(self, counter: UsageCounter): + self.in_memory_keys_to_update.add(counter) - def get_key_pattern_to_sync(self) -> str | None: - """ - Get the key pattern to sync - """ - return None - - def get_in_memory_keys_to_update(self) -> set[str]: + def get_in_memory_keys_to_update(self) -> set[UsageCounter]: return self.in_memory_keys_to_update - def get_and_reset_in_memory_keys_to_update(self) -> set[str]: + def get_and_reset_in_memory_keys_to_update(self) -> set[UsageCounter]: """Atomic get and reset in-memory keys to update""" keys: Final = self.in_memory_keys_to_update self.in_memory_keys_to_update = set() @@ -221,7 +185,7 @@ class BaseRoutingStrategy(ABC): try: # No need to sync if Redis cache is not initialized - if self.dual_cache.redis_cache is None: + if not self.usage.has_redis: return # 2. Fetch all current provider spend from Redis to update in-memory cache @@ -232,10 +196,9 @@ class BaseRoutingStrategy(ABC): cache_keys_list: Final = list(cache_keys) # 1. Snapshot in-memory before - in_memory_before_dict: Final = {} - in_memory_before: Final = await self.dual_cache.in_memory_cache.async_batch_get_cache(keys=cache_keys_list) - for k, v in zip(cache_keys_list, in_memory_before): - in_memory_before_dict[k] = float(v or 0) + in_memory_before_dict: Final = { + k: float(v or 0) for k, v in zip(cache_keys_list, self.usage.get_local_many(cache_keys_list)) + } # 1. Push all provider spend increments to Redis redis_values: Final = await self._push_in_memory_increments_to_redis() @@ -246,7 +209,7 @@ class BaseRoutingStrategy(ABC): for key in cache_keys_list: redis_val = float(redis_values.get(key, 0) or 0) before = float(in_memory_before_dict.get(key, 0) or 0) - after = float(await self.dual_cache.in_memory_cache.async_get_cache(key=key) or 0) + after = float(self.usage.get_local(key) or 0) delta = after - before if after <= redis_val: merged = redis_val + delta @@ -259,7 +222,7 @@ class BaseRoutingStrategy(ABC): # import os # os._exit(1) # raise Exception(f"Redis is behind in-memory cache for key: {key}. This should not happen, since we should be updating redis with in-memory cache.") - await self.dual_cache.in_memory_cache.async_set_cache(key=key, value=merged) + self.usage.set_local(key, merged) except Exception as e: verbose_router_logger.exception("Error syncing in-memory cache with Redis: %s", e) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2ce1bda173a..79dfd64f3e8 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -65,6 +65,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG +from litellm.redis.stores.router_session_pins import TierModelPin, TierSessionPin from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param from litellm.router_strategy.adaptive_router.classifier import classify_prompt from litellm.router_strategy.complexity_router.context_compaction import compaction_pending @@ -3182,17 +3183,16 @@ class ComplexityRouter(CustomLogger): session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if session_id is None: return model - caller: Final = DeploymentAffinityCheck.get_user_key_from_request_kwargs(request_kwargs) - identity: Final = (self.model_name, self._tier_affinity_config, caller, session_id, _tier_name(tier)) - cache_identity: Final = ( - (*identity, ("replay_fallback", retained_model)) if retained_model is not None else identity - ) - cache_key: Final = ( - "complexity_router_tier_model_affinity:v1:" - + hashlib.sha256(json.dumps(cache_identity).encode()).hexdigest() + cache_key: Final = TierModelPin( + router=self.model_name, + tiers=self._tier_affinity_config, + caller=DeploymentAffinityCheck.get_user_key_from_request_kwargs(request_kwargs), + session_id=session_id, + tier=_tier_name(tier), + replay_fallback=retained_model, ) winner: Final = await claim_affinity_pin( - self.ctx.health().store, + self.ctx.health().session_pins, cache_key, MappingProxyType({"model": model}), self.config.session_affinity_ttl_seconds, @@ -4506,13 +4506,12 @@ class ComplexityRouter(CustomLogger): return str(user_key) return None - def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str: - # Namespace by the caller's API key hash so two different callers reusing the - # same client-supplied session_id can't poison each other's routing pin. Falls - # back to "unscoped" only when there's no authenticated caller to scope by - # (e.g. direct Router usage without the proxy layer). - caller_scope: Final = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped" - return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}" + def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> TierSessionPin: + return TierSessionPin( + router=self.model_name, + caller=self._get_user_api_key_hash_from_request_kwargs(request_kwargs), + session_id=session_id, + ) @property def _uses_tier_pin(self) -> bool: @@ -4594,7 +4593,7 @@ class ComplexityRouter(CustomLogger): if cache_key is not None and pin_replay_allowed: pinned_value: Final = await read_affinity_pin( - self.ctx.health().store, cache_key, self.config.session_affinity_ttl_seconds + self.ctx.health().session_pins, cache_key, self.config.session_affinity_ttl_seconds ) pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names()) if pinned_pin is not None: @@ -4693,10 +4692,10 @@ class ComplexityRouter(CustomLogger): else floor_model ) if escalated: - await self.ctx.health().store.async_set_cache( - key=cache_key, - value=_session_affinity_cache_value(session_model, resolved_pin_tier), - ttl=self.config.session_affinity_ttl_seconds, + await self.ctx.health().session_pins.set( + cache_key, + _session_affinity_cache_value(session_model, resolved_pin_tier), + self.config.session_affinity_ttl_seconds, ) if self.config.adaptive: from litellm.router_strategy.adaptive_router.config import ( @@ -4802,13 +4801,13 @@ class ComplexityRouter(CustomLogger): and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None ) if pinnable and cache_key is not None and response is not None: - await self.ctx.health().store.async_set_cache( - key=cache_key, - value=_session_affinity_cache_value( + await self.ctx.health().session_pins.set( + cache_key, + _session_affinity_cache_value( response.model, response.routing_decision.get("tier") if response.routing_decision is not None else None, ), - ttl=self.config.session_affinity_ttl_seconds, + self.config.session_affinity_ttl_seconds, ) return self._with_session_deployment_affinity(response) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 8d2f45756e4..db024d47a53 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -15,14 +15,17 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_logger, verbose_router_logger -from litellm.caching.caching import DualCache from litellm.constants import USAGE_BASED_ROUTING_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type -from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models from litellm.litellm_core_utils.token_counter import embedding_aware_token_count -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET +from litellm.redis.stores.router_usage import ( + ROUTER_USAGE_TARGET, + CounterIncrement, + RouterUsage, + UsageCounter, +) from litellm.router_utils.router_callbacks.track_deployment_metrics import get_counted_usage_tokens from litellm.types.router import RouterErrors from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload @@ -55,21 +58,21 @@ _active_prefetched_usage: Final[ContextVar["PrefetchedUsage | None"]] = ContextV @dataclass(frozen=True) class PrefetchedUsage: """ - tpm/rpm counter values another read of this request already fetched from the router cache. + tpm/rpm counter values another read of this request already fetched from the usage store. `values` is None when that read failed, which is what `async_batch_get_cache` returns on failure. """ - keys: frozenset[str] - values: Mapping[str, object] | None + counters: frozenset[UsageCounter] + values: Mapping[UsageCounter, float | None] | None - def covers(self, keys: Sequence[str]) -> bool: - return self.keys.issuperset(keys) + def covers(self, counters: Sequence[UsageCounter]) -> bool: + return self.counters.issuperset(counters) - def values_for(self, keys: Sequence[str]) -> list[object | None] | None: + def values_for(self, counters: Sequence[UsageCounter]) -> list[float | None] | None: if self.values is None: return None - return [self.values.get(key) for key in keys] + return [self.values.get(counter) for counter in counters] @staticmethod @contextmanager @@ -103,12 +106,11 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): logged_failure: int = 0 default_cache_time_seconds: int = 1 * 60 * 60 # 1 hour - def __init__(self, router_cache: DualCache, routing_args: dict = {}): - self.router_cache = router_cache + def __init__(self, usage: RouterUsage, routing_args: dict = {}): self.routing_args = RoutingArgs(**routing_args) BaseRoutingStrategy.__init__( self, - dual_cache=router_cache, + usage=usage, should_batch_redis_writes=True, default_sync_interval=0.1, ) @@ -129,11 +131,11 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") - model_id: Final = deployment.get("model_info", {}).get("id") - deployment_name: Final = deployment.get("litellm_params", {}).get("model") - rpm_key: Final = f"{model_id}:{deployment_name}:rpm:{current_minute}" + model_id: Final[object] = deployment.get("model_info", {}).get("id") + deployment_name: Final[object] = deployment.get("litellm_params", {}).get("model") + rpm_counter: Final = UsageCounter.for_deployment(f"{model_id}", f"{deployment_name}", "rpm", current_minute) - local_result: Final = self.router_cache.get_cache(key=rpm_key, local_only=True) # check local result first + local_result: Final = self.usage.get_local(rpm_counter) # check local result first deployment_rpm = None if deployment_rpm is None: @@ -168,7 +170,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): else: # if local result below limit, check redis ## prevent unnecessary redis checks - result: Final = self.router_cache.increment_cache(key=rpm_key, value=1, ttl=self.routing_args.ttl) + result: Final = self.usage.increment_sync(rpm_counter, 1, self.routing_args.ttl) if result is not None and result > deployment_rpm: raise litellm.RateLimitError( message=f"Deployment over defined rpm limit={deployment_rpm}. current usage={result}", @@ -208,13 +210,11 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): # ------------ dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") - model_id: Final = deployment.get("model_info", {}).get("id") - deployment_name: Final = deployment.get("litellm_params", {}).get("model") + model_id: Final[object] = deployment.get("model_info", {}).get("id") + deployment_name: Final[object] = deployment.get("litellm_params", {}).get("model") - rpm_key: Final = f"{model_id}:{deployment_name}:rpm:{current_minute}" - local_result: Final = await self.router_cache.async_get_cache( - key=rpm_key, local_only=True - ) # check local result first + rpm_counter: Final = UsageCounter.for_deployment(f"{model_id}", f"{deployment_name}", "rpm", current_minute) + local_result: Final = self.usage.get_local(rpm_counter) # check local result first deployment_rpm = None if deployment_rpm is None: @@ -243,7 +243,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): ) else: # if local result below limit, check redis ## prevent unnecessary redis checks - result = await self._increment_value_in_current_window(key=rpm_key, value=1, ttl=self.routing_args.ttl) + result = await self._increment_value_in_current_window(rpm_counter, 1, self.routing_args.ttl) if result is not None and result > deployment_rpm: raise litellm.RateLimitError( message=f"Deployment over defined rpm limit={deployment_rpm}. current usage={result}", @@ -278,8 +278,8 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): if standard_logging_object is None: raise ValueError("standard_logging_object not passed in.") model_group: Final = standard_logging_object.get("model_group") - model: Final = standard_logging_object["hidden_params"].get("litellm_model_name") - id = standard_logging_object.get("model_id") + model: Final[object] = standard_logging_object["hidden_params"].get("litellm_model_name") + id: object = standard_logging_object.get("model_id") if model_group is None or id is None or model is None: return elif isinstance(id, int): @@ -293,14 +293,12 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") # use the same timezone regardless of system clock - tpm_key: Final = f"{id}:{model}:tpm:{current_minute}" - # ------------ - # Update usage - # ------------ - # update cache - ## TPM - self.router_cache.increment_cache(key=tpm_key, value=total_tokens, ttl=self.routing_args.ttl) + self.usage.increment_sync( + UsageCounter.for_deployment(f"{id}", f"{model}", "tpm", current_minute), + total_tokens, + self.routing_args.ttl, + ) ### TESTING ### if self.test_flag: self.logged_success += 1 @@ -334,7 +332,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): deployment_id=str(model_id), deployment_name=model if isinstance(model, str) else None, tokens=total_tokens if counted_tokens is None else max(0, total_tokens - counted_tokens), - parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), ) ### TESTING ### @@ -346,16 +343,10 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): ) @with_service_target(ROUTER_USAGE_TARGET) - async def async_increment_tpm( - self, *, deployment_id: str, deployment_name: str | None, tokens: int, parent_otel_span: Span | None - ) -> None: + async def async_increment_tpm(self, *, deployment_id: str, deployment_name: str | None, tokens: int) -> None: current_minute: Final = get_utc_datetime().strftime("%H-%M") - await self.router_cache.async_increment_cache_post_call( - key=f"{deployment_id}:{deployment_name}:tpm:{current_minute}", - value=tokens, - ttl=self.routing_args.ttl, - parent_otel_span=parent_otel_span, - ) + counter: Final = UsageCounter.for_deployment(deployment_id, f"{deployment_name}", "tpm", current_minute) + await self.usage.increment_post_call((CounterIncrement(counter, tokens, self.routing_args.ttl),)) def _return_potential_deployments( self, @@ -414,10 +405,10 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): self, model_group: str, healthy_deployments: list, - tpm_keys: list, - tpm_values: Sequence | None, - rpm_keys: list, - rpm_values: Sequence | None, + tpm_counters: Sequence[UsageCounter], + tpm_values: Sequence[float | None] | None, + rpm_counters: Sequence[UsageCounter], + rpm_values: Sequence[float | None] | None, messages: list[dict[str, str]] | None = None, input: str | list | None = None, ) -> dict | None: @@ -428,13 +419,12 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): if tpm_values is None or rpm_values is None: return None - tpm_dict = {} # {model_id: 1, ..} - for idx, key in enumerate(tpm_keys): - tpm_dict[tpm_keys[idx].split(":")[0]] = tpm_values[idx] - - rpm_dict: Final = {} # {model_id: 1, ..} - for idx, key in enumerate(rpm_keys): - rpm_dict[rpm_keys[idx].split(":")[0]] = rpm_values[idx] + tpm_dict = { + counter.deployment_id.split(":")[0]: value for counter, value in zip(tpm_counters, tpm_values) + } # {model_id: 1, ..} + rpm_dict: Final = { + counter.deployment_id.split(":")[0]: value for counter, value in zip(rpm_counters, rpm_values) + } # {model_id: 1, ..} try: input_tokens = embedding_aware_token_count( @@ -473,17 +463,25 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): else: return None - def usage_counter_keys(self, healthy_deployments: list) -> tuple[list[str], list[str]]: - """The `::tpm:` and `::rpm:` counter keys selection reads.""" + def usage_counters( + self, healthy_deployments: Sequence[object] + ) -> tuple[tuple[UsageCounter, ...], tuple[UsageCounter, ...]]: + """The tpm and rpm counters selection reads, this minute's for each deployment.""" current_minute: Final = get_utc_datetime().strftime("%H-%M") - prefixes: Final = tuple( - f"{m.get('model_info', {}).get('id')}:{m.get('litellm_params', {}).get('model')}" + named: Final[tuple[tuple[object, object], ...]] = tuple( + (m.get("model_info", {}).get("id"), m.get("litellm_params", {}).get("model")) for m in healthy_deployments if isinstance(m, dict) ) return ( - [f"{prefix}:tpm:{current_minute}" for prefix in prefixes], - [f"{prefix}:rpm:{current_minute}" for prefix in prefixes], + tuple( + UsageCounter.for_deployment(f"{model_id}", f"{model}", "tpm", current_minute) + for model_id, model in named + ), + tuple( + UsageCounter.for_deployment(f"{model_id}", f"{model}", "rpm", current_minute) + for model_id, model in named + ), ) @with_service_target(ROUTER_USAGE_TARGET) @@ -508,20 +506,18 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): healthy_deployments, ) - tpm_keys, rpm_keys = self.usage_counter_keys(healthy_deployments) - combined_tpm_rpm_keys: Final = tpm_keys + rpm_keys + tpm_counters, rpm_counters = self.usage_counters(healthy_deployments) + combined_tpm_rpm_counters: Final = tpm_counters + rpm_counters prefetched_usage: Final = PrefetchedUsage.active() - if prefetched_usage is not None and prefetched_usage.covers(combined_tpm_rpm_keys): - combined_tpm_rpm_values = prefetched_usage.values_for(combined_tpm_rpm_keys) + if prefetched_usage is not None and prefetched_usage.covers(combined_tpm_rpm_counters): + combined_tpm_rpm_values = prefetched_usage.values_for(combined_tpm_rpm_counters) else: - combined_tpm_rpm_values = await self.router_cache.async_batch_get_cache( - keys=combined_tpm_rpm_keys - ) # [1, 2, None, ..] + combined_tpm_rpm_values = await self.usage.get_many(combined_tpm_rpm_counters) # [1, 2, None, ..] if combined_tpm_rpm_values is not None: - tpm_values = combined_tpm_rpm_values[: len(tpm_keys)] - rpm_values = combined_tpm_rpm_values[len(tpm_keys) :] + tpm_values = combined_tpm_rpm_values[: len(tpm_counters)] + rpm_values = combined_tpm_rpm_values[len(tpm_counters) :] else: tpm_values = None rpm_values = None @@ -529,9 +525,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): deployment: Final = self._common_checks_available_deployment( model_group=model_group, healthy_deployments=healthy_deployments, - tpm_keys=tpm_keys, + tpm_counters=tpm_counters, tpm_values=tpm_values, - rpm_keys=rpm_keys, + rpm_counters=rpm_counters, rpm_values=rpm_values, messages=messages, input=input, @@ -614,35 +610,16 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): healthy_deployments, ) - dt: Final = get_utc_datetime() - current_minute: Final = dt.strftime("%H-%M") - tpm_keys: Final = [] - rpm_keys: Final = [] - for m in healthy_deployments: - if isinstance(m, dict): - id = m.get("model_info", {}).get( - "id" - ) # a deployment should always have an 'id'. this is set in router.py - deployment_name = m.get("litellm_params", {}).get("model") - tpm_key = f"{id}:{deployment_name}:tpm:{current_minute}" - rpm_key = f"{id}:{deployment_name}:rpm:{current_minute}" - - tpm_keys.append(tpm_key) - rpm_keys.append(rpm_key) - - tpm_values: Final = self.router_cache.batch_get_cache( - keys=tpm_keys, parent_otel_span=parent_otel_span - ) # [1, 2, None, ..] - rpm_values: Final = self.router_cache.batch_get_cache( - keys=rpm_keys, parent_otel_span=parent_otel_span - ) # [1, 2, None, ..] + tpm_counters, rpm_counters = self.usage_counters(healthy_deployments) + tpm_values: Final = self.usage.get_many_sync(tpm_counters) # [1, 2, None, ..] + rpm_values: Final = self.usage.get_many_sync(rpm_counters) # [1, 2, None, ..] deployment: Final = self._common_checks_available_deployment( model_group=model_group, healthy_deployments=healthy_deployments, - tpm_keys=tpm_keys, + tpm_counters=tpm_counters, tpm_values=tpm_values, - rpm_keys=rpm_keys, + rpm_counters=rpm_counters, rpm_values=rpm_values, messages=messages, input=input, diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 88687269d4b..2f8cddf7740 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -1,11 +1,12 @@ import hashlib import json -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime from enum import Enum +from functools import partial, reduce from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # the router passes the request kwargs as an untyped dict import httpx from pydantic import TypeAdapter @@ -13,6 +14,9 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2.enums import Field, Hook +from litellm.integrations.custom_logger_v2.legacy import Admit, hook_subscribers, kwargs_after, run_subscribers +from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2 from litellm.litellm_core_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, get_fallback_error_info, @@ -975,60 +979,99 @@ async def run_async_fallback( raise error_from_fallbacks -async def log_success_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): - """ - Log a successful fallback event to all registered callbacks. +async def log_success_fallback_event( + original_model_group: str, + kwargs: dict, # mutable-ok: the request kwargs the custom loggers receive + original_exception: Exception, +): + await _log_fallback_event( + True, + original_model_group, + cast( # cast-ok: the request kwargs dict, keyed by parameter name + dict[str, object], kwargs + ), + original_exception, + ) - Uses LoggingCallbackManager.get_custom_loggers_for_type() to get deduplicated - CustomLogger instances from all callback lists. - Args: - original_model_group (str): The original model group before fallback. - kwargs (dict): kwargs for the request +async def log_failure_fallback_event( + original_model_group: str, + kwargs: dict, # mutable-ok: the request kwargs the custom loggers receive + original_exception: Exception, +): + await _log_fallback_event( + False, + original_model_group, + cast( # cast-ok: the request kwargs dict, keyed by parameter name + dict[str, object], kwargs + ), + original_exception, + ) - Note: - Errors during logging are caught and reported but do not interrupt the process. - """ - # Get deduplicated CustomLogger instances from all callback lists - custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger) - for _callback_custom_logger in custom_loggers: - try: - await _callback_custom_logger.log_success_fallback_event( - original_model_group=original_model_group, - kwargs=kwargs, - original_exception=original_exception, +async def _log_fallback_event( + succeeded: bool, + original_model_group: str, + kwargs: dict[str, object], # mutable-ok: the request kwargs the custom loggers receive + original_exception: Exception, +) -> None: + callbacks: Final = reduce(_with_event_logger, litellm.logging_callback_manager.get_all_callbacks(), ()) + fields: Final = MappingProxyType( + { + Field.deprecated_kwargs: kwargs, + Field.model: original_model_group, + Field.error: original_exception, + Field.outcome: "fallback_success" if succeeded else "fallback_failure", + } + ) + subscribers: Final = hook_subscribers(Hook.router_event, callbacks, admit=Admit.every_logger) + _ = kwargs_after( + await run_subscribers( + subscribers, fields, partial(_start_fallback_event, succeeded, original_model_group, original_exception) + ) + ) + + +def _with_event_logger(kept: tuple[object, ...], callback: object) -> tuple[object, ...]: + if not isinstance(callback, CustomLogger | CustomLoggerV2) or callback in kept: + return kept + return (*kept, callback) + + +def _start_fallback_event( + succeeded: bool, original_model_group: str, original_exception: Exception, callback: object, kwargs: object +) -> Awaitable[None] | None: + if not isinstance(callback, CustomLogger): + return None + return _deliver_fallback_event( + callback, + succeeded, + original_model_group, + kwargs, # pyright: ignore[reportArgumentType] # the request kwargs as the router passed them + original_exception, + ) + + +async def _deliver_fallback_event( + callback: CustomLogger, + succeeded: bool, + original_model_group: str, + kwargs: dict[str, object], # mutable-ok: the request kwargs the custom loggers receive + original_exception: Exception, +) -> None: + try: + if succeeded: + await callback.log_success_fallback_event( + original_model_group=original_model_group, kwargs=kwargs, original_exception=original_exception ) - except Exception as e: - verbose_router_logger.error("Error in log_success_fallback_event: %s", e) - - -async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): - """ - Log a failed fallback event to all registered callbacks. - - Uses LoggingCallbackManager.get_custom_loggers_for_type() to get deduplicated - CustomLogger instances from all callback lists. - - Args: - original_model_group (str): The original model group before fallback. - kwargs (dict): kwargs for the request - - Note: - Errors during logging are caught and reported but do not interrupt the process. - """ - # Get deduplicated CustomLogger instances from all callback lists - custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger) - - for _callback_custom_logger in custom_loggers: - try: - await _callback_custom_logger.log_failure_fallback_event( - original_model_group=original_model_group, - kwargs=kwargs, - original_exception=original_exception, + else: + await callback.log_failure_fallback_event( + original_model_group=original_model_group, kwargs=kwargs, original_exception=original_exception ) - except Exception as e: - verbose_router_logger.error("Error in log_failure_fallback_event: %s", e) + except Exception as e: + verbose_router_logger.error( + "Error in log_success_fallback_event: %s" if succeeded else "Error in log_failure_fallback_event: %s", e + ) def check_non_standard_fallback_format(fallbacks: Sequence[object] | None) -> bool: diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index c4cc9350cfa..0f90fbbe1d8 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -1,5 +1,5 @@ """ -Wrapper around router cache for health-check-driven routing. +Health-check-driven routing state, kept in the native ``DeploymentHealthStore``. Stores per-deployment health state from background health checks and exposes it for router candidate filtering. @@ -9,14 +9,10 @@ import time from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final -from pydantic import TypeAdapter -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger -from litellm._internal_context import with_service_target -from litellm.caching.caching import DualCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError -from litellm.constants import HEALTH_CHECKS_TARGET +from litellm.redis.stores.deployment_health import DeploymentHealthStates, DeploymentHealthStore if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -26,26 +22,6 @@ else: Span = Any -_CACHED_HEALTH_SNAPSHOT: Final = TypeAdapter(object) - - -class DeploymentHealthStateValue(TypedDict): - is_healthy: bool - timestamp: float - reason: str - - -@with_service_target(HEALTH_CHECKS_TARGET) -def _read_shared_health_snapshot(cache: DualCache, key: str) -> object: - redis_cache: Final = cache.redis_cache - if redis_cache is None: - return None - try: - return redis_cache.get_cache(key) - except RedisCircuitBreakerOpenError: - return None - - class DeploymentHealthCache: """ Cache for deployment health states produced by background health checks. @@ -55,14 +31,11 @@ class DeploymentHealthCache: are treated as healthy (unknown). """ - CACHE_KEY = "litellm:health_check:deployment_health_state" - - def __init__(self, cache: DualCache, staleness_threshold: float): - self.cache = cache + def __init__(self, store: DeploymentHealthStore, staleness_threshold: float): + self.store: Final = store self.staleness_threshold = staleness_threshold - @with_service_target(HEALTH_CHECKS_TARGET) - def set_deployment_health_states(self, states: dict[str, DeploymentHealthStateValue]) -> None: + def set_deployment_health_states(self, states: DeploymentHealthStates) -> None: """Merge the given states into the shared cache entry, pruning expired ones. Merging instead of replacing lets writers probing different deployment @@ -75,46 +48,46 @@ class DeploymentHealthCache: merge is used so peers are not erased. """ try: - redis_raw: Final = _read_shared_health_snapshot(self.cache, self.CACHE_KEY) - raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY) - existing: Final = raw if isinstance(raw, dict) else {} + shared: Final = self.store.get_shared_sync() + existing: Final = shared if shared is not None else self.store.get_sync() or {} expiry_seconds: Final = self.staleness_threshold * 1.5 now: Final = time.time() - merged: Final = { - model_id: state - for model_id, state in {**existing, **states}.items() - if isinstance(state, dict) and (now - _timestamp_of(state)) < expiry_seconds # pyright: ignore[reportUnknownArgumentType] # a cached record - } - self.cache.set_cache( - key=self.CACHE_KEY, - value=merged, - ttl=int(expiry_seconds), - ) + self.store.set_sync(_unexpired({**existing, **states}, now - expiry_seconds), int(expiry_seconds)) except Exception as e: verbose_logger.error( "DeploymentHealthCache::set_deployment_health_states - Exception: %s", str(e), ) - def _extract_unhealthy_ids(self, raw: object) -> set[str]: - """Given raw cache value, return set of non-stale unhealthy deployment IDs.""" - if not raw or not isinstance(raw, dict): - return set() + def _extract_unhealthy_ids(self, states: DeploymentHealthStates | None) -> set[str]: + """The non-stale unhealthy deployment IDs.""" now: Final = time.time() return { model_id - for model_id, state in raw.items() - if isinstance(state, dict) - and not state.get("is_healthy", True) - and (now - _timestamp_of(state)) < self.staleness_threshold # pyright: ignore[reportUnknownArgumentType] # a cached record + for model_id, state in (states or {}).items() + if not state["is_healthy"] and now - state["timestamp"] < self.staleness_threshold } - @with_service_target(HEALTH_CHECKS_TARGET) + async def async_get_states(self) -> DeploymentHealthStates | None: + """Every deployment's state, as the router core's health filter reads it; a failed read is None.""" + try: + return await self.store.get() + except Exception as e: # noqa: BLE001 # a failed read counts as no health state, as the DualCache read it replaced swallowed errors + verbose_logger.debug("DeploymentHealthCache::async_get_states - Exception: %s", str(e)) + return None + + def get_states(self) -> DeploymentHealthStates | None: + """``async_get_states`` for a sync call.""" + try: + return self.store.get_sync() + except Exception as e: # noqa: BLE001 # a failed read counts as no health state, as the DualCache read it replaced swallowed errors + verbose_logger.debug("DeploymentHealthCache::get_states - Exception: %s", str(e)) + return None + async def async_get_unhealthy_deployment_ids(self, parent_otel_span: Span | None = None) -> set[str]: """Return set of deployment IDs currently marked unhealthy and not stale.""" try: - raw: Final = _CACHED_HEALTH_SNAPSHOT.validate_python(await self.cache.async_get_cache(key=self.CACHE_KEY)) - return self._extract_unhealthy_ids(raw) + return self._extract_unhealthy_ids(await self.store.get()) except Exception as e: verbose_logger.debug( "DeploymentHealthCache::async_get_unhealthy_deployment_ids - Exception: %s", @@ -122,12 +95,10 @@ class DeploymentHealthCache: ) return set() - @with_service_target(HEALTH_CHECKS_TARGET) def get_unhealthy_deployment_ids(self, parent_otel_span: Span | None = None) -> set[str]: """Sync version: return set of deployment IDs currently marked unhealthy and not stale.""" try: - raw: Final = _CACHED_HEALTH_SNAPSHOT.validate_python(self.cache.get_cache(key=self.CACHE_KEY)) - return self._extract_unhealthy_ids(raw) + return self._extract_unhealthy_ids(self.store.get_sync()) except Exception as e: verbose_logger.debug( "DeploymentHealthCache::get_unhealthy_deployment_ids - Exception: %s", @@ -136,8 +107,18 @@ class DeploymentHealthCache: return set() -def _timestamp_of(state: Mapping[str, object]) -> float: - """A health record's timestamp; one that is no number reads as 0 (stale), so a bad record - drops out alone instead of turning health routing off for every deployment.""" - timestamp: Final = state.get("timestamp", 0) - return float(timestamp) if isinstance(timestamp, int | float) and not isinstance(timestamp, bool) else 0.0 +class DeploymentHealthCheck(TypedDict): + is_healthy: ReadOnly[bool] + timestamp: ReadOnly[float] + + +def health_states_of(states: DeploymentHealthStates | None) -> Mapping[str, DeploymentHealthCheck]: + """The cached states as router-core reads them, which is everything but the reason.""" + return { + model_id: {"is_healthy": state["is_healthy"], "timestamp": float(state["timestamp"])} + for model_id, state in (states or {}).items() + } + + +def _unexpired(states: DeploymentHealthStates, cutoff: float) -> DeploymentHealthStates: + return {model_id: state for model_id, state in states.items() if state["timestamp"] > cutoff} diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index dbf471d875c..19d80f0c0db 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -24,13 +24,17 @@ from litellm._logging import verbose_router_logger from litellm.caching.affinity_cache import ( ROUTER_SESSION_PINS_TARGET, claim_affinity_pin, - claim_affinity_pin_in_memory, set_local_affinity_pin, ) -from litellm.caching.dual_cache import DualCache from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request +from litellm.redis.stores.router_session_pins import ( + DeploymentPin, + DeploymentSessionPin, + PinKey, + RouterSessionPinStore, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes @@ -97,12 +101,11 @@ class DeploymentAffinityCheck(CustomLogger): `Router(optional_pre_call_checks=[...])`. """ - CACHE_KEY_PREFIX = "deployment_affinity:v1" USER_ID_AFFINITY_PREFIX: Final = "user_id:" def __init__( self, - cache: DualCache, + pins: RouterSessionPinStore, ttl_seconds: int, enable_user_key_affinity: bool, enable_responses_api_affinity: bool, @@ -112,7 +115,7 @@ class DeploymentAffinityCheck(CustomLogger): session_affinity_repin_on_failover: bool = False, ): super().__init__() - self.cache = cache + self.pins: Final = pins self.ttl_seconds = ttl_seconds self.enable_user_key_affinity = enable_user_key_affinity self.enable_responses_api_affinity = enable_responses_api_affinity @@ -261,18 +264,16 @@ class DeploymentAffinityCheck(CustomLogger): return f"{value[:keep]}..." @classmethod - def get_affinity_cache_key(cls, model_group: str, user_key: str) -> str: - hashed_user_key: Final = cls._hash_user_key(user_key=user_key) - return f"{cls.CACHE_KEY_PREFIX}:{model_group}:{hashed_user_key}" + def get_affinity_cache_key(cls, model_group: str, user_key: str) -> DeploymentPin: + return DeploymentPin(model_group=model_group, caller=user_key) @classmethod - def get_session_affinity_cache_key(cls, model_group: str, session_id: str, user_key: str | None) -> str: - """Session pins are scoped by the caller's hashed API key so two callers reusing - the same client-supplied session_id cannot read or steer each other's pin. - `"unscoped"` covers direct Router usage with no authenticated caller, matching - the complexity router's own session pin key.""" - hashed_user_key: Final = cls._hash_user_key(user_key) if user_key is not None else "unscoped" - return f"{cls.CACHE_KEY_PREFIX}:session:{model_group}:{hashed_user_key}:{session_id}" + def get_session_affinity_cache_key( + cls, model_group: str, session_id: str, user_key: str | None + ) -> DeploymentSessionPin: + """Session pins are scoped by the caller so two callers reusing the same client-supplied session_id cannot + read or steer each other's pin.""" + return DeploymentSessionPin(model_group=model_group, caller=user_key, session_id=session_id) @staticmethod def _get_session_id_from_metadata_dict(metadata: Mapping[object, object]) -> str | None: @@ -352,17 +353,13 @@ class DeploymentAffinityCheck(CustomLogger): return stored return None - def _set_local_pin(self, cache_key: str, value: object, ttl_seconds: int) -> None: - set_local_affinity_pin(self.cache, cache_key, value, ttl_seconds) + def _set_local_pin(self, cache_key: PinKey, value: object, ttl_seconds: int) -> None: + set_local_affinity_pin(self.pins, cache_key, value, ttl_seconds) - async def _claim_pin(self, cache_key: str, pin_value: DeploymentAffinityCacheValue, ttl_seconds: int) -> str | None: - winner: Final = await claim_affinity_pin(self.cache, cache_key, pin_value, ttl_seconds) - return self._pinned_model_id(winner) - - def _claim_pin_in_memory( - self, cache_key: str, pin_value: DeploymentAffinityCacheValue, ttl_seconds: int + async def _claim_pin( + self, cache_key: PinKey, pin_value: DeploymentAffinityCacheValue, ttl_seconds: int ) -> str | None: - winner: Final = claim_affinity_pin_in_memory(self.cache, cache_key, pin_value, ttl_seconds) + winner: Final = await claim_affinity_pin(self.pins, cache_key, pin_value, ttl_seconds) return self._pinned_model_id(winner) @staticmethod @@ -453,7 +450,7 @@ class DeploymentAffinityCheck(CustomLogger): parent_otel_span, ) if self.session_affinity_repin_on_failover - else await self.cache.async_get_cache(key=session_cache_key) + else await self.pins.get(session_cache_key) ) session_model_id: str | None = None @@ -488,7 +485,7 @@ class DeploymentAffinityCheck(CustomLogger): return typed_healthy_deployments cache_key: Final = self.get_affinity_cache_key(model_group=stable_model_map_key, user_key=user_key) - cache_result: Final = await self.cache.async_get_cache(key=cache_key) + cache_result: Final = await self.pins.get(cache_key) model_id: str | None = None if isinstance(cache_result, dict): @@ -518,21 +515,20 @@ class DeploymentAffinityCheck(CustomLogger): ) return [deployment] - async def _read_shared_session_pin(self, cache_key: str, ttl_seconds: int, parent_otel_span: Span | None) -> object: - redis_cache: Final = self.cache.redis_cache - if redis_cache is None: - return TypeAdapter(object).validate_python(await self.cache.async_get_cache(key=cache_key)) - local_before: Final = TypeAdapter(object).validate_python(self.cache.in_memory_cache.get_cache(cache_key)) + async def _read_shared_session_pin( + self, cache_key: PinKey, ttl_seconds: int, parent_otel_span: Span | None + ) -> object: + if not self.pins.has_redis: + return await self.pins.get(cache_key) + local_before: Final = self.pins.get_local(cache_key) try: - shared: Final[object] = await redis_cache.async_get_cache( - key=cache_key, parent_otel_span=parent_otel_span, raise_on_error=True - ) + shared: Final = await self.pins.get_shared(cache_key) except Exception: # noqa: BLE001 # Optional affinity falls back locally on Redis admission or read failure verbose_router_logger.debug("Session pin read failed, using pod-local affinity") - return TypeAdapter(object).validate_python(await self.cache.async_get_cache(key=cache_key, local_only=True)) - if self.cache.in_memory_cache.get_cache(cache_key) is local_before: + return self.pins.get_local(cache_key) + if self.pins.get_local(cache_key) == local_before: if shared is None: - self.cache.in_memory_cache.delete_cache(cache_key) + self.pins.delete_local(cache_key) else: self._set_local_pin(cache_key, shared, ttl_seconds) return shared @@ -575,7 +571,7 @@ class DeploymentAffinityCheck(CustomLogger): model_group, session_id, self.get_user_key_from_request_kwargs(request_kwargs) ) await claim_affinity_pin( - self.cache, + self.pins, cache_key, DeploymentAffinityCacheValue(model_id=str(model_id)), marker_ttl if marker_ttl is not None else self.ttl_seconds, diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 4b618aba476..0e551de7842 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -18,21 +18,19 @@ import threading import time from collections import OrderedDict from collections.abc import Callable, Mapping, MutableMapping -from dataclasses import dataclass +from dataclasses import asdict, dataclass, replace from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import Any, Final import httpx -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid -from litellm.caching.dual_cache import DualCache -from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION from litellm.litellm_core_utils.attempt_settlement import current_attempt_id, release_in_background, released from litellm.litellm_core_utils.core_helpers import coerce_token_limit @@ -40,25 +38,19 @@ from litellm.litellm_core_utils.internal_call_metadata import ( IO_TOKEN_ATTEMPT_KEY, IO_TOKEN_RESERVATION_ID_KEY, IO_TOKEN_RESERVATION_KEYS, - ITPM_CACHE_KEY, + ITPM_COUNTER_KEY, ITPM_RESERVED_KEY, - OTPM_CACHE_KEY, + OTPM_COUNTER_KEY, OTPM_RESERVED_KEY, ) from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET -from litellm.types.router import RouterCacheEnum, RouterErrors +from litellm.redis.stores.router_usage import MAX_INCREMENT, ROUTER_USAGE_TARGET, RouterUsage, UsageCounter +from litellm.types.router import RouterErrors from litellm.utils import get_utc_datetime _MODEL_INFO_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) - -if TYPE_CHECKING: - from opentelemetry.trace import Span as _Span - - Span = _Span -else: - Span = Any +_STASHED_COUNTER: Final[TypeAdapter[UsageCounter]] = TypeAdapter(UsageCounter) RoutingArgsTTL: Final = 60 @@ -177,16 +169,17 @@ def _without_io_token_limit_keys(section: object) -> object: return {key: value for key, value in section.items() if key not in _IO_TOKEN_LIMIT_KEYS} -def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | None: - model_id: Final = deployment.get("model_info", {}).get("id") - deployment_name: Final = deployment.get("litellm_params", {}).get("model") +def _get_counters(deployment: dict, current_minute: str) -> tuple[UsageCounter, UsageCounter] | None: + model_id: Final[object] = deployment.get("model_info", {}).get("id") + deployment_name: Final[object] = deployment.get("litellm_params", {}).get("model") # Without both a deployment id and model name the key would collapse to a # shared "None:None" bucket across misconfigured deployments, so bail out. if model_id is None or deployment_name is None: return None - itpm_key = RouterCacheEnum.ITPM.value.format(id=model_id, model=deployment_name, current_minute=current_minute) - otpm_key = RouterCacheEnum.OTPM.value.format(id=model_id, model=deployment_name, current_minute=current_minute) - return itpm_key, otpm_key + return ( + UsageCounter("global_router", f"{model_id}", f"{deployment_name}", current_minute, "itpm"), + UsageCounter("global_router", f"{model_id}", f"{deployment_name}", current_minute, "otpm"), + ) def _served_model(deployment: object) -> str: @@ -387,15 +380,16 @@ def _resolve_reconcile_usage_tokens( def _reservation( - itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None -) -> Mapping[str, int | str | None]: + itpm_reserved: int, otpm_reserved: int, itpm: UsageCounter | None, otpm: UsageCounter | None +) -> Mapping[str, object]: + """The reservation as plain fields, which survive the copies logging makes of the request metadata.""" return MappingProxyType( { IO_TOKEN_RESERVATION_ID_KEY: uuid.uuid4().hex, ITPM_RESERVED_KEY: itpm_reserved, OTPM_RESERVED_KEY: otpm_reserved, - ITPM_CACHE_KEY: itpm_cache_key, - OTPM_CACHE_KEY: otpm_cache_key, + ITPM_COUNTER_KEY: None if itpm is None else asdict(itpm), + OTPM_COUNTER_KEY: None if otpm is None else asdict(otpm), } ) @@ -407,33 +401,44 @@ def _stash_reservation_in_metadata(request: _IoTokenRateLimitRequest | None, res metadata[IO_TOKEN_ATTEMPT_KEY] = current_attempt_id() -def _key_for_this_minute(reserved_key: str) -> str: - counter, _, _ = reserved_key.rpartition(":") - return f"{counter}:{get_utc_datetime().strftime('%H-%M')}" +def _this_minute(counter: UsageCounter) -> UsageCounter: + return replace(counter, window=get_utc_datetime().strftime("%H-%M")) + + +_Reservation = tuple[int, int, UsageCounter | None, UsageCounter | None] def _reconcile_increments( - reservation: tuple[int, int, str | None, str | None], billable_input: int, completion_tokens: int -) -> tuple[tuple[str, int], ...]: - itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation - deltas: Final = ((itpm_key, billable_input - itpm_reserved), (otpm_key, completion_tokens - otpm_reserved)) + reservation: _Reservation, billable_input: int, completion_tokens: int +) -> tuple[tuple[UsageCounter, int], ...]: + itpm_reserved, otpm_reserved, itpm, otpm = reservation + deltas: Final = ((itpm, billable_input - itpm_reserved), (otpm, completion_tokens - otpm_reserved)) return tuple( - (key if delta < 0 else _key_for_this_minute(key), delta) for key, delta in deltas if key is not None and delta + (counter if delta < 0 else _this_minute(counter), delta) + for counter, delta in deltas + if counter is not None and delta ) +def _stashed_counter(value: object) -> UsageCounter | None: + if value is None: + return None + try: + return _STASHED_COUNTER.validate_python(value) + except ValidationError: + return None + + def _reserved_tokens(value: object) -> int: return value if isinstance(value, int) and not isinstance(value, bool) else 0 -def _extract_reservation(reservation: Mapping[str, object]) -> tuple[int, int, str | None, str | None]: - itpm_cache_key: Final = reservation.get(ITPM_CACHE_KEY) - otpm_cache_key: Final = reservation.get(OTPM_CACHE_KEY) +def _extract_reservation(reservation: Mapping[str, object]) -> _Reservation: return ( _reserved_tokens(reservation.get(ITPM_RESERVED_KEY)), _reserved_tokens(reservation.get(OTPM_RESERVED_KEY)), - itpm_cache_key if isinstance(itpm_cache_key, str) else None, - otpm_cache_key if isinstance(otpm_cache_key, str) else None, + _stashed_counter(reservation.get(ITPM_COUNTER_KEY)), + _stashed_counter(reservation.get(OTPM_COUNTER_KEY)), ) @@ -465,9 +470,7 @@ def _stashed_reservation(kwargs: Mapping[str, object] | None) -> Mapping[str, ob return next((channel for channel in channels if channel is not None and ITPM_RESERVED_KEY in channel), None) -def _claim_reservation_from_kwargs( - kwargs: Mapping[str, object] | None, *, any_attempt: bool = False -) -> tuple[int, int, str | None, str | None]: +def _claim_reservation_from_kwargs(kwargs: Mapping[str, object] | None, *, any_attempt: bool = False) -> _Reservation: """ Take the stashed reservation so nothing else settles it. Under a routed attempt only that attempt's own reservation is taken, unless ``any_attempt``: a late callback of an earlier attempt sharing these @@ -535,8 +538,8 @@ def _rate_limit_error(limit_label: str, limit: int, current: float) -> litellm.R @with_service_target(ROUTER_USAGE_TARGET) def _sync_increment_with_rollback( - dual_cache: DualCache, - key: str, + usage: RouterUsage, + counter: UsageCounter, value: int, limit: int | None, *, @@ -544,51 +547,36 @@ def _sync_increment_with_rollback( ) -> None: if value <= 0 or limit is None: return - current: Final = dual_cache.increment_cache( - key=key, - value=value, - ttl=RoutingArgsTTL, - ) + if value > min(limit, MAX_INCREMENT): + raise _rate_limit_error(limit_label, limit, value) + current: Final = usage.increment_sync(counter, value, RoutingArgsTTL) if current is not None and current > limit: - dual_cache.increment_cache( - key=key, - value=-value, - ttl=RoutingArgsTTL, - ) + usage.increment_sync(counter, -value, RoutingArgsTTL) raise _rate_limit_error(limit_label, limit, current) @with_service_target(ROUTER_USAGE_TARGET) async def _increment_with_rollback( - dual_cache: DualCache, - key: str, + usage: RouterUsage, + counter: UsageCounter, value: int, limit: int | None, *, - parent_otel_span: Span | None = None, limit_label: str, ) -> None: if value <= 0 or limit is None: return - current: Final = await dual_cache.async_increment_cache( - key=key, - value=value, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) + if value > min(limit, MAX_INCREMENT): + raise _rate_limit_error(limit_label, limit, value) + current: Final = await usage.increment(counter, value, RoutingArgsTTL) if current is not None and current > limit: - await dual_cache.async_increment_cache( - key=key, - value=-value, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) + await usage.increment(counter, -value, RoutingArgsTTL) raise _rate_limit_error(limit_label, limit, current) @with_service_target(ROUTER_USAGE_TARGET) def io_token_pre_call_check( - dual_cache: DualCache, + usage: RouterUsage, deployment: dict, ) -> dict | None: itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment) @@ -605,10 +593,10 @@ def io_token_pre_call_check( dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") - cache_keys: Final = _get_cache_keys(deployment, current_minute) - if cache_keys is None: + counters: Final = _get_counters(deployment, current_minute) + if counters is None: return deployment - itpm_key, otpm_key = cache_keys + itpm_counter, otpm_counter = counters itpm_reserved = 0 otpm_reserved = 0 @@ -616,8 +604,8 @@ def io_token_pre_call_check( if itpm_limit is not None: itpm_reserved = _reservation_value(estimated_input, itpm_limit) _sync_increment_with_rollback( - dual_cache, - itpm_key, + usage, + itpm_counter, itpm_reserved, itpm_limit, limit_label="ITPM", @@ -627,19 +615,15 @@ def io_token_pre_call_check( otpm_reserved = otpm_to_reserve try: _sync_increment_with_rollback( - dual_cache, - otpm_key, + usage, + otpm_counter, otpm_reserved, otpm_limit, limit_label="OTPM", ) except Exception: if itpm_reserved > 0: - dual_cache.increment_cache( - key=itpm_key, - value=-itpm_reserved, - ttl=RoutingArgsTTL, - ) + usage.increment_sync(itpm_counter, -itpm_reserved, RoutingArgsTTL) raise _stash_reservation_in_metadata( @@ -647,8 +631,8 @@ def io_token_pre_call_check( _reservation( itpm_reserved, otpm_reserved, - itpm_key if itpm_limit is not None else None, - otpm_key if otpm_limit is not None else None, + itpm_counter if itpm_limit is not None else None, + otpm_counter if otpm_limit is not None else None, ), ) return deployment @@ -656,9 +640,8 @@ def io_token_pre_call_check( @with_service_target(ROUTER_USAGE_TARGET) async def async_io_token_pre_call_check( - dual_cache: DualCache, + usage: RouterUsage, deployment: dict, - parent_otel_span: Span | None = None, ) -> dict | None: itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment) if itpm_limit is None and otpm_limit is None: @@ -669,7 +652,7 @@ async def async_io_token_pre_call_check( if request is not None and request.opens_responses_socket: _stash_reservation_in_metadata( request, - await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span), + await async_admit_io_token_turn(usage, deployment, get_utc_datetime()), ) return deployment _model, served_model = _tokenizer_and_served_models(deployment) @@ -680,72 +663,66 @@ async def async_io_token_pre_call_check( dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") - cache_keys: Final = _get_cache_keys(deployment, current_minute) - if cache_keys is None: + counters: Final = _get_counters(deployment, current_minute) + if counters is None: return deployment - itpm_key, otpm_key = cache_keys + itpm_counter, otpm_counter = counters # The reservation runs in a task of its own: a request cancelled mid-reservation never knows how much # it took, so the task finishes and then hands back whatever it reserved. reserving: Final = asyncio.get_running_loop().create_task( _async_reserve( - dual_cache, - itpm_key, - otpm_key, + usage, + itpm_counter, + otpm_counter, _reservation_value(estimated_input, itpm_limit) if itpm_limit is not None else 0, otpm_to_reserve, itpm_limit, otpm_limit, - parent_otel_span, ) ) try: reservation: Final = await asyncio.shield(reserving) except asyncio.CancelledError: - _ = release_in_background(_refund_once_reserved(dual_cache, reserving, parent_otel_span)) + _ = release_in_background(_refund_once_reserved(usage, reserving)) raise _stash_reservation_in_metadata(request, reservation) return deployment -async def _refund_once_reserved( - dual_cache: DualCache, reserving: asyncio.Task[Mapping[str, int | str | None]], parent_otel_span: Span | None -) -> None: +async def _refund_once_reserved(usage: RouterUsage, reserving: asyncio.Task[Mapping[str, object]]) -> None: try: reservation: Final = await reserving except Exception: # noqa: BLE001 # a refused or failed reservation holds nothing return - await _async_increment_all(dual_cache, _refunds(_extract_reservation(reservation)), parent_otel_span) + await _async_increment_all(usage, _refunds(_extract_reservation(reservation))) async def _async_reserve( - dual_cache: DualCache, - itpm_key: str, - otpm_key: str, + usage: RouterUsage, + itpm_counter: UsageCounter, + otpm_counter: UsageCounter, itpm_reserved: int, otpm_reserved: int, itpm_limit: int | None, otpm_limit: int | None, - parent_otel_span: Span | None, -) -> Mapping[str, int | str | None]: +) -> Mapping[str, object]: if itpm_limit is not None: await _increment_with_rollback( - dual_cache, - itpm_key, + usage, + itpm_counter, itpm_reserved, itpm_limit, - parent_otel_span=parent_otel_span, limit_label="ITPM", ) if otpm_limit is not None: try: await _increment_with_rollback( - dual_cache, - otpm_key, + usage, + otpm_counter, otpm_reserved, otpm_limit, - parent_otel_span=parent_otel_span, limit_label="OTPM", ) except Exception: @@ -753,88 +730,64 @@ async def _async_reserve( # release the ITPM reservation already made, or it stays inflated # until the TTL expires. if itpm_reserved > 0: - await dual_cache.async_increment_cache( - key=itpm_key, - value=-itpm_reserved, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) + await usage.increment(itpm_counter, -itpm_reserved, RoutingArgsTTL) raise return _reservation( itpm_reserved, otpm_reserved if otpm_limit is not None else 0, - itpm_key if itpm_limit is not None else None, - otpm_key if otpm_limit is not None else None, + itpm_counter if itpm_limit is not None else None, + otpm_counter if otpm_limit is not None else None, ) -async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None: - if redis_cache is None: - return None - with contextlib.suppress(RedisCircuitBreakerOpenError): - return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) - return None - - -@with_service_target(ROUTER_USAGE_TARGET) -async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None: - shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span) - if shared is not None: - return shared - return await dual_cache.async_get_cache(key=key, local_only=True) - - -async def _raise_if_at_limit( - dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None -) -> None: +async def _raise_if_at_limit(usage: RouterUsage, counter: UsageCounter, limit: int | None, limit_label: str) -> None: if limit is None: return - usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span) - if usage is not None and usage >= limit: - raise _rate_limit_error(limit_label, limit, usage) + count: Final = await usage.shared_or_local(counter) + if count is not None and count >= limit: + raise _rate_limit_error(limit_label, limit, count) @with_service_target(ROUTER_USAGE_TARGET) async def async_admit_io_token_turn( - dual_cache: DualCache, + usage: RouterUsage, deployment: Mapping[str, object], now: datetime, - parent_otel_span: Span | None = None, ) -> Mapping[str, object]: configured: Final = dict(deployment) itpm_limit, otpm_limit = get_deployment_io_token_limits(configured) if itpm_limit is None and otpm_limit is None: return MappingProxyType({}) - cache_keys: Final = _get_cache_keys(configured, now.strftime("%H-%M")) - if cache_keys is None: + counters: Final = _get_counters(configured, now.strftime("%H-%M")) + if counters is None: return MappingProxyType({}) - itpm_key, otpm_key = cache_keys - await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span) - await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span) + itpm_counter, otpm_counter = counters + await _raise_if_at_limit(usage, itpm_counter, itpm_limit, "ITPM") + await _raise_if_at_limit(usage, otpm_counter, otpm_limit, "OTPM") return _reservation( - 0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None + 0, 0, itpm_counter if itpm_limit is not None else None, otpm_counter if otpm_limit is not None else None ) @with_service_target(ROUTER_USAGE_TARGET) def io_token_reconcile_success( - dual_cache: DualCache, + usage: RouterUsage, kwargs: Mapping[str, object] | None, response_obj: object, ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj reservation: Final = _claim_reservation_from_kwargs(request_kwargs) - itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation - if itpm_key is None and otpm_key is None: + itpm_reserved, otpm_reserved, itpm_counter, otpm_counter = reservation + if itpm_counter is None and otpm_counter is None: return billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) if usage_resolved: - for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens): - dual_cache.increment_cache(key=key, value=delta, ttl=RoutingArgsTTL) + for counter, delta in _reconcile_increments(reservation, billable_input, completion_tokens): + usage.increment_sync(counter, delta, RoutingArgsTTL) else: verbose_router_logger.debug( "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", @@ -854,17 +807,15 @@ def io_token_reconcile_success( @with_service_target(ROUTER_USAGE_TARGET) async def async_io_token_reconcile_success( - dual_cache: DualCache, + usage: RouterUsage, kwargs: Mapping[str, object] | None, response_obj: object, - *, - parent_otel_span: Span | None = None, ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj reservation: Final = _claim_reservation_from_kwargs(request_kwargs) - itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation - if itpm_key is None and otpm_key is None: + itpm_reserved, otpm_reserved, itpm_counter, otpm_counter = reservation + if itpm_counter is None and otpm_counter is None: return billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) @@ -877,9 +828,8 @@ async def async_io_token_reconcile_success( ( release_in_background( _async_increment_all( - dual_cache, + usage, _reconcile_increments(reservation, billable_input, completion_tokens), - parent_otel_span, ) ), ) @@ -903,7 +853,7 @@ async def async_io_token_reconcile_success( @with_service_target(ROUTER_USAGE_TARGET) def io_token_refund_failure( - dual_cache: DualCache, + usage: RouterUsage, kwargs: Mapping[str, object] | None, *, any_attempt: bool = False, @@ -911,12 +861,12 @@ def io_token_refund_failure( request_kwargs: Final[Mapping[str, object] | None] = kwargs reservation: Final = _claim_reservation_from_kwargs(request_kwargs, any_attempt=any_attempt) itpm_reserved, otpm_reserved, _, _ = reservation - for key, delta in _refunds(reservation): - dual_cache.increment_cache(key=key, value=delta, ttl=RoutingArgsTTL) + for counter, delta in _refunds(reservation): + usage.increment_sync(counter, delta, RoutingArgsTTL) verbose_router_logger.debug("[IO TOKEN LIMIT] refunded ITPM=%s OTPM=%s", itpm_reserved, otpm_reserved) -def refund_stale_reservation_before_retry(dual_cache: DualCache, kwargs: Mapping[str, object] | None) -> None: +def refund_stale_reservation_before_retry(usage: RouterUsage, kwargs: Mapping[str, object] | None) -> None: """ Synchronously refund and clear any reservation a previous deployment attempt stashed in ``kwargs``, before it's overwritten for the next @@ -940,48 +890,40 @@ def refund_stale_reservation_before_retry(dual_cache: DualCache, kwargs: Mapping """ if not kwargs: return - io_token_refund_failure(dual_cache, kwargs, any_attempt=True) + io_token_refund_failure(usage, kwargs, any_attempt=True) @with_service_target(ROUTER_USAGE_TARGET) async def async_io_token_refund_failure( - dual_cache: DualCache, + usage: RouterUsage, kwargs: Mapping[str, object] | None, - *, - parent_otel_span: Span | None = None, ) -> None: - refund: Final = release_io_token_reservation(dual_cache, kwargs, parent_otel_span=parent_otel_span) + refund: Final = release_io_token_reservation(usage, kwargs) if refund is not None: await released((refund,)) def release_io_token_reservation( - dual_cache: DualCache, + usage: RouterUsage, kwargs: Mapping[str, object] | None, - *, - parent_otel_span: Span | None = None, ) -> asyncio.Task[None] | None: reservation: Final = _claim_reservation_from_kwargs(kwargs) refunds: Final = _refunds(reservation) if not refunds: return None verbose_router_logger.debug("[IO TOKEN LIMIT] refunding ITPM=%s OTPM=%s", reservation[0], reservation[1]) - return release_in_background(_async_increment_all(dual_cache, refunds, parent_otel_span)) + return release_in_background(_async_increment_all(usage, refunds)) -def _refunds(reservation: tuple[int, int, str | None, str | None]) -> tuple[tuple[str, int], ...]: - itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation - held: Final = ((itpm_key, itpm_reserved), (otpm_key, otpm_reserved)) - return tuple((key, -reserved) for key, reserved in held if key is not None and reserved > 0) +def _refunds(reservation: _Reservation) -> tuple[tuple[UsageCounter, int], ...]: + itpm_reserved, otpm_reserved, itpm_counter, otpm_counter = reservation + held: Final = ((itpm_counter, itpm_reserved), (otpm_counter, otpm_reserved)) + return tuple((counter, -reserved) for counter, reserved in held if counter is not None and reserved > 0) -async def _async_increment_all( - dual_cache: DualCache, increments: tuple[tuple[str, int], ...], parent_otel_span: Span | None -) -> None: - for key, delta in increments: - await dual_cache.async_increment_cache( - key=key, value=delta, ttl=RoutingArgsTTL, parent_otel_span=parent_otel_span - ) +async def _async_increment_all(usage: RouterUsage, increments: tuple[tuple[UsageCounter, int], ...]) -> None: + for counter, delta in increments: + await usage.increment(counter, delta, RoutingArgsTTL) def build_io_token_rate_limit_headers( diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index 38ae7e0d951..10650def2f3 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -22,10 +22,8 @@ from pydantic import BaseModel, ConfigDict import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger -from litellm.caching.dual_cache import DualCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.integrations.custom_logger import CustomLogger -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET +from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, CounterIncrement, RouterUsage, UsageCounter from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( async_io_token_pre_call_check, async_io_token_reconcile_success, @@ -36,7 +34,6 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( io_token_reconcile_success, io_token_refund_failure, ) -from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.router import RouterErrors, parse_daily_reset_time from litellm.types.utils import StandardLoggingPayload from litellm.utils import get_utc_datetime @@ -97,12 +94,18 @@ class _DailyLimitDeployment(BaseModel): def reset_offset(self) -> timedelta: return self.litellm_params.reset_offset() - def cache_keys(self, now: datetime) -> tuple[str, str]: - prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}" + def counters(self, now: datetime) -> tuple[UsageCounter, UsageCounter]: current_day: Final = daily_window_day(now, self.reset_offset()) - return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}" + model_id: Final = self.model_info.id + model: Final = self.litellm_params.model + return ( + UsageCounter.for_deployment(f"{model_id}", f"{model}", "tpd", current_day), + UsageCounter.for_deployment(f"{model_id}", f"{model}", "rpd", current_day), + ) - def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError: + def limit_error( + self, kind: Literal["tpd", "rpd"], limit: int, usage: float, now: datetime + ) -> litellm.RateLimitError: return litellm.RateLimitError( message=f"Model rate limit exceeded. {kind.upper()} limit={limit}, current usage={usage}", llm_provider="", @@ -134,58 +137,44 @@ class ModelRateLimitingCheck(CustomLogger): this check actively enforces those limits across ALL routing strategies. """ - def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime) -> None: - self.dual_cache = dual_cache + def __init__(self, usage: RouterUsage, clock: Callable[[], datetime] = get_utc_datetime) -> None: + self.usage = usage self.clock = clock # model_ids already warned about itpm/otpm + tpm/rpm on the same deployment, # so the warning is logged once per deployment rather than per request. self._io_token_conflict_warned_ids: set[str] = set() - def _increment(self, key: str, value: int, ttl: int) -> int: - return self.dual_cache.increment_cache(key=key, value=value, ttl=ttl) - - async def _async_increment( - self, key: str, value: int, ttl: int, parent_otel_span: Span | None = None - ) -> int | None: - count: Final = await self.dual_cache.async_increment_cache( - key=key, value=value, ttl=ttl, parent_otel_span=parent_otel_span - ) - return None if count is None else int(count) + def _increment(self, counter: UsageCounter, value: int, ttl: int) -> float | None: + return self.usage.increment_sync(counter, value, ttl) @staticmethod def _request_count_operations( - rpm_key: str | None, rpd_key: str | None, increment_value: int - ) -> tuple[RedisPipelineIncrementOperation, ...]: + rpm: UsageCounter | None, rpd: UsageCounter | None, increment_value: int + ) -> tuple[CounterIncrement, ...]: return tuple( - RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=ttl) - for key, ttl in ((rpm_key, RoutingArgs.ttl), (rpd_key, RoutingArgs.daily_ttl)) - if key is not None + CounterIncrement(counter=counter, amount=increment_value, ttl=ttl) + for counter, ttl in ((rpm, RoutingArgs.ttl), (rpd, RoutingArgs.daily_ttl)) + if counter is not None ) - async def _async_refund_request_counts( - self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None - ) -> None: - operations: Final = self._request_count_operations(rpm_key, rpd_key, -1) + async def _async_refund_request_counts(self, rpm: UsageCounter | None, rpd: UsageCounter | None) -> None: + operations: Final = self._request_count_operations(rpm, rpd, -1) if operations: - await self.dual_cache.async_increment_cache_pipeline( - increment_list=list(operations), parent_otel_span=parent_otel_span - ) + await self.usage.increment_pipeline(operations) async def _async_increment_request_counts( - self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None + self, rpm: UsageCounter | None, rpd: UsageCounter | None ) -> tuple[int | None, int | None]: - operations: Final = self._request_count_operations(rpm_key, rpd_key, 1) + operations: Final = self._request_count_operations(rpm, rpd, 1) if not operations: return None, None - counts: Final = await self.dual_cache.async_increment_cache_pipeline( - increment_list=list(operations), parent_otel_span=parent_otel_span - ) + counts: Final = await self.usage.increment_pipeline(operations) if counts is None or len(counts) != len(operations): return None, None - count_by_key: Final = {op["key"]: int(count) for op, count in zip(operations, counts)} + count_by_counter: Final = {op.counter: int(count) for op, count in zip(operations, counts)} return ( - None if rpm_key is None else count_by_key.get(rpm_key), - None if rpd_key is None else count_by_key.get(rpd_key), + None if rpm is None else count_by_counter.get(rpm), + None if rpd is None else count_by_counter.get(rpd), ) def _warn_io_token_and_tpm_rpm_coexist_once(self, deployment: dict) -> None: @@ -207,19 +196,12 @@ class ModelRateLimitingCheck(CustomLogger): def _refund_io_token_reservation_if_any(self) -> None: request_kwargs: Final = get_io_token_rate_limit_request_kwargs() if request_kwargs is not None: - io_token_refund_failure(self.dual_cache, request_kwargs) + io_token_refund_failure(self.usage, request_kwargs) - async def _async_refund_io_token_reservation_if_any( - self, - parent_otel_span: Span | None = None, - ) -> None: + async def _async_refund_io_token_reservation_if_any(self) -> None: request_kwargs: Final = get_io_token_rate_limit_request_kwargs() if request_kwargs is not None: - await async_io_token_refund_failure( - self.dual_cache, - request_kwargs, - parent_otel_span=parent_otel_span, - ) + await async_io_token_refund_failure(self.usage, request_kwargs) def _get_deployment_limits(self, deployment: dict) -> tuple[int | None, int | None]: """ @@ -255,48 +237,25 @@ class ModelRateLimitingCheck(CustomLogger): tpd_limit, rpd_limit = deployment.limits() if tpd_limit is None and rpd_limit is None: return - tpd_key, rpd_key = deployment.cache_keys(now) + tpd, rpd = deployment.counters(now) if tpd_limit is not None: - current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit) + current_tpd: Final = self.usage.current_sync(tpd, tpd_limit) if current_tpd is not None and current_tpd >= tpd_limit: raise deployment.limit_error("tpd", tpd_limit, current_tpd, now) if rpd_limit is not None: - current_rpd: Final = self._increment(rpd_key, 1, RoutingArgs.daily_ttl) - if current_rpd > rpd_limit: - self._increment(rpd_key, -1, RoutingArgs.daily_ttl) + current_rpd: Final = self._increment(rpd, 1, RoutingArgs.daily_ttl) + if current_rpd is not None and current_rpd > rpd_limit: + self._increment(rpd, -1, RoutingArgs.daily_ttl) raise deployment.limit_error("rpd", rpd_limit, current_rpd, now) - def _get_cache_keys(self, deployment: dict, current_minute: str) -> tuple[str, str]: - """Get the cache keys for TPM and RPM tracking.""" - model_id: Final = deployment.get("model_info", {}).get("id") - deployment_name: Final = deployment.get("litellm_params", {}).get("model") - - tpm_key: Final = f"{model_id}:{deployment_name}:tpm:{current_minute}" - rpm_key: Final = f"{model_id}:{deployment_name}:rpm:{current_minute}" - - return tpm_key, rpm_key - - @with_service_target(ROUTER_USAGE_TARGET) - def _get_current_tpm(self, tpm_key: str, tpm_limit: int) -> int | None: - local_tpm: Final = self.dual_cache.get_cache(key=tpm_key, local_only=True) - redis_cache: Final = self.dual_cache.redis_cache - if redis_cache is None or (local_tpm is not None and local_tpm >= tpm_limit): - return local_tpm - try: - return redis_cache.get_cache(key=tpm_key) - except RedisCircuitBreakerOpenError: - return local_tpm - - @with_service_target(ROUTER_USAGE_TARGET) - async def _async_get_current_tpm(self, tpm_key: str, tpm_limit: int, parent_otel_span: Span | None) -> int | None: - local_tpm: Final = await self.dual_cache.async_get_cache(key=tpm_key, local_only=True) - redis_cache: Final = self.dual_cache.redis_cache - if redis_cache is None or (local_tpm is not None and local_tpm >= tpm_limit): - return local_tpm - try: - return await redis_cache.async_get_cache(key=tpm_key, parent_otel_span=parent_otel_span) - except RedisCircuitBreakerOpenError: - return local_tpm + def _counters(self, deployment: dict, current_minute: str) -> tuple[UsageCounter, UsageCounter]: + """The counters TPM and RPM are tracked in.""" + model_id: Final[object] = deployment.get("model_info", {}).get("id") + deployment_name: Final[object] = deployment.get("litellm_params", {}).get("model") + return ( + UsageCounter.for_deployment(f"{model_id}", f"{deployment_name}", "tpm", current_minute), + UsageCounter.for_deployment(f"{model_id}", f"{deployment_name}", "rpm", current_minute), + ) @with_service_target(ROUTER_USAGE_TARGET) def pre_call_check(self, deployment: dict) -> dict | None: @@ -310,7 +269,7 @@ class ModelRateLimitingCheck(CustomLogger): if deployment_has_io_token_limits(deployment): self._warn_io_token_and_tpm_rpm_coexist_once(deployment) io_token_pre_call_check( - self.dual_cache, + self.usage, deployment, ) io_reservation_made = True @@ -320,7 +279,7 @@ class ModelRateLimitingCheck(CustomLogger): tpm_limit, rpm_limit = self._get_deployment_limits(deployment) current_minute: Final = dt.strftime("%H-%M") - tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) + tpm, rpm = self._counters(deployment, current_minute) model_id: Final = deployment.get("model_info", {}).get("id") model_name: Final = deployment.get("litellm_params", {}).get("model") @@ -328,7 +287,7 @@ class ModelRateLimitingCheck(CustomLogger): # Check TPM limit if tpm_limit is not None: - current_tpm: Final = self._get_current_tpm(tpm_key, tpm_limit) + current_tpm: Final = self.usage.current_sync(tpm, tpm_limit) if current_tpm is not None and current_tpm >= tpm_limit: raise litellm.RateLimitError( message=f"Model rate limit exceeded. TPM limit={tpm_limit}, current usage={current_tpm}", @@ -347,9 +306,9 @@ class ModelRateLimitingCheck(CustomLogger): # Check RPM limit (atomic increment-first to avoid race conditions) if rpm_limit is not None: - current_rpm: Final = self._increment(rpm_key, 1, RoutingArgs.ttl) + current_rpm: Final = self._increment(rpm, 1, RoutingArgs.ttl) if current_rpm is not None and current_rpm > rpm_limit: - self._increment(rpm_key, -1, RoutingArgs.ttl) + self._increment(rpm, -1, RoutingArgs.ttl) raise litellm.RateLimitError( message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", llm_provider="", @@ -369,7 +328,7 @@ class ModelRateLimitingCheck(CustomLogger): self._check_daily_limits(daily, dt) except litellm.RateLimitError: if rpm_limit is not None: - self._increment(rpm_key, -1, RoutingArgs.ttl) + self._increment(rpm, -1, RoutingArgs.ttl) raise return deployment @@ -393,11 +352,7 @@ class ModelRateLimitingCheck(CustomLogger): io_reservation_made = False if deployment_has_io_token_limits(deployment): self._warn_io_token_and_tpm_rpm_coexist_once(deployment) - await async_io_token_pre_call_check( - self.dual_cache, - deployment, - parent_otel_span=parent_otel_span, - ) + await async_io_token_pre_call_check(self.usage, deployment) io_reservation_made = True dt: Final = self.clock() @@ -406,8 +361,8 @@ class ModelRateLimitingCheck(CustomLogger): tpm_limit, rpm_limit = self._get_deployment_limits(deployment) current_minute: Final = dt.strftime("%H-%M") - tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute) - tpd_key, rpd_key = daily.cache_keys(dt) + tpm, rpm = self._counters(deployment, current_minute) + tpd, rpd = daily.counters(dt) model_id: Final = deployment.get("model_info", {}).get("id") model_name: Final = deployment.get("litellm_params", {}).get("model") @@ -415,7 +370,7 @@ class ModelRateLimitingCheck(CustomLogger): # Check TPM limit if tpm_limit is not None: - current_tpm: Final = await self._async_get_current_tpm(tpm_key, tpm_limit, parent_otel_span) + current_tpm: Final = await self.usage.current(tpm, tpm_limit) if current_tpm is not None and current_tpm >= tpm_limit: raise litellm.RateLimitError( message=f"Model rate limit exceeded. TPM limit={tpm_limit}, current usage={current_tpm}", @@ -434,17 +389,15 @@ class ModelRateLimitingCheck(CustomLogger): ) if tpd_limit is not None: - current_tpd: Final = await self._async_get_current_tpm(tpd_key, tpd_limit, parent_otel_span) + current_tpd: Final = await self.usage.current(tpd, tpd_limit) if current_tpd is not None and current_tpd >= tpd_limit: raise daily.limit_error("tpd", tpd_limit, current_tpd, dt) - charged_rpm_key: Final = rpm_key if rpm_limit is not None else None - charged_rpd_key: Final = rpd_key if rpd_limit is not None else None - current_rpm, current_rpd = await self._async_increment_request_counts( - charged_rpm_key, charged_rpd_key, parent_otel_span - ) + charged_rpm: Final = rpm if rpm_limit is not None else None + charged_rpd: Final = rpd if rpd_limit is not None else None + current_rpm, current_rpd = await self._async_increment_request_counts(charged_rpm, charged_rpd) if rpm_limit is not None and current_rpm is not None and current_rpm > rpm_limit: - await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span) + await self._async_refund_request_counts(charged_rpm, charged_rpd) raise litellm.RateLimitError( message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}", llm_provider="", @@ -462,14 +415,14 @@ class ModelRateLimitingCheck(CustomLogger): ) if rpd_limit is not None and current_rpd is not None and current_rpd > rpd_limit: - await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span) + await self._async_refund_request_counts(charged_rpm, charged_rpd) raise daily.limit_error("rpd", rpd_limit, current_rpd, dt) return deployment except litellm.RateLimitError: if io_reservation_made: - await self._async_refund_io_token_reservation_if_any(parent_otel_span=parent_otel_span) + await self._async_refund_io_token_reservation_if_any() raise except Exception as e: verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_pre_call_check: %s", e) @@ -478,28 +431,19 @@ class ModelRateLimitingCheck(CustomLogger): @with_service_target(ROUTER_USAGE_TARGET) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - from litellm.litellm_core_utils.core_helpers import ( - get_parent_otel_span_from_kwargs, - ) - try: standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") - await async_io_token_reconcile_success( - self.dual_cache, - kwargs, - response_obj, - parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), - ) + await async_io_token_reconcile_success(self.usage, kwargs, response_obj) if standard_logging_object is None: return - model_id: Final = standard_logging_object.get("model_id") + model_id: Final[object] = standard_logging_object.get("model_id") if model_id is None: return total_tokens: Final = standard_logging_object.get("total_tokens", 0) - model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") + model: Final[object] = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") verbose_router_logger.debug( "[TPM TRACKING] model_id=%s, total_tokens=%s, model=%s", model_id, total_tokens, model @@ -509,32 +453,25 @@ class ModelRateLimitingCheck(CustomLogger): return dt: Final = self.clock() - current_minute: Final = dt.strftime("%H-%M") - tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" + tpm: Final = UsageCounter.for_deployment(f"{model_id}", f"{model}", "tpm", dt.strftime("%H-%M")) reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict - tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" + tpd: Final = UsageCounter.for_deployment( + f"{model_id}", f"{model}", "tpd", daily_window_day(dt, reset_offset) + ) - verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens) + verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm, total_tokens) tokens: Final = int(total_tokens) - await self._async_increment(tpm_key, tokens, RoutingArgs.ttl) - await self._async_increment(tpd_key, tokens, RoutingArgs.daily_ttl) + await self.usage.increment(tpm, tokens, RoutingArgs.ttl) + await self.usage.increment(tpd, tokens, RoutingArgs.daily_ttl) except Exception as e: verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_log_success_event: %s", e) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - from litellm.litellm_core_utils.core_helpers import ( - get_parent_otel_span_from_kwargs, - ) - # Never fail the primary logging pipeline over an io-token refund error. with contextlib.suppress(Exception): - await async_io_token_refund_failure( - self.dual_cache, - kwargs, - parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), - ) + await async_io_token_refund_failure(self.usage, kwargs) @with_service_target(ROUTER_USAGE_TARGET) def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -544,30 +481,31 @@ class ModelRateLimitingCheck(CustomLogger): """ try: standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") - io_token_reconcile_success(self.dual_cache, kwargs, response_obj) + io_token_reconcile_success(self.usage, kwargs, response_obj) if standard_logging_object is None: return - model_id: Final = standard_logging_object.get("model_id") + model_id: Final[object] = standard_logging_object.get("model_id") if model_id is None: return total_tokens: Final = standard_logging_object.get("total_tokens", 0) - model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") + model: Final[object] = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") if not model or not total_tokens: return dt: Final = self.clock() - current_minute: Final = dt.strftime("%H-%M") - tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}" + tpm: Final = UsageCounter.for_deployment(f"{model_id}", f"{model}", "tpm", dt.strftime("%H-%M")) reset_offset: Final = _reset_offset_from_kwargs(kwargs) # pyright: ignore[reportUnknownArgumentType] # CustomLogger passes an untyped kwargs dict - tpd_key: Final = f"{model_id}:{model}:tpd:{daily_window_day(dt, reset_offset)}" + tpd: Final = UsageCounter.for_deployment( + f"{model_id}", f"{model}", "tpd", daily_window_day(dt, reset_offset) + ) tokens: Final = int(total_tokens) - self._increment(tpm_key, tokens, RoutingArgs.ttl) - self._increment(tpd_key, tokens, RoutingArgs.daily_ttl) + self._increment(tpm, tokens, RoutingArgs.ttl) + self._increment(tpd, tokens, RoutingArgs.daily_ttl) except Exception as e: verbose_router_logger.debug("Error in ModelRateLimitingCheck.log_success_event: %s", e) @@ -575,6 +513,6 @@ class ModelRateLimitingCheck(CustomLogger): def log_failure_event(self, kwargs, response_obj, start_time, end_time): with contextlib.suppress(Exception): io_token_refund_failure( - self.dual_cache, + self.usage, kwargs, ) diff --git a/litellm/router_utils/router_callbacks/track_deployment_metrics.py b/litellm/router_utils/router_callbacks/track_deployment_metrics.py index 2cf4e595296..0a4017c4dde 100644 --- a/litellm/router_utils/router_callbacks/track_deployment_metrics.py +++ b/litellm/router_utils/router_callbacks/track_deployment_metrics.py @@ -51,6 +51,10 @@ async def count_before_callbacks( raise +def deployment_successes_key(deployment_id: str) -> str: + return f"{deployment_id}:successes" + + @renamed_router_param("litellm_router_instance") def increment_deployment_successes_for_current_minute( ctx: "RoutingContext", @@ -59,7 +63,7 @@ def increment_deployment_successes_for_current_minute( """ In-Memory: Increments the number of successes for the current minute for a deployment_id """ - key: Final = f"{deployment_id}:successes" + key: Final = deployment_successes_key(deployment_id) ctx.health().store.increment_cache( local_only=True, key=key, @@ -96,7 +100,7 @@ def get_deployment_successes_for_current_minute( Returns 0 if no value found """ - key: Final = f"{deployment_id}:successes" + key: Final = deployment_successes_key(deployment_id) return ( ctx.health().store.get_cache( local_only=True, diff --git a/litellm/router_utils/routing_prefetch.py b/litellm/router_utils/routing_prefetch.py index 57d76220311..6d41eb867b5 100644 --- a/litellm/router_utils/routing_prefetch.py +++ b/litellm/router_utils/routing_prefetch.py @@ -1,7 +1,6 @@ from __future__ import annotations -import asyncio -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Awaitable, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final @@ -10,70 +9,26 @@ if TYPE_CHECKING: from litellm._internal_context import service_target from litellm._logging import verbose_router_logger -from litellm.caching.dual_cache import DualCache -from litellm.caching.redis_batch import Batch, BatchResult, active_request_redis_batches +from litellm.caching.redis_batch import Batch, active_request_redis_batches from litellm.redis.stores.router_cooldowns import ROUTER_COOLDOWNS_TARGET, ActiveCooldowns, RouterCooldowns -from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET +from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage, UsageCounter, UsageRead -_PREFETCH_SLOT: Final = "routing_read" +_USAGE_SLOT: Final = "routing_read" _COOLDOWNS_SLOT: Final = "routing_cooldowns" @dataclass(frozen=True, slots=True) -class TierRead: - cache: DualCache - keys: tuple[str, ...] +class ArmedUsage: + counters: tuple[UsageCounter, ...] + read: UsageRead -def selector_read(cache: DualCache, keys: Sequence[str]) -> TierRead: - return TierRead(cache, tuple(keys)) - - -@dataclass(frozen=True, slots=True) -class ArmedRead: - read: TierRead - fetched: frozenset[str] - result: BatchResult[Mapping[str, object]] - previous: Mapping[str, float | None] - - def release(self) -> None: - self.read.cache._rollback_redis_batch_key_reservations( # pyright: ignore[reportPrivateUsage] # the selector owns this tier - dict(self.previous) - ) - - async def settle(self, future: asyncio.Future[Mapping[str, object]]) -> None: - if future.cancelled() or future.exception() is not None: - self.release() - return - try: - pending: Final = await self.read.cache._prepare_batch_get( # pyright: ignore[reportPrivateUsage] # the selector owns this tier - list(self.read.keys), local_only=True - ) - values: Final = future.result() - missed: Final = { - key: values[key] - for key, local in zip(self.read.keys, pending.result) - if local is None and values.get(key) is not None - } - await self.read.cache._apply_batch_get(pending, missed) # pyright: ignore[reportPrivateUsage] # the selector owns this tier - except Exception: - self.release() - raise - - -def arm(read: TierRead) -> None: +def arm(store: RouterUsage, counters: Sequence[UsageCounter]) -> None: + """The usage read a usage-based-routing-v2 pick takes later, declared on the request's native pre-call batch.""" request: Final = active_request_redis_batches() - redis: Final = read.cache.redis_cache - if request is None or redis is None or _PREFETCH_SLOT in request.prefetched: + if request is None or not store.has_redis or _USAGE_SLOT in request.prefetched: return - due, previous = read.cache.reserve_redis_batch_reads(read.keys) - if not due: - return - with service_target(ROUTER_USAGE_TARGET): - result: Final = request.batch(redis).mget(tuple(due)) - armed_read: Final = ArmedRead(read=read, fetched=frozenset(due), result=result, previous=previous) - result.on_settled(armed_read.settle) - request.prefetched[_PREFETCH_SLOT] = armed_read + request.prefetched[_USAGE_SLOT] = ArmedUsage(tuple(counters), store.usage(counters, batch=Batch.PRE_CALL)) @dataclass(frozen=True, slots=True) @@ -92,7 +47,7 @@ def arm_cooldowns(store: RouterCooldowns, model_ids: Sequence[str]) -> None: def armed() -> bool: request: Final = active_request_redis_batches() - return request is not None and (_PREFETCH_SLOT in request.prefetched or _COOLDOWNS_SLOT in request.prefetched) + return request is not None and (_USAGE_SLOT in request.prefetched or _COOLDOWNS_SLOT in request.prefetched) async def _armed_cooldowns(model_ids: Sequence[str]) -> ActiveCooldowns | None: @@ -122,43 +77,29 @@ async def read_cooldowns( return await store.active(model_ids, batch=Batch.NONE) -def _take(read: TierRead) -> ArmedRead | None: +async def _armed_usage(counters: Sequence[UsageCounter]) -> Sequence[float | None] | None: request: Final = active_request_redis_batches() - if request is None: + taken: Final = None if request is None else request.prefetched.pop(_USAGE_SLOT, None) + if not isinstance(taken, ArmedUsage): return None - taken: Final = request.prefetched.pop(_PREFETCH_SLOT, None) - if isinstance(taken, ArmedRead) and set(taken.read.keys).issuperset(read.keys): - return taken - if isinstance(taken, ArmedRead): - taken.release() - return None - - -async def _read_prefetched(read: TierRead) -> Sequence[object | None] | None: - prefetch: Final = _take(read) - if prefetch is None: + if not frozenset(taken.counters).issuperset(counters): + taken.read.release() return None try: - values: Final = await prefetch.result - except Exception as e: # noqa: BLE001 # the read falls back to the tier's own Redis read + values: Final = await taken.read.result + except Exception as e: # noqa: BLE001 # the read falls back to the store's own Redis read verbose_router_logger.debug("routing prefetch failed, reading again: %s", e) return None - pending: Final = await read.cache._prepare_batch_get( # pyright: ignore[reportPrivateUsage] # the selector owns this tier - list(read.keys), local_only=True - ) - if any(key not in prefetch.fetched for key, value in zip(read.keys, pending.result) if value is None): - return None - return await read.cache._apply_batch_get( # pyright: ignore[reportPrivateUsage] # the selector owns this tier - pending, {key: values.get(key) for key, value in zip(read.keys, pending.result) if value is None} - ) + read: Final = dict(zip(taken.counters, values)) + return [read[counter] for counter in counters] -async def read(read: TierRead, parent_otel_span: Span | None) -> Sequence[object | None] | None: +async def read( + store: RouterUsage, counters: Sequence[UsageCounter], parent_otel_span: Span | None +) -> Sequence[float | None]: + """The armed usage read when it covers ``counters``, taken once; otherwise the store reads on its own.""" with service_target(ROUTER_USAGE_TARGET): - prefetched: Final = await _read_prefetched(read) - if prefetched is not None: - return prefetched - results: Final = await DualCache.async_batch_get_cache_shared( - ((read.cache, list(read.keys)),), parent_otel_span=parent_otel_span - ) - return results[0] + prefetched: Final = await _armed_usage(counters) + if prefetched is not None: + return prefetched + return await store.usage(counters, batch=Batch.NONE).result diff --git a/litellm/router_utils/routing_read_batch.py b/litellm/router_utils/routing_read_batch.py index 8f54a96e305..2d5aed65883 100644 --- a/litellm/router_utils/routing_read_batch.py +++ b/litellm/router_utils/routing_read_batch.py @@ -5,10 +5,10 @@ from contextvars import ContextVar from typing import TYPE_CHECKING, Final from litellm._logging import verbose_router_logger +from litellm.redis.stores.router_usage import UsageCounter from litellm.router_deprecated import renamed_router_param from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2, PrefetchedUsage from litellm.router_utils import routing_prefetch -from litellm.router_utils.routing_prefetch import TierRead, selector_read if TYPE_CHECKING: from opentelemetry.trace import Span @@ -18,13 +18,9 @@ if TYPE_CHECKING: __all__ = ("RoutingPrefetch", "RoutingReadBatch") -def _selector_keys(selector: LowestTPMLoggingHandler_v2, deployments: Sequence[object]) -> list[str]: - tpm_keys, rpm_keys = selector.usage_counter_keys(list(deployments)) - return tpm_keys + rpm_keys - - -def _selector_read(selector: LowestTPMLoggingHandler_v2 | None, deployments: Sequence[object]) -> TierRead | None: - return None if selector is None else selector_read(selector.router_cache, _selector_keys(selector, deployments)) +def _selector_counters(selector: LowestTPMLoggingHandler_v2, deployments: Sequence[object]) -> tuple[UsageCounter, ...]: + tpm_counters, rpm_counters = selector.usage_counters(deployments) + return tpm_counters + rpm_counters class RoutingPrefetch: @@ -34,9 +30,8 @@ class RoutingPrefetch: ctx: "RoutingContext", usage_selector: LowestTPMLoggingHandler_v2 | None, deployments: Sequence[object] ) -> None: RoutingPrefetch.arm_cooldowns(ctx) - read: Final = _selector_read(usage_selector, deployments) - if read is not None: - routing_prefetch.arm(read) + if usage_selector is not None: + routing_prefetch.arm(usage_selector.usage, _selector_counters(usage_selector, deployments)) @staticmethod def arm_cooldowns(ctx: "RoutingContext") -> None: @@ -92,11 +87,9 @@ class RoutingReadBatch: async def read_usage(self, deployments: Sequence[object], parent_otel_span: "Span | None") -> None: """The selector's tpm/rpm counters for ``deployments``, kept as ``prefetched_usage``.""" - usage_read: Final = _selector_read(self.usage_selector, deployments) - if usage_read is None: + selector: Final = self.usage_selector + if selector is None: return - values: Final = await routing_prefetch.read(usage_read, parent_otel_span) - self.prefetched_usage = PrefetchedUsage( - keys=frozenset(usage_read.keys), - values=None if values is None else dict(zip(usage_read.keys, values)), - ) + counters: Final = _selector_counters(selector, deployments) + values: Final = await routing_prefetch.read(selector.usage, counters, parent_otel_span) + self.prefetched_usage = PrefetchedUsage(counters=frozenset(counters), values=dict(zip(counters, values))) diff --git a/litellm/rust/Cargo.lock b/litellm/rust/Cargo.lock index 2cdc9ee91c8..102a43a2be4 100644 --- a/litellm/rust/Cargo.lock +++ b/litellm/rust/Cargo.lock @@ -3610,6 +3610,7 @@ version = "0.1.0" dependencies = [ "litellm-cache", "litellm-cache-testing", + "parking_lot", "rstest", "tokio", ] @@ -3691,6 +3692,7 @@ dependencies = [ "proptest", "pyo3", "rstest", + "serde", "serde_json", "strum", "thiserror 2.0.19", @@ -4009,6 +4011,7 @@ dependencies = [ name = "litellm-host" version = "0.1.0" dependencies = [ + "bytes", "futures-util", "litellm-coroutine", "litellm-secret-value", @@ -4345,6 +4348,7 @@ dependencies = [ name = "litellm-model-catalog" version = "0.1.0" dependencies = [ + "encoding_rs", "indexmap 2.14.0", "jiff", "jsonschema", @@ -4370,10 +4374,14 @@ dependencies = [ "aws-credential-types", "backon", "bytes", + "litellm-auth-gcp", + "litellm-auth-types", "litellm-error", "litellm-http", "litellm-s3", + "litellm-secret-value", "litellm-tracing", + "percent-encoding", "reqwest", "rstest", "thiserror 2.0.19", @@ -4546,11 +4554,14 @@ dependencies = [ "litellm-redis-testing", "litellm-secret-value", "litellm-tracing", + "parking_lot", "rstest", + "sha2 0.10.9", "strum", "thiserror 2.0.19", "tokio", "tracing", + "uuid", ] [[package]] diff --git a/litellm/rust/crates/cache-memory/Cargo.toml b/litellm/rust/crates/cache-memory/Cargo.toml index 88124f5401e..e8c58839292 100644 --- a/litellm/rust/crates/cache-memory/Cargo.toml +++ b/litellm/rust/crates/cache-memory/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true +parking_lot.workspace = true [dev-dependencies] litellm-cache-testing.workspace = true diff --git a/litellm/rust/crates/cache-memory/src/cache.rs b/litellm/rust/crates/cache-memory/src/cache.rs index 809ffe3044e..c386ef26322 100644 --- a/litellm/rust/crates/cache-memory/src/cache.rs +++ b/litellm/rust/crates/cache-memory/src/cache.rs @@ -1,18 +1,15 @@ use std::{ - cmp::Reverse, - collections::{BinaryHeap, HashMap, HashSet}, - hash::Hash, sync::{ - Arc, Mutex, MutexGuard, + Arc, atomic::{AtomicUsize, Ordering}, }, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use litellm_cache::{ - BaseCache, BatchCache, ClaimCache, CounterCache, DeleteCache, DisconnectCache, Error, - ExactCacheContext, FlushCache, SetCache, TtlCache, -}; +use litellm_cache::Error; +use parking_lot::Mutex; + +use crate::state::{Deadline, State}; const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200; const DEFAULT_TTL: Duration = Duration::from_secs(600); @@ -26,14 +23,16 @@ pub enum CacheWrite { TooLarge, } -struct CacheState { - values: HashMap, - expirations: HashMap, - expiration_heap: BinaryHeap>, +/// The TTL a write gives its key, `None` being the cache's default. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Expiry { + /// Only a key the cache does not hold takes the TTL, so a counter keeps its window. + KeepExisting(Option), + Refresh(Option), } pub struct InMemoryCache { - state: Mutex>, + state: Mutex>, max_size_in_memory: AtomicUsize, default_ttl: Duration, max_entry_bytes: Option, @@ -72,11 +71,7 @@ impl InMemoryCache { now: impl Fn() -> Duration + Send + Sync + 'static, ) -> Self { Self { - state: Mutex::new(CacheState { - values: HashMap::new(), - expirations: HashMap::new(), - expiration_heap: BinaryHeap::new(), - }), + state: Mutex::new(State::new()), max_size_in_memory: AtomicUsize::new( max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), ), @@ -89,39 +84,39 @@ impl InMemoryCache { } } - fn lock(&self) -> Result>, Error> { - Ok(self.state.lock()?) - } - pub fn set_cache( &self, key: impl Into, value: V, - ttl: Option, + expiry: Expiry, ) -> Result { - if self.max_size_in_memory() == 0 { - return Ok(CacheWrite::Disabled); - } let now = (self.now)(); - let mut state = self.lock()?; - self.store(&mut state, key.into(), value, ttl, now) + self.write(&mut self.state.lock(), key.into(), value, expiry, now) } pub fn get_cache(&self, key: &str) -> Result, Error> { let now = (self.now)(); - let mut state = self.lock()?; - if state - .expirations - .get(key) - .is_some_and(|expiration| *expiration < now) - { - Self::remove(&mut state, key); - } - Ok(state.values.get(key).cloned()) + Ok(self.state.lock().get(key, now).cloned()) } - /// `check_value_size`: whether `value` fits `max_entry_bytes`. Always `true` without a - /// limit and a measure, since typed values have no generic size. + /// Reads `key` and writes back under one lock. `update` answers the value to write, or `None` + /// to leave the entry as it is, next to what the caller gets back. + pub fn update( + &self, + key: &str, + expiry: Expiry, + update: impl FnOnce(Option<&V>) -> Result<(Option, T), Error>, + ) -> Result { + let now = (self.now)(); + let mut state = self.state.lock(); + let (value, output) = update(state.get(key, now))?; + if let Some(value) = value { + self.write(&mut state, key.to_owned(), value, expiry, now)?; + } + Ok(output) + } + + /// Always `true` without a limit and a measure, since typed values have no generic size. pub fn check_value_size(&self, value: &V) -> Result { match (self.max_entry_bytes, &self.measure_value) { (Some(limit), Some(measure)) => Ok(measure(value)? <= limit), @@ -129,57 +124,21 @@ impl InMemoryCache { } } - /// `evict_cache`: drops expired entries, then the earliest-expiring ones until a new key - /// fits. - pub fn evict_cache(&self) -> Result<(), Error> { - let now = (self.now)(); - let mut state = self.lock()?; - Self::evict(&mut state, self.max_size_in_memory(), now, None); - Ok(()) - } - - /// `evict_element_if_expired`: `true` when `key` had expired and was removed. - pub fn evict_element_if_expired(&self, key: &str) -> Result { - let now = (self.now)(); - let mut state = self.lock()?; - let expired = state - .expirations - .get(key) - .is_some_and(|expiration| *expiration < now); - if expired { - Self::remove(&mut state, key); - } - Ok(expired) - } - - /// `allow_ttl_override`: a write may set the TTL when the key has none or it has passed. - pub fn allow_ttl_override(&self, key: &str) -> Result { - let now = (self.now)(); - Ok(self - .expires_at(key)? - .is_none_or(|expiration| expiration < now)) - } - - /// The number of stored entries, expired ones included until they are evicted. pub fn len(&self) -> Result { - Ok(self.lock()?.values.len()) + let now = (self.now)(); + Ok(self.state.lock().len(now)) } pub fn is_empty(&self) -> Result { Ok(self.len()? == 0) } - /// Entries in the expiration heap, stale ones included; bounded by eviction. - pub fn expiration_heap_len(&self) -> Result { - Ok(self.lock()?.expiration_heap.len()) - } - pub fn max_size_in_memory(&self) -> usize { self.max_size_in_memory.load(Ordering::Relaxed) } - /// `DualCache.update_in_memory_max_size`: `None` restores the default. Entries already held stay - /// until the next write evicts down to the new size. + /// `None` restores the default. Entries already held stay until the next write trims to the + /// new size. pub fn update_max_size_in_memory(&self, max_size_in_memory: Option) { self.max_size_in_memory.store( max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), @@ -187,12 +146,13 @@ impl InMemoryCache { ); } - pub fn max_entry_bytes(&self) -> Option { - self.max_entry_bytes + pub fn default_ttl(&self) -> Duration { + self.default_ttl } pub fn expires_at(&self, key: &str) -> Result, Error> { - Ok(self.lock()?.expirations.get(key).copied()) + let now = (self.now)(); + Ok(self.state.lock().expires_at(key, now)) } pub async fn async_get_ttl(&self, key: &str) -> Result, Error> { @@ -200,253 +160,406 @@ impl InMemoryCache { } pub async fn async_get_oldest_n_keys(&self, count: usize) -> Result, Error> { - let state = self.lock()?; - let mut expirations = state - .expirations - .iter() - .map(|(key, expiration)| (key.clone(), *expiration)) - .collect::>(); - expirations.sort_unstable_by_key(|(_, expiration)| *expiration); - Ok(expirations - .into_iter() + let now = (self.now)(); + Ok(self + .state + .lock() + .keys_by_expiry(now) .take(count) - .map(|(key, _)| key) + .map(str::to_owned) .collect()) } - /// Every live entry, for callers that inspect the whole tier. pub fn entries(&self) -> Result, Error> { let now = (self.now)(); - let state = self.lock()?; - Ok(state - .values - .iter() - .filter(|(key, _)| { - state - .expirations - .get(*key) - .is_none_or(|expiration| *expiration >= now) - }) - .map(|(key, value)| (key.clone(), value.clone())) + Ok(self + .state + .lock() + .live(now) + .map(|(key, value)| (key.to_owned(), value.clone())) .collect()) } pub fn delete_cache(&self, key: &str) -> Result<(), Error> { - let mut state = self.lock()?; - Self::remove(&mut state, key); + self.state.lock().remove(key); Ok(()) } pub fn flush_cache(&self) -> Result<(), Error> { - let mut state = self.lock()?; - state.values.clear(); - state.expirations.clear(); - state.expiration_heap.clear(); + self.state.lock().clear(); Ok(()) } - /// Writing an existing `key` never evicts another entry, unlike Python, which pops the - /// earliest-expiring entry whenever the cache is full. - fn evict(state: &mut CacheState, capacity: usize, now: Duration, key: Option<&str>) { - while let Some(Reverse((expiration, key))) = state.expiration_heap.peek().cloned() { - if state.expirations.get(&key).copied() != Some(expiration) { - state.expiration_heap.pop(); - } else if expiration <= now { - state.expiration_heap.pop(); - Self::remove(state, &key); - } else { - break; - } - } - if key.is_some_and(|key| state.values.contains_key(key)) { - return; - } - while state.values.len() >= capacity { - let Some(Reverse((expiration, key))) = state.expiration_heap.pop() else { - break; - }; - if state.expirations.get(&key).copied() == Some(expiration) { - Self::remove(state, &key); - } - } - } - - fn set_expiration(state: &mut CacheState, key: &str, expiration: Duration) { - if state.expirations.get(key).copied() != Some(expiration) { - state.expirations.insert(key.into(), expiration); - state - .expiration_heap - .push(Reverse((expiration, key.into()))); - } - } - - fn remove(state: &mut CacheState, key: &str) { - state.values.remove(key); - state.expirations.remove(key); - } - - /// `get_cache` under the held lock: an expired entry is removed and reads as missing. - fn live(state: &mut CacheState, key: &str, now: Duration) -> Option { - if state - .expirations - .get(key) - .is_some_and(|expiration| *expiration < now) - { - Self::remove(state, key); - } - state.values.get(key).cloned() - } - - /// Python `set_cache` under the held lock: evict first (even when `key` already exists), - /// then skip oversized values, then write, keeping a live key's expiry. - fn store( + fn write( &self, - state: &mut CacheState, + state: &mut State, key: String, value: V, - ttl: Option, + expiry: Expiry, now: Duration, ) -> Result { - Self::evict(state, self.max_size_in_memory(), now, None); + let capacity = self.max_size_in_memory(); + if capacity == 0 { + return Ok(CacheWrite::Disabled); + } if !self.check_value_size(&value)? { return Ok(CacheWrite::TooLarge); } - let expiration = state.expirations.get(&key).copied(); - if expiration.is_none_or(|expiration| expiration < now) { - Self::set_expiration(state, &key, now + ttl.unwrap_or(self.default_ttl)); - } - state.values.insert(key, value); + let expires_at = + |ttl: Option| now.saturating_add(ttl.unwrap_or(self.default_ttl)); + let deadline = match expiry { + Expiry::KeepExisting(ttl) => Deadline::UnlessLive(expires_at(ttl)), + Expiry::Refresh(ttl) => Deadline::Always(expires_at(ttl)), + }; + state.write(key, value, deadline, capacity, now); Ok(CacheWrite::Stored) } } -impl ClaimCache for InMemoryCache -where - V: Clone + PartialEq + Send + Sync + 'static, -{ - fn claim_cache( - &self, - key: &str, - candidate: V, - eligible: &[V], - context: ExactCacheContext, - ) -> Result { - if self.max_size_in_memory() == 0 { - return Ok(candidate); - } - let now = (self.now)(); - let mut state = self.lock()?; - Self::evict(&mut state, self.max_size_in_memory(), now, Some(key)); - let existing = state - .values - .get(key) - .filter(|existing| eligible.is_empty() || eligible.contains(existing)) - .cloned(); - if let Some(existing) = &existing - && eligible.is_empty() - && *existing != candidate - { - return Ok(existing.clone()); - } - let winner = existing.unwrap_or(candidate); - Self::set_expiration( - &mut state, - key, - now + self.get_ttl(&context).unwrap_or(self.default_ttl), +#[cfg(test)] +mod tests { + use std::{ + panic::{AssertUnwindSafe, catch_unwind}, + sync::Arc, + time::Duration, + }; + + use litellm_cache::Error; + use rstest::rstest; + + use super::{CacheWrite, Expiry, InMemoryCache}; + use crate::test_support::{Clock, at, cache, clock, secs}; + + const DEFAULT: Expiry = Expiry::KeepExisting(None); + + fn keep(seconds: u64) -> Expiry { + Expiry::KeepExisting(secs(seconds)) + } + + fn measured(capacity: usize) -> InMemoryCache { + InMemoryCache::with_clock_and_size_measurement( + Some(capacity), + secs(60), + Some(4), + Some(Arc::new(|value: &String| { + if value.is_empty() { + return Err(Error::InvalidEntry(None)); + } + Ok(value.len()) + })), + || Duration::from_secs(100), + ) + } + + fn count(current: Option<&u64>) -> Result<(Option, u64), Error> { + let next = current.copied().unwrap_or_default() + 1; + Ok((Some(next), next)) + } + + #[rstest] + #[case::unset(None, secs(600))] + #[case::zero(Some(Duration::ZERO), secs(600))] + #[case::explicit(secs(5), secs(5))] + fn a_write_without_a_ttl_takes_the_default( + #[case] default_ttl: Option, + #[case] expected: Option, + ) { + let cache = InMemoryCache::::with_clock(None, default_ttl, || Duration::ZERO); + + cache.set_cache("key", "value".into(), DEFAULT).unwrap(); + + assert_eq!(cache.expires_at("key"), Ok(expected)); + assert_eq!(cache.max_size_in_memory(), 200); + } + + #[rstest] + #[case::before(109, Some("value"))] + #[case::at_the_deadline(110, None)] + #[case::after(111, None)] + fn an_entry_is_readable_until_its_deadline( + clock: Clock, + #[case] now: u64, + #[case] expected: Option<&str>, + ) { + let cache = cache::(&clock, 4); + cache.set_cache("key", "value".into(), keep(10)).unwrap(); + + at(&clock, now); + + assert_eq!(cache.get_cache("key"), Ok(expected.map(str::to_owned))); + } + + #[rstest] + #[case::keep_existing(keep(10), secs(160))] + #[case::refresh(Expiry::Refresh(secs(10)), secs(115))] + fn a_write_to_a_live_key_keeps_its_deadline_unless_it_refreshes( + clock: Clock, + #[case] expiry: Expiry, + #[case] expected: Option, + ) { + let cache = cache::(&clock, 4); + cache.set_cache("key", "first".into(), DEFAULT).unwrap(); + at(&clock, 105); + + cache.set_cache("key", "second".into(), expiry).unwrap(); + + assert_eq!(cache.get_cache("key"), Ok(Some("second".into()))); + assert_eq!(cache.expires_at("key"), Ok(expected)); + } + + #[rstest] + fn an_expired_key_takes_the_ttl_of_the_next_write(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("key", "first".into(), keep(10)).unwrap(); + at(&clock, 110); + + cache.set_cache("key", "second".into(), keep(10)).unwrap(); + + assert_eq!(cache.expires_at("key"), Ok(secs(120))); + } + + #[rstest] + #[case::keep_existing(Expiry::KeepExisting(Some(Duration::MAX)))] + #[case::refresh(Expiry::Refresh(Some(Duration::MAX)))] + fn a_ttl_past_the_end_of_time_never_expires(clock: Clock, #[case] forever: Expiry) { + let cache = cache::(&clock, 3); + + cache.set_cache("forever", "a".into(), forever).unwrap(); + at(&clock, 1_000_000_000); + cache.set_cache("after", "b".into(), keep(1)).unwrap(); + + assert_eq!(cache.get_cache("forever"), Ok(Some("a".into()))); + assert_eq!(cache.get_cache("after"), Ok(Some("b".into()))); + } + + #[rstest] + fn a_panic_inside_an_update_leaves_the_cache_usable(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("held", "a".into(), DEFAULT).unwrap(); + + let panicked = catch_unwind(AssertUnwindSafe(|| { + cache.update::<()>("held", DEFAULT, |_| panic!("the caller's update panics")) + })); + + assert!(panicked.is_err()); + assert_eq!( + cache.set_cache("next", "b".into(), DEFAULT), + Ok(CacheWrite::Stored) ); - state.values.insert(key.into(), winner.clone()); - Ok(winner) - } -} - -impl CounterCache for InMemoryCache { - fn increment_cache( - &self, - key: &str, - amount: f64, - context: ExactCacheContext, - ) -> Result { - if self.max_size_in_memory() == 0 { - return Ok(amount); - } - let now = (self.now)(); - let mut state = self.lock()?; - let value = Self::live(&mut state, key, now).unwrap_or_default() + amount; - self.store(&mut state, key.into(), value, self.get_ttl(&context), now)?; - Ok(value) - } -} - -impl BaseCache for InMemoryCache { - type Value = V; - type Context = ExactCacheContext; - - fn get_ttl(&self, context: &Self::Context) -> Option { - context.ttl.or(Some(self.default_ttl)) - } - - fn set_cache( - &self, - key: &str, - value: Self::Value, - context: &ExactCacheContext, - ) -> Result<(), Error> { - let ttl = self.get_ttl(context).unwrap_or(self.default_ttl); - self.set_cache(key, value, Some(ttl)).map(|_| ()) - } - - fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { - self.get_cache(key) - } -} - -impl DisconnectCache for InMemoryCache { - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } -} - -impl BatchCache for InMemoryCache {} - -impl DeleteCache for InMemoryCache { - fn delete_cache(&self, key: &str) -> Result<(), Error> { - InMemoryCache::delete_cache(self, key) - } -} - -impl FlushCache for InMemoryCache { - fn flush_cache(&self) -> Result<(), Error> { - InMemoryCache::flush_cache(self) - } -} - -impl TtlCache for InMemoryCache { - async fn async_get_ttl(&self, key: &str) -> Result, Error> { - InMemoryCache::async_get_ttl(self, key).await - } -} - -impl SetCache for InMemoryCache> -where - T: Clone + Eq + Hash + Send + Sync + 'static, -{ - type SetValue = T; - type SetResult = Vec; - - async fn async_set_cache_sadd( - &self, - key: &str, - values: Vec, - ttl: Option, - ) -> Result { - if self.max_size_in_memory() == 0 { - return Ok(values); - } - let now = (self.now)(); - let mut state = self.lock()?; - let mut stored = Self::live(&mut state, key, now).unwrap_or_default(); - stored.extend(values.iter().cloned()); - self.store(&mut state, key.into(), stored, ttl, now)?; - Ok(values) + assert_eq!(cache.get_cache("held"), Ok(Some("a".into()))); + } + + #[rstest] + fn a_full_cache_admits_a_new_key_by_evicting_the_soonest_to_expire(clock: Clock) { + let cache = cache::(&clock, 2); + cache.set_cache("early", "a".into(), keep(10)).unwrap(); + cache.set_cache("late", "b".into(), keep(20)).unwrap(); + + cache.set_cache("new", "c".into(), keep(5)).unwrap(); + + assert_eq!(cache.get_cache("early"), Ok(None)); + assert_eq!(cache.get_cache("late"), Ok(Some("b".into()))); + assert_eq!(cache.get_cache("new"), Ok(Some("c".into()))); + } + + #[rstest] + #[tokio::test] + async fn a_key_deleted_and_set_again_is_tracked_once(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("older", "pin".into(), keep(3600)).unwrap(); + at(&clock, 101); + cache.set_cache("pinned", "pin".into(), keep(3600)).unwrap(); + at(&clock, 102); + cache.delete_cache("pinned").unwrap(); + cache.set_cache("pinned", "pin".into(), keep(3600)).unwrap(); + at(&clock, 103); + cache + .set_cache("pinned", "pin".into(), Expiry::Refresh(secs(3600))) + .unwrap(); + + assert_eq!( + cache.async_get_oldest_n_keys(10).await.unwrap(), + ["older", "pinned"] + ); + assert_eq!(cache.expires_at("pinned"), Ok(secs(3703))); + } + + #[rstest] + fn a_smaller_capacity_applies_on_the_next_write(clock: Clock) { + let cache = cache::(&clock, 3); + cache.set_cache("a", "1".into(), keep(10)).unwrap(); + cache.set_cache("b", "2".into(), keep(20)).unwrap(); + cache.set_cache("c", "3".into(), keep(30)).unwrap(); + + cache.update_max_size_in_memory(Some(1)); + let held = cache.len(); + cache.set_cache("d", "4".into(), keep(40)).unwrap(); + + assert_eq!(held, Ok(3)); + assert_eq!(cache.entries(), Ok(vec![("d".to_owned(), "4".to_owned())])); + } + + #[rstest] + fn an_unset_capacity_restores_the_default(clock: Clock) { + let cache = cache::(&clock, 3); + + cache.update_max_size_in_memory(None); + + assert_eq!(cache.max_size_in_memory(), 200); + } + + #[rstest] + fn a_cache_without_capacity_stores_nothing(clock: Clock) { + let cache = cache::(&clock, 0); + + let written = cache.set_cache("key", 1, DEFAULT); + let counted = cache.update("key", DEFAULT, count); + + assert_eq!(written, Ok(CacheWrite::Disabled)); + assert_eq!(counted, Ok(1)); + assert_eq!(cache.is_empty(), Ok(true)); + } + + #[rstest] + #[case::fits("ok", Ok(true))] + #[case::at_limit("four", Ok(true))] + #[case::too_large("oversized", Ok(false))] + #[case::measure_error("", Err(Error::InvalidEntry(None)))] + fn check_value_size_applies_the_entry_limit( + #[case] value: &str, + #[case] expected: Result, + ) { + assert_eq!(measured(2).check_value_size(&value.to_owned()), expected); + } + + #[rstest] + fn values_are_unbounded_without_a_measure() { + let cache = InMemoryCache::::default(); + + assert_eq!(cache.check_value_size(&"x".repeat(1 << 20)), Ok(true)); + } + + #[rstest] + #[case::too_large("oversized", Ok(CacheWrite::TooLarge))] + #[case::rejected_by_the_measure("", Err(Error::InvalidEntry(None)))] + fn a_value_that_does_not_fit_is_not_stored_and_keeps_the_held_one( + #[case] value: &str, + #[case] expected: Result, + ) { + let cache = measured(2); + cache.set_cache("key", "ok".into(), DEFAULT).unwrap(); + + let written = cache.set_cache("key", value.to_owned(), DEFAULT); + + assert_eq!(written, expected); + assert_eq!(cache.get_cache("key"), Ok(Some("ok".into()))); + } + + #[rstest] + fn an_update_writes_what_it_answers_and_keeps_the_window(clock: Clock) { + let cache = cache::(&clock, 4); + + let first = cache.update("count", keep(10), count); + at(&clock, 105); + let second = cache.update("count", keep(10), count); + + assert_eq!((first, second), (Ok(1), Ok(2))); + assert_eq!(cache.get_cache("count"), Ok(Some(2))); + assert_eq!(cache.expires_at("count"), Ok(secs(110))); + } + + #[rstest] + fn an_update_that_answers_no_value_leaves_the_entry_as_it_is(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("count", 5, keep(10)).unwrap(); + at(&clock, 105); + + let read = cache.update("count", Expiry::Refresh(secs(50)), |current| { + Ok((None, current.copied())) + }); + + assert_eq!(read, Ok(Some(5))); + assert_eq!(cache.get_cache("count"), Ok(Some(5))); + assert_eq!(cache.expires_at("count"), Ok(secs(110))); + } + + #[rstest] + fn an_update_that_fails_writes_nothing(clock: Clock) { + let cache = cache::(&clock, 4); + + let failed = cache.update::<()>("count", DEFAULT, |_| Err(Error::InvalidEntry(None))); + + assert_eq!(failed, Err(Error::InvalidEntry(None))); + assert_eq!(cache.get_cache("count"), Ok(None)); + } + + #[rstest] + fn an_update_fails_when_its_value_cannot_be_measured() { + let updated = measured(2).update("key", DEFAULT, |_| Ok((Some(String::new()), ()))); + + assert_eq!(updated, Err(Error::InvalidEntry(None))); + } + + #[rstest] + fn only_live_entries_are_counted_and_listed(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("short", "a".into(), keep(1)).unwrap(); + cache.set_cache("long", "b".into(), keep(50)).unwrap(); + + at(&clock, 101); + + assert_eq!(cache.len(), Ok(1)); + assert_eq!(cache.is_empty(), Ok(false)); + assert_eq!( + cache.entries(), + Ok(vec![("long".to_owned(), "b".to_owned())]) + ); + assert_eq!(cache.expires_at("short"), Ok(None)); + } + + #[rstest] + #[case::none(0, &[])] + #[case::one(1, &["first"])] + #[case::more_than_held(10, &["first", "later", "latest"])] + #[tokio::test] + async fn the_oldest_keys_are_the_soonest_to_expire( + clock: Clock, + #[case] count: usize, + #[case] expected: &[&str], + ) { + let cache = cache::(&clock, 3); + cache.set_cache("later", "2".into(), keep(20)).unwrap(); + cache.set_cache("first", "1".into(), keep(10)).unwrap(); + cache.set_cache("latest", "3".into(), keep(30)).unwrap(); + + assert_eq!( + cache.async_get_oldest_n_keys(count).await.unwrap(), + expected + ); + } + + #[rstest] + #[tokio::test] + async fn the_ttl_of_a_key_is_its_deadline(clock: Clock) { + let cache = cache::(&clock, 3); + cache.set_cache("key", "1".into(), keep(10)).unwrap(); + + assert_eq!(cache.async_get_ttl("key").await, Ok(secs(110))); + assert_eq!(cache.async_get_ttl("missing").await, Ok(None)); + } + + #[rstest] + fn delete_drops_one_key_and_flush_drops_the_rest(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("a", "1".into(), DEFAULT).unwrap(); + cache.set_cache("b", "2".into(), DEFAULT).unwrap(); + cache.set_cache("c", "3".into(), DEFAULT).unwrap(); + + cache.delete_cache("a").unwrap(); + let after_delete = (cache.get_cache("a"), cache.len()); + cache.flush_cache().unwrap(); + + assert_eq!(after_delete, (Ok(None), Ok(2))); + assert_eq!(cache.is_empty(), Ok(true)); } } diff --git a/litellm/rust/crates/cache-memory/src/capabilities.rs b/litellm/rust/crates/cache-memory/src/capabilities.rs new file mode 100644 index 00000000000..6179732ce19 --- /dev/null +++ b/litellm/rust/crates/cache-memory/src/capabilities.rs @@ -0,0 +1,528 @@ +use std::{collections::HashSet, hash::Hash, time::Duration}; + +use litellm_cache::{ + BaseCache, BatchCache, ClaimCache, CounterCache, DeleteCache, DisconnectCache, Error, + ExactCacheContext, FlushCache, SetCache, TtlCache, +}; + +use crate::cache::{Expiry, InMemoryCache}; + +impl BaseCache for InMemoryCache { + type Value = V; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl.or(Some(self.default_ttl())) + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &ExactCacheContext, + ) -> Result<(), Error> { + InMemoryCache::set_cache(self, key, value, Expiry::KeepExisting(context.ttl)).map(drop) + } + + fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { + InMemoryCache::get_cache(self, key) + } +} + +impl ClaimCache for InMemoryCache +where + V: Clone + PartialEq + Send + Sync + 'static, +{ + /// A held value outside `eligible` gives way to `candidate`. With no `eligible` list the held + /// value always wins, and only a claim that agrees with it refreshes the TTL. + fn claim_cache( + &self, + key: &str, + candidate: V, + eligible: &[V], + context: ExactCacheContext, + ) -> Result { + self.update(key, Expiry::Refresh(context.ttl), |held| { + let held = held.filter(|held| eligible.is_empty() || eligible.contains(held)); + Ok(match held { + Some(held) if eligible.is_empty() && *held != candidate => (None, held.clone()), + Some(held) => (Some(held.clone()), held.clone()), + None => (Some(candidate.clone()), candidate), + }) + }) + } +} + +impl CounterCache for InMemoryCache { + fn increment_cache( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + self.update(key, Expiry::KeepExisting(context.ttl), |held| { + let total = held.copied().unwrap_or_default() + amount; + Ok((Some(total), total)) + }) + } +} + +impl DisconnectCache for InMemoryCache { + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } +} + +impl BatchCache for InMemoryCache {} + +impl DeleteCache for InMemoryCache { + fn delete_cache(&self, key: &str) -> Result<(), Error> { + InMemoryCache::delete_cache(self, key) + } +} + +impl FlushCache for InMemoryCache { + fn flush_cache(&self) -> Result<(), Error> { + InMemoryCache::flush_cache(self) + } +} + +impl TtlCache for InMemoryCache { + async fn async_get_ttl(&self, key: &str) -> Result, Error> { + InMemoryCache::async_get_ttl(self, key).await + } +} + +impl SetCache for InMemoryCache> +where + T: Clone + Eq + Hash + Send + Sync + 'static, +{ + type SetValue = T; + type SetResult = Vec; + + async fn async_set_cache_sadd( + &self, + key: &str, + values: Vec, + ttl: Option, + ) -> Result { + self.update(key, Expiry::KeepExisting(ttl), |held| { + let members = held.into_iter().flatten().chain(&values).cloned().collect(); + Ok((Some(members), ())) + })?; + Ok(values) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashSet, sync::Arc, time::Duration}; + + use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheBackend, ClaimCache, CounterCache, DeleteCache, + DisconnectCache, ExactCacheContext, FlushCache, IncrementOperation, SetCache, TtlCache, + get_cache, set_cache, + }; + use rstest::rstest; + + use crate::{ + Expiry, InMemoryCache, + test_support::{Clock, at, cache, clock, secs}, + }; + + const DEFAULT: Expiry = Expiry::KeepExisting(None); + + fn keep(seconds: u64) -> Expiry { + Expiry::KeepExisting(secs(seconds)) + } + + fn ttl(seconds: u64) -> ExactCacheContext { + ExactCacheContext { ttl: secs(seconds) } + } + + #[rstest] + #[tokio::test] + async fn generic_consumers_share_typed_values_and_honor_expiration(clock: Clock) { + let cache: CacheBackend> = Arc::new(cache(&clock, 4)); + let reader = Arc::clone(&cache); + let context = ttl(5); + set_cache(cache.as_ref(), "sync", "first".into(), &context).unwrap(); + cache + .batch_cache_write("async", "second".into(), context.clone()) + .await + .unwrap(); + cache + .async_set_cache_pipeline(vec![("batch".into(), "third".into())], context.clone()) + .await + .unwrap(); + drop(cache); + + let written = ( + get_cache(reader.as_ref(), "sync", &context), + reader.async_get_cache("async", &context).await, + reader.async_get_cache("batch", &context).await, + ); + reader.async_delete_cache("async").await.unwrap(); + let deleted = reader.async_get_cache("async", &context).await; + at(&clock, 105); + let expired = ( + get_cache(reader.as_ref(), "sync", &context), + reader.async_get_cache("batch", &context).await, + ); + + assert_eq!( + written, + ( + Ok(Some("first".into())), + Ok(Some("second".into())), + Ok(Some("third".into())) + ) + ); + assert_eq!(deleted, Ok(None)); + assert_eq!(expired, (Ok(None), Ok(None))); + } + + #[rstest] + #[case::context_ttl(ttl(5), secs(105))] + #[case::default_ttl(ExactCacheContext::default(), secs(160))] + #[tokio::test] + async fn pipeline_writes_use_the_context_ttl_or_the_default( + clock: Clock, + #[case] context: ExactCacheContext, + #[case] expected: Option, + ) { + let cache = cache::(&clock, 4); + + cache + .async_set_cache_pipeline( + vec![("a".into(), "1".into()), ("b".into(), "2".into())], + context, + ) + .await + .unwrap(); + + assert_eq!(cache.expires_at("a"), Ok(expected)); + assert_eq!(cache.expires_at("b"), Ok(expected)); + } + + #[rstest] + #[tokio::test] + async fn batch_reads_return_one_entry_per_key_and_miss_expired_ones(clock: Clock) { + let cache = cache::(&clock, 4); + cache.set_cache("short", "a".into(), keep(1)).unwrap(); + cache.set_cache("long", "b".into(), keep(50)).unwrap(); + let keys = vec!["short".to_owned(), "missing".into(), "long".into()]; + + let live = cache.batch_get_cache(&keys, &ExactCacheContext::default()); + at(&clock, 101); + let later = cache + .async_batch_get_cache(keys, ExactCacheContext::default()) + .await; + + assert_eq!( + live, + Ok(vec![ + BatchEntry::Hit("a".to_owned()), + BatchEntry::Miss, + BatchEntry::Hit("b".into()), + ]) + ); + assert_eq!( + later, + Ok(vec![ + BatchEntry::Miss, + BatchEntry::Miss, + BatchEntry::Hit("b".into()), + ]) + ); + } + + #[rstest] + #[tokio::test] + async fn disconnect_keeps_entries_and_an_async_flush_drops_them() { + let cache = InMemoryCache::::default(); + cache.set_cache("key", "value".into(), DEFAULT).unwrap(); + + cache.disconnect().await.unwrap(); + let after_disconnect = cache.get_cache("key"); + FlushCache::async_flush_cache(&cache).await.unwrap(); + + assert_eq!(after_disconnect, Ok(Some("value".into()))); + assert_eq!(cache.is_empty(), Ok(true)); + } + + #[rstest] + #[tokio::test] + async fn the_ttl_capability_reads_the_deadline(clock: Clock) { + let cache = cache::(&clock, 3); + cache.set_cache("key", "1".into(), keep(20)).unwrap(); + + assert_eq!(TtlCache::async_get_ttl(&cache, "key").await, Ok(secs(120))); + } + + #[rstest] + fn a_claim_without_an_eligible_list_keeps_the_held_value_and_its_ttl(clock: Clock) { + let cache = cache::(&clock, 4); + let first = cache.claim_cache("affinity", "first".to_owned(), &[], ttl(10)); + at(&clock, 103); + + let second = cache.claim_cache("affinity", "second".to_owned(), &[], ttl(10)); + + assert_eq!((first, second), (Ok("first".into()), Ok("first".into()))); + assert_eq!(cache.expires_at("affinity"), Ok(secs(110))); + } + + #[rstest] + #[case::the_held_value_is_eligible(&["first", "second"], "first")] + #[case::the_held_value_is_not_eligible(&["second"], "second")] + fn a_claim_with_an_eligible_list_refreshes_the_ttl_of_the_winner( + clock: Clock, + #[case] eligible: &[&str], + #[case] winner: &str, + ) { + let cache = cache::(&clock, 4); + let eligible = eligible + .iter() + .copied() + .map(String::from) + .collect::>(); + cache + .set_cache("affinity", "first".into(), keep(10)) + .unwrap(); + at(&clock, 105); + + let claimed = cache.claim_cache("affinity", "second".to_owned(), &eligible, ttl(10)); + + assert_eq!(claimed, Ok(winner.to_owned())); + assert_eq!(cache.get_cache("affinity"), Ok(Some(winner.to_owned()))); + assert_eq!(cache.expires_at("affinity"), Ok(secs(115))); + } + + #[rstest] + fn claiming_a_held_key_at_capacity_keeps_other_entries(clock: Clock) { + let cache = cache::(&clock, 2); + cache.set_cache("hot", "1".into(), keep(10)).unwrap(); + cache.set_cache("cold", "2".into(), keep(20)).unwrap(); + + cache + .claim_cache("cold", "2".into(), &[], ExactCacheContext::default()) + .unwrap(); + + assert_eq!(cache.get_cache("hot"), Ok(Some("1".into()))); + assert_eq!(cache.expires_at("cold"), Ok(secs(160))); + } + + #[rstest] + fn counters_accumulate() { + let cache = InMemoryCache::::default(); + + let totals = ( + cache.increment_cache("counter", 1.5, ExactCacheContext::default()), + cache.increment_cache("counter", 2.0, ExactCacheContext::default()), + ); + + assert_eq!(totals, (Ok(1.5), Ok(3.5))); + } + + #[rstest] + fn concurrent_increments_lose_no_update() { + let cache = InMemoryCache::::default(); + + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + for _ in 0..10_000 { + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(); + } + }); + } + }); + + assert_eq!(cache.get_cache("counter"), Ok(Some(80_000.0))); + } + + #[rstest] + fn a_trait_write_to_a_live_key_keeps_its_deadline(clock: Clock) { + let cache = cache::(&clock, 4); + set_cache(&cache, "key", "first".into(), &ttl(10)).unwrap(); + at(&clock, 105); + + set_cache(&cache, "key", "second".into(), &ttl(10)).unwrap(); + + assert_eq!(cache.get_cache("key"), Ok(Some("second".into()))); + assert_eq!(cache.expires_at("key"), Ok(secs(110))); + } + + #[rstest] + #[case::its_own(ttl(5), secs(5))] + #[case::the_default_of_the_cache(ExactCacheContext::default(), secs(60))] + fn a_context_resolves_to_its_ttl_or( + clock: Clock, + #[case] context: ExactCacheContext, + #[case] expected: Option, + ) { + assert_eq!(cache::(&clock, 4).get_ttl(&context), expected); + } + + #[rstest] + #[case::window_semantics(false)] + #[case::refresh_ttl_is_ignored(true)] + #[tokio::test] + async fn an_increment_keeps_the_window_of_a_live_counter( + clock: Clock, + #[case] refresh_ttl: bool, + ) { + let cache = cache::(&clock, 4); + + let first = cache + .async_increment("counter", 2.0, ttl(10), refresh_ttl) + .await; + at(&clock, 105); + let second = cache + .async_increment("counter", 3.0, ttl(10), refresh_ttl) + .await; + + assert_eq!((first, second), (Ok(2.0), Ok(5.0))); + assert_eq!(cache.expires_at("counter"), Ok(secs(110))); + } + + #[rstest] + fn an_expired_counter_restarts_from_zero_with_a_new_window(clock: Clock) { + let cache = cache::(&clock, 4); + cache.increment_cache("counter", 2.0, ttl(10)).unwrap(); + at(&clock, 110); + + let restarted = cache.increment_cache("counter", 1.0, ttl(10)); + + assert_eq!(restarted, Ok(1.0)); + assert_eq!(cache.expires_at("counter"), Ok(secs(120))); + } + + #[rstest] + fn incrementing_a_held_counter_at_capacity_evicts_nothing(clock: Clock) { + let cache = cache::(&clock, 2); + let increment = |key| cache.increment_cache(key, 1.0, ExactCacheContext::default()); + increment("a").unwrap(); + increment("b").unwrap(); + + let totals = (increment("a"), increment("b")); + + assert_eq!(totals, (Ok(2.0), Ok(2.0))); + } + + #[rstest] + #[tokio::test] + async fn a_cache_without_capacity_answers_claims_counters_and_sets_but_keeps_none() { + let claims = InMemoryCache::::new(Some(0), None); + let counters = InMemoryCache::::new(Some(0), None); + let sets = InMemoryCache::>::new(Some(0), None); + + let claimed = claims.claim_cache("key", "first".into(), &[], ExactCacheContext::default()); + let counted = counters.increment_cache("key", 2.0, ExactCacheContext::default()); + let added = sets + .async_set_cache_sadd("key", vec!["a".into()], None) + .await; + + assert_eq!(claimed, Ok("first".into())); + assert_eq!(counted, Ok(2.0)); + assert_eq!(added, Ok(vec!["a".to_owned()])); + assert_eq!( + (claims.is_empty(), counters.is_empty(), sets.is_empty()), + (Ok(true), Ok(true), Ok(true)) + ); + } + + #[rstest] + #[tokio::test] + async fn an_increment_pipeline_answers_in_operation_order(clock: Clock) { + let cache = cache::(&clock, 3); + let operation = |key: &str, amount, ttl| IncrementOperation { + key: key.into(), + amount, + ttl: secs(ttl), + }; + + let totals = cache + .async_increment_pipeline(vec![ + operation("a", 1.0, 10), + operation("b", 5.0, 30), + operation("a", 2.0, 20), + ]) + .await; + + assert_eq!(totals, Ok(vec![1.0, 5.0, 3.0])); + assert_eq!(cache.expires_at("a"), Ok(secs(110))); + assert_eq!(cache.expires_at("b"), Ok(secs(130))); + } + + #[rstest] + #[tokio::test] + async fn set_additions_answer_what_was_added_and_store_the_union(clock: Clock) { + let cache = cache::>(&clock, 4); + let added = vec!["a".to_owned(), "a".into(), "b".into()]; + + let first = cache + .async_set_cache_sadd("members", added.clone(), secs(10)) + .await; + let second = cache + .async_set_cache_sadd("members", vec!["c".into()], secs(99)) + .await; + + assert_eq!((first, second), (Ok(added), Ok(vec!["c".to_owned()]))); + assert_eq!( + cache.get_cache("members"), + Ok(Some(HashSet::from(["a".into(), "b".into(), "c".into()]))) + ); + assert_eq!(cache.expires_at("members"), Ok(secs(110))); + } + + #[rstest] + #[tokio::test] + async fn an_expired_set_restarts_empty_with_a_new_ttl(clock: Clock) { + let cache = cache::>(&clock, 4); + cache + .async_set_cache_sadd("members", vec!["a".into()], secs(10)) + .await + .unwrap(); + at(&clock, 110); + + cache + .async_set_cache_sadd("members", vec!["d".into()], None) + .await + .unwrap(); + + assert_eq!( + cache.get_cache("members"), + Ok(Some(HashSet::from(["d".into()]))) + ); + assert_eq!(cache.expires_at("members"), Ok(secs(170))); + } + + #[rstest] + #[tokio::test] + async fn a_set_addition_that_outgrows_the_entry_limit_is_not_stored() { + let cache = InMemoryCache::>::with_clock_and_size_measurement( + Some(4), + None, + Some(2), + Some(Arc::new(|value: &HashSet| Ok(value.len()))), + || Duration::ZERO, + ); + cache + .async_set_cache_sadd("members", vec!["a".into(), "b".into()], None) + .await + .unwrap(); + + let added = cache + .async_set_cache_sadd("members", vec!["c".into()], None) + .await; + + assert_eq!(added, Ok(vec!["c".to_owned()])); + assert_eq!( + cache.get_cache("members"), + Ok(Some(HashSet::from(["a".into(), "b".into()]))) + ); + } +} diff --git a/litellm/rust/crates/cache-memory/src/lib.rs b/litellm/rust/crates/cache-memory/src/lib.rs index c5b7fb6cb54..215e9d89edf 100644 --- a/litellm/rust/crates/cache-memory/src/lib.rs +++ b/litellm/rust/crates/cache-memory/src/lib.rs @@ -1,3 +1,7 @@ mod cache; +mod capabilities; +mod state; +#[cfg(test)] +mod test_support; -pub use cache::{CacheWrite, InMemoryCache}; +pub use cache::{CacheWrite, Expiry, InMemoryCache}; diff --git a/litellm/rust/crates/cache-memory/src/state.rs b/litellm/rust/crates/cache-memory/src/state.rs new file mode 100644 index 00000000000..06e42bc0308 --- /dev/null +++ b/litellm/rust/crates/cache-memory/src/state.rs @@ -0,0 +1,320 @@ +use std::{ + collections::{BTreeSet, HashMap}, + time::Duration, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Deadline { + UnlessLive(Duration), + Always(Duration), +} + +struct Entry { + value: V, + expires_at: Duration, +} + +/// Entries with an exact index of their deadlines: `by_expiry` holds one record per entry, so it +/// never outgrows `entries`. Every read and write first drops what has expired by `now`. +pub(crate) struct State { + entries: HashMap>, + by_expiry: BTreeSet<(Duration, String)>, +} + +impl State { + pub(crate) fn new() -> Self { + Self { + entries: HashMap::new(), + by_expiry: BTreeSet::new(), + } + } + + pub(crate) fn get(&mut self, key: &str, now: Duration) -> Option<&V> { + self.drop_expired(now); + self.entries.get(key).map(|entry| &entry.value) + } + + pub(crate) fn expires_at(&mut self, key: &str, now: Duration) -> Option { + self.drop_expired(now); + self.entries.get(key).map(|entry| entry.expires_at) + } + + pub(crate) fn len(&mut self, now: Duration) -> usize { + self.drop_expired(now); + self.entries.len() + } + + pub(crate) fn live(&mut self, now: Duration) -> impl Iterator { + self.drop_expired(now); + self.entries + .iter() + .map(|(key, entry)| (key.as_str(), &entry.value)) + } + + pub(crate) fn keys_by_expiry(&mut self, now: Duration) -> impl Iterator { + self.drop_expired(now); + self.by_expiry.iter().map(|(_, key)| key.as_str()) + } + + /// A key the cache does not hold evicts the entries soonest to expire until it fits `capacity`. + pub(crate) fn write( + &mut self, + key: String, + value: V, + deadline: Deadline, + capacity: usize, + now: Duration, + ) { + self.drop_expired(now); + let live = self.take(&key).map(|entry| entry.expires_at); + let expires_at = match (deadline, live) { + (Deadline::UnlessLive(_), Some(live)) => live, + (Deadline::UnlessLive(at) | Deadline::Always(at), _) => at, + }; + while self.entries.len() >= capacity { + let Some((_, evicted)) = self.by_expiry.pop_first() else { + break; + }; + self.entries.remove(&evicted); + } + self.by_expiry.insert((expires_at, key.clone())); + self.entries.insert(key, Entry { value, expires_at }); + } + + pub(crate) fn remove(&mut self, key: &str) { + self.take(key); + } + + pub(crate) fn clear(&mut self) { + self.entries.clear(); + self.by_expiry.clear(); + } + + fn take(&mut self, key: &str) -> Option> { + let entry = self.entries.remove(key)?; + self.by_expiry.remove(&(entry.expires_at, key.to_owned())); + Some(entry) + } + + fn drop_expired(&mut self, now: Duration) { + while self + .by_expiry + .first() + .is_some_and(|(expires_at, _)| *expires_at <= now) + { + if let Some((_, key)) = self.by_expiry.pop_first() { + self.entries.remove(&key); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use rstest::{fixture, rstest}; + + use super::{Deadline, State}; + + const START: Duration = Duration::ZERO; + + fn secs(seconds: u64) -> Duration { + Duration::from_secs(seconds) + } + + fn put(state: &mut State<&'static str>, key: &str, value: &'static str, expires_at: u64) { + state.write( + key.to_owned(), + value, + Deadline::UnlessLive(secs(expires_at)), + 2, + START, + ); + } + + /// Full at capacity 2: `early` expires at 10 s and `late` at 20 s. + #[fixture] + fn full() -> State<&'static str> { + let mut state = State::new(); + put(&mut state, "early", "a", 10); + put(&mut state, "late", "b", 20); + state + } + + #[rstest] + #[case::before(9, Some("a"))] + #[case::at_the_deadline(10, None)] + #[case::after(11, None)] + fn an_entry_is_held_until_its_deadline( + mut full: State<&'static str>, + #[case] now: u64, + #[case] expected: Option<&'static str>, + ) { + assert_eq!(full.get("early", secs(now)).copied(), expected); + } + + #[rstest] + #[case::keeps_its_deadline(Deadline::UnlessLive(secs(50)), 10)] + #[case::takes_the_new_deadline(Deadline::Always(secs(50)), 50)] + fn a_write_to_a_live_key_replaces_the_value_and( + mut full: State<&'static str>, + #[case] deadline: Deadline, + #[case] expected: u64, + ) { + full.write("early".to_owned(), "new", deadline, 2, secs(5)); + + assert_eq!(full.get("early", secs(5)).copied(), Some("new")); + assert_eq!(full.expires_at("early", secs(5)), Some(secs(expected))); + } + + #[rstest] + fn an_expired_key_takes_the_deadline_of_the_next_write(mut full: State<&'static str>) { + full.write( + "early".to_owned(), + "new", + Deadline::UnlessLive(secs(30)), + 2, + secs(10), + ); + + assert_eq!(full.expires_at("early", secs(10)), Some(secs(30))); + } + + #[rstest] + fn a_new_key_at_capacity_evicts_the_entry_soonest_to_expire(mut full: State<&'static str>) { + put(&mut full, "new", "c", 5); + + assert_eq!( + full.keys_by_expiry(START).collect::>(), + ["new", "late"] + ); + } + + #[rstest] + fn entries_with_one_deadline_are_evicted_in_key_order() { + let mut state = State::new(); + put(&mut state, "b", "2", 10); + put(&mut state, "a", "1", 10); + + put(&mut state, "c", "3", 10); + + assert_eq!(state.keys_by_expiry(START).collect::>(), ["b", "c"]); + } + + #[rstest] + fn an_expired_entry_makes_room_before_a_live_one_is_evicted(mut full: State<&'static str>) { + full.write( + "new".to_owned(), + "c", + Deadline::UnlessLive(secs(15)), + 2, + secs(10), + ); + + assert_eq!( + full.keys_by_expiry(secs(10)).collect::>(), + ["new", "late"] + ); + } + + #[rstest] + #[case::keeping_its_deadline(Deadline::UnlessLive(secs(50)))] + #[case::taking_a_new_deadline(Deadline::Always(secs(50)))] + fn a_write_to_a_held_key_at_capacity_evicts_nothing( + mut full: State<&'static str>, + #[case] deadline: Deadline, + ) { + full.write("late".to_owned(), "new", deadline, 2, START); + + assert_eq!(full.get("early", START).copied(), Some("a")); + assert_eq!(full.len(START), 2); + } + + #[rstest] + fn a_key_given_a_later_deadline_is_evicted_by_that_deadline(mut full: State<&'static str>) { + full.write( + "early".to_owned(), + "a", + Deadline::Always(secs(30)), + 2, + START, + ); + + put(&mut full, "new", "c", 40); + + assert_eq!( + full.keys_by_expiry(START).collect::>(), + ["early", "new"] + ); + } + + #[rstest] + fn a_removed_key_written_again_is_indexed_once(mut full: State<&'static str>) { + full.remove("early"); + put(&mut full, "early", "a", 30); + full.remove("early"); + put(&mut full, "early", "a", 40); + + assert_eq!( + full.keys_by_expiry(START).collect::>(), + ["late", "early"] + ); + assert_eq!(full.len(START), 2); + } + + #[rstest] + #[case::a_new_key("new", &["new"])] + #[case::a_held_key("early", &["early"])] + fn a_smaller_capacity_applies_on_the_next_write_of( + mut full: State<&'static str>, + #[case] key: &str, + #[case] kept: &[&str], + ) { + full.write(key.to_owned(), "c", Deadline::Always(secs(40)), 1, START); + + assert_eq!(full.keys_by_expiry(START).collect::>(), kept); + } + + #[rstest] + fn an_expired_entry_is_not_listed(mut full: State<&'static str>) { + assert_eq!(full.live(secs(10)).collect::>(), [("late", &"b")]); + } + + #[rstest] + fn an_expired_entry_is_not_counted(mut full: State<&'static str>) { + assert_eq!(full.len(secs(10)), 1); + } + + #[rstest] + fn an_expired_entry_has_no_deadline(mut full: State<&'static str>) { + assert_eq!(full.expires_at("early", secs(10)), None); + } + + #[rstest] + fn an_expired_entry_is_not_among_the_keys_by_expiry(mut full: State<&'static str>) { + assert_eq!(full.keys_by_expiry(secs(10)).collect::>(), ["late"]); + } + + #[rstest] + fn clear_drops_every_entry_and_its_deadline(mut full: State<&'static str>) { + full.clear(); + + assert_eq!(full.len(START), 0); + assert_eq!(full.keys_by_expiry(START).count(), 0); + } + + #[rstest] + fn a_deadline_at_the_end_of_time_is_never_reached() { + let mut state = State::new(); + state.write( + "forever".to_owned(), + "a", + Deadline::Always(Duration::MAX), + 2, + START, + ); + + assert_eq!(state.get("forever", secs(u64::MAX)).copied(), Some("a")); + } +} diff --git a/litellm/rust/crates/cache-memory/src/test_support.rs b/litellm/rust/crates/cache-memory/src/test_support.rs new file mode 100644 index 00000000000..53f68e5e035 --- /dev/null +++ b/litellm/rust/crates/cache-memory/src/test_support.rs @@ -0,0 +1,33 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use rstest::fixture; + +use crate::InMemoryCache; + +pub(crate) type Clock = Arc; + +#[fixture] +pub(crate) fn clock() -> Clock { + Arc::new(AtomicU64::new(100)) +} + +pub(crate) fn cache(clock: &Clock, capacity: usize) -> InMemoryCache { + let clock = clock.clone(); + InMemoryCache::with_clock(Some(capacity), Some(Duration::from_secs(60)), move || { + Duration::from_secs(clock.load(Ordering::SeqCst)) + }) +} + +pub(crate) fn at(clock: &Clock, seconds: u64) { + clock.store(seconds, Ordering::SeqCst); +} + +pub(crate) fn secs(seconds: u64) -> Option { + Some(Duration::from_secs(seconds)) +} diff --git a/litellm/rust/crates/cache-memory/tests/cache.rs b/litellm/rust/crates/cache-memory/tests/cache.rs deleted file mode 100644 index e6c74e5ccda..00000000000 --- a/litellm/rust/crates/cache-memory/tests/cache.rs +++ /dev/null @@ -1,731 +0,0 @@ -use std::{ - collections::HashSet, - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, - time::Duration, -}; - -use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheBackend, ClaimCache, CounterCache, DeleteCache, - DisconnectCache, Error, ExactCacheContext, FlushCache, IncrementOperation, SetCache, TtlCache, - get_cache, set_cache, -}; -use litellm_cache_memory::{CacheWrite, InMemoryCache}; -use rstest::{fixture, rstest}; - -type Clock = Arc; - -#[fixture] -fn clock() -> Clock { - Arc::new(AtomicU64::new(100)) -} - -fn cache_with(clock: &Clock, capacity: usize) -> InMemoryCache { - let clock = clock.clone(); - InMemoryCache::with_clock(Some(capacity), Some(Duration::from_secs(60)), move || { - Duration::from_millis(clock.load(Ordering::SeqCst) * 1000) - }) -} - -fn cache(clock: &Clock, capacity: usize) -> InMemoryCache { - cache_with(clock, capacity) -} - -fn at(clock: &Clock, seconds: u64) { - clock.store(seconds, Ordering::SeqCst); -} - -fn secs(seconds: u64) -> Option { - Some(Duration::from_secs(seconds)) -} - -fn ttl(seconds: u64) -> ExactCacheContext { - ExactCacheContext { ttl: secs(seconds) } -} - -fn measured(capacity: usize) -> InMemoryCache { - InMemoryCache::with_clock_and_size_measurement( - Some(capacity), - secs(60), - Some(4), - Some(Arc::new(|value: &String| { - if value.is_empty() { - return Err(Error::InvalidEntry(None)); - } - Ok(value.len()) - })), - || Duration::from_secs(100), - ) -} - -#[rstest] -fn default_explicit_and_override_ttls_follow_python_rules(clock: Clock) { - let cache = cache(&clock, 4); - cache.set_cache("key", "first".into(), None).unwrap(); - assert_eq!(cache.expires_at("key").unwrap(), secs(160)); - cache.set_cache("key", "second".into(), secs(10)).unwrap(); - assert_eq!(cache.expires_at("key").unwrap(), secs(160)); - at(&clock, 160); - assert_eq!(cache.get_cache("key").unwrap(), Some("second".into())); - at(&clock, 161); - assert_eq!(cache.get_cache("key").unwrap(), None); - assert_eq!(cache.expires_at("key").unwrap(), None); - cache.set_cache("key", "third".into(), secs(10)).unwrap(); - assert_eq!(cache.expires_at("key").unwrap(), secs(171)); -} - -#[rstest] -#[case::unset(None, secs(600))] -#[case::zero_falls_back_like_python_or(Some(Duration::ZERO), secs(600))] -#[case::explicit(secs(5), secs(5))] -fn default_ttl_falls_back_to_ten_minutes( - #[case] default_ttl: Option, - #[case] expected: Option, -) { - let cache = InMemoryCache::::with_clock(None, default_ttl, || Duration::ZERO); - assert_eq!(cache.get_ttl(&ExactCacheContext::default()), expected); - cache.set_cache("key", "value".into(), None).unwrap(); - assert_eq!(cache.expires_at("key").unwrap(), expected); - assert_eq!(cache.max_size_in_memory(), 200); -} - -#[rstest] -fn write_at_expiry_boundary_refreshes_ttl(clock: Clock) { - let cache = cache(&clock, 4); - cache.set_cache("key", "first".into(), secs(10)).unwrap(); - at(&clock, 110); - cache.set_cache("key", "second".into(), secs(10)).unwrap(); - assert_eq!(cache.expires_at("key").unwrap(), secs(120)); - at(&clock, 115); - assert_eq!(cache.get_cache("key").unwrap(), Some("second".into())); -} - -#[rstest] -fn expired_key_without_a_read_allows_a_ttl_override(clock: Clock) { - let cache = cache(&clock, 4); - cache.set_cache("key", "first".into(), secs(1)).unwrap(); - assert_eq!(cache.allow_ttl_override("key"), Ok(false)); - at(&clock, 102); - assert_eq!(cache.allow_ttl_override("key"), Ok(true)); - cache.set_cache("key", "second".into(), secs(1)).unwrap(); - assert_eq!(cache.expires_at("key").unwrap(), secs(103)); - assert_eq!(cache.allow_ttl_override("missing"), Ok(true)); -} - -#[rstest] -fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Clock) { - let cache = cache(&clock, 2); - cache.set_cache("early", "a".into(), secs(10)).unwrap(); - cache.set_cache("late", "b".into(), secs(20)).unwrap(); - cache.delete_cache("early").unwrap(); - cache.set_cache("new", "c".into(), secs(30)).unwrap(); - assert_eq!(cache.get_cache("late").unwrap(), Some("b".into())); - cache.set_cache("last", "d".into(), secs(40)).unwrap(); - assert_eq!(cache.get_cache("late").unwrap(), None); -} - -#[rstest] -fn max_size_is_respected_when_every_item_has_a_long_ttl(clock: Clock) { - let cache = cache(&clock, 3); - for index in 0..3 { - at(&clock, 100 + index); - cache - .set_cache( - format!("key_{index}"), - format!("value_{index}"), - secs(86_400), - ) - .unwrap(); - } - assert_eq!(cache.len(), Ok(3)); - cache - .set_cache("key_3", "value_3".into(), secs(86_400)) - .unwrap(); - assert_eq!(cache.len(), Ok(3)); - assert_eq!(cache.get_cache("key_0").unwrap(), None); - assert_eq!(cache.expires_at("key_0").unwrap(), None); - for key in ["key_1", "key_2", "key_3"] { - assert!(cache.get_cache(key).unwrap().is_some(), "{key}"); - } -} - -#[rstest] -fn expired_items_are_evicted_before_live_ones(clock: Clock) { - let cache = cache(&clock, 3); - cache.set_cache("expired_1", "1".into(), secs(1)).unwrap(); - cache.set_cache("expired_2", "2".into(), secs(1)).unwrap(); - cache - .set_cache("long_lived", "3".into(), secs(86_400)) - .unwrap(); - assert_eq!(cache.len(), Ok(3)); - at(&clock, 102); - cache - .set_cache("new_item", "4".into(), secs(86_400)) - .unwrap(); - assert_eq!(cache.len(), Ok(2)); - assert_eq!(cache.get_cache("long_lived").unwrap(), Some("3".into())); - assert_eq!(cache.get_cache("new_item").unwrap(), Some("4".into())); - for key in ["expired_1", "expired_2"] { - assert_eq!(cache.expires_at(key).unwrap(), None, "{key}"); - } -} - -#[rstest] -fn injected_clock_controls_expiry_and_eviction(clock: Clock) { - let cache = cache(&clock, 2); - at(&clock, 0); - cache - .set_cache("first", "original".into(), secs(10)) - .unwrap(); - at(&clock, 9); - cache.set_cache("second", "survivor".into(), None).unwrap(); - assert_eq!(cache.get_cache("first").unwrap(), Some("original".into())); - at(&clock, 11); - assert_eq!(cache.get_cache("first").unwrap(), None); - cache - .set_cache("third", "replacement".into(), None) - .unwrap(); - assert_eq!(cache.get_cache("second").unwrap(), Some("survivor".into())); - at(&clock, 70); - cache.set_cache("fourth", "new".into(), None).unwrap(); - assert_eq!(cache.get_cache("second").unwrap(), None); - assert_eq!( - cache.get_cache("third").unwrap(), - Some("replacement".into()) - ); - assert_eq!(cache.get_cache("fourth").unwrap(), Some("new".into())); -} - -#[rstest] -fn rewriting_one_key_keeps_one_heap_entry(clock: Clock) { - let cache = cache(&clock, 10); - for index in 0..1_000 { - cache - .set_cache("hot_key", format!("value_{index}"), secs(60)) - .unwrap(); - } - assert_eq!(cache.expiration_heap_len(), Ok(1)); -} - -#[rstest] -fn repeated_increments_keep_one_heap_entry_per_expiration() { - let cache = InMemoryCache::::new(Some(4), None); - for _ in 0..100 { - cache - .increment_cache("counter", 1.0, ExactCacheContext::default()) - .unwrap(); - } - assert_eq!(cache.expiration_heap_len(), Ok(1)); -} - -#[rstest] -fn reinserting_expired_keys_below_capacity_prunes_the_heap(clock: Clock) { - let cache = cache(&clock, 200); - for cycle in 0..3 { - for index in 0..5 { - cache - .set_cache(format!("key_{index}"), format!("value_{cycle}"), secs(1)) - .unwrap(); - } - at(&clock, 100 + 2 * (cycle + 1)); - } - for index in 0..5 { - cache - .set_cache(format!("key_{index}"), "final".into(), secs(1)) - .unwrap(); - } - assert_eq!(cache.len(), Ok(5)); - assert_eq!(cache.expiration_heap_len(), Ok(5)); -} - -#[rstest] -fn evict_cache_drops_expired_entries_then_makes_room(clock: Clock) { - let cache = cache(&clock, 2); - assert_eq!(cache.is_empty(), Ok(true)); - cache.set_cache("short", "a".into(), secs(1)).unwrap(); - cache.set_cache("long", "b".into(), secs(50)).unwrap(); - at(&clock, 102); - cache.evict_cache().unwrap(); - assert_eq!(cache.len(), Ok(1)); - assert_eq!(cache.expires_at("short").unwrap(), None); - cache.set_cache("longer", "c".into(), secs(90)).unwrap(); - cache.evict_cache().unwrap(); - assert_eq!(cache.len(), Ok(1)); - assert_eq!(cache.get_cache("long").unwrap(), None); - assert_eq!(cache.get_cache("longer").unwrap(), Some("c".into())); -} - -#[rstest] -fn evict_element_if_expired_reports_removal(clock: Clock) { - let cache = cache(&clock, 4); - cache.set_cache("key", "value".into(), secs(10)).unwrap(); - assert_eq!(cache.evict_element_if_expired("key"), Ok(false)); - assert_eq!(cache.evict_element_if_expired("missing"), Ok(false)); - at(&clock, 110); - assert_eq!(cache.evict_element_if_expired("key"), Ok(false)); - at(&clock, 111); - assert_eq!(cache.evict_element_if_expired("key"), Ok(true)); - assert_eq!(cache.len(), Ok(0)); - assert_eq!(cache.expires_at("key").unwrap(), None); -} - -#[rstest] -#[case::fits("ok", Ok(true))] -#[case::at_limit("four", Ok(true))] -#[case::too_large("oversized", Ok(false))] -#[case::measure_error("", Err(Error::InvalidEntry(None)))] -fn check_value_size_applies_the_entry_limit( - #[case] value: &str, - #[case] expected: Result, -) { - assert_eq!(measured(2).check_value_size(&value.to_string()), expected); -} - -#[rstest] -fn values_are_unbounded_without_a_measure() { - let cache = InMemoryCache::::default(); - assert_eq!(cache.max_entry_bytes(), None); - assert_eq!(cache.check_value_size(&"x".repeat(1 << 20)), Ok(true)); -} - -#[rstest] -fn disabled_size_limited_and_validated_writes_are_observable() { - assert_eq!( - measured(0).set_cache("a", "x".into(), None).unwrap(), - CacheWrite::Disabled - ); - let cache = measured(2); - assert_eq!(cache.max_entry_bytes(), Some(4)); - assert_eq!( - cache.set_cache("large", "oversized".into(), None).unwrap(), - CacheWrite::TooLarge - ); - assert_eq!(cache.get_cache("large").unwrap(), None); - assert_eq!( - cache.set_cache("small", "ok".into(), None).unwrap(), - CacheWrite::Stored - ); - assert_eq!(cache.get_cache("small").unwrap(), Some("ok".into())); - assert_eq!( - cache.set_cache("invalid", String::new(), None), - Err(Error::InvalidEntry(None)) - ); - assert_eq!(cache.get_cache("invalid").unwrap(), None); - cache.delete_cache("small").unwrap(); - assert_eq!(cache.get_cache("small").unwrap(), None); -} - -#[rstest] -#[tokio::test] -async fn disconnect_is_a_no_op_that_keeps_entries() { - let cache = InMemoryCache::::default(); - cache.set_cache("key", "value".into(), None).unwrap(); - cache.disconnect().await.unwrap(); - assert_eq!(cache.get_cache("key").unwrap(), Some("value".into())); -} - -#[rstest] -#[tokio::test] -async fn generic_consumers_share_typed_values_and_honor_expiration(clock: Clock) { - let cache: CacheBackend> = Arc::new(self::cache(&clock, 4)); - let reader = Arc::clone(&cache); - let context = ttl(5); - set_cache(cache.as_ref(), "sync", "first".into(), &context).unwrap(); - assert_eq!( - get_cache(reader.as_ref(), "sync", &context).unwrap(), - Some("first".into()) - ); - cache - .batch_cache_write("async", "second".into(), context.clone()) - .await - .unwrap(); - cache - .async_set_cache_pipeline(vec![("batch".into(), "third".into())], context.clone()) - .await - .unwrap(); - drop(cache); - for (key, value) in [("sync", "first"), ("async", "second"), ("batch", "third")] { - assert_eq!( - reader.async_get_cache(key, &context).await.unwrap(), - Some(value.into()) - ); - } - reader.async_delete_cache("async").await.unwrap(); - assert_eq!( - reader.async_get_cache("async", &context).await.unwrap(), - None - ); - at(&clock, 106); - assert_eq!(get_cache(reader.as_ref(), "sync", &context).unwrap(), None); - assert_eq!( - reader.async_get_cache("batch", &context).await.unwrap(), - None - ); -} - -#[rstest] -#[case::context_ttl(ttl(5), secs(105))] -#[case::default_ttl(ExactCacheContext::default(), secs(160))] -#[tokio::test] -async fn pipeline_writes_use_the_context_ttl_or_the_default( - clock: Clock, - #[case] context: ExactCacheContext, - #[case] expected: Option, -) { - let cache = cache(&clock, 4); - cache - .async_set_cache_pipeline( - vec![("a".into(), "1".into()), ("b".into(), "2".into())], - context, - ) - .await - .unwrap(); - assert_eq!(cache.expires_at("a").unwrap(), expected); - assert_eq!(cache.expires_at("b").unwrap(), expected); -} - -#[rstest] -#[tokio::test] -async fn batch_reads_return_one_entry_per_key_and_drop_expired_ones(clock: Clock) { - let cache = cache(&clock, 4); - cache.set_cache("short", "a".into(), secs(1)).unwrap(); - cache.set_cache("long", "b".into(), secs(50)).unwrap(); - let keys = vec!["short".to_string(), "missing".into(), "long".into()]; - assert_eq!( - cache - .batch_get_cache(&keys, &ExactCacheContext::default()) - .unwrap(), - [ - BatchEntry::Hit("a".to_string()), - BatchEntry::Miss, - BatchEntry::Hit("b".into()), - ] - ); - at(&clock, 102); - assert_eq!( - cache - .async_batch_get_cache(keys, ExactCacheContext::default()) - .await - .unwrap(), - [ - BatchEntry::Miss, - BatchEntry::Miss, - BatchEntry::Hit("b".into()) - ] - ); -} - -#[rstest] -#[tokio::test] -async fn flush_clears_values_and_expirations(clock: Clock) { - let cache = cache(&clock, 4); - cache.set_cache("a", "1".into(), None).unwrap(); - cache.set_cache("b", "2".into(), None).unwrap(); - cache.flush_cache().unwrap(); - assert_eq!(cache.len(), Ok(0)); - assert_eq!(cache.expiration_heap_len(), Ok(0)); - cache.set_cache("c", "3".into(), None).unwrap(); - FlushCache::async_flush_cache(&cache).await.unwrap(); - assert_eq!(cache.is_empty(), Ok(true)); - assert_eq!( - cache.async_get_oldest_n_keys(5).await.unwrap(), - Vec::::new() - ); -} - -#[rstest] -fn claims_are_atomic_and_refresh_eligible_winners(clock: Clock) { - let cache = cache(&clock, 4); - let context = ttl(10); - assert_eq!( - cache - .claim_cache("affinity", "first".to_string(), &[], context.clone()) - .unwrap(), - "first" - ); - at(&clock, 103); - assert_eq!( - cache - .claim_cache("affinity", "second".to_string(), &[], context.clone()) - .unwrap(), - "first" - ); - assert_eq!(cache.expires_at("affinity").unwrap(), secs(110)); - at(&clock, 105); - assert_eq!( - cache - .claim_cache( - "affinity", - "second".to_string(), - &["first".to_string(), "second".to_string()], - context, - ) - .unwrap(), - "first" - ); - assert_eq!(cache.expires_at("affinity").unwrap(), secs(115)); -} - -#[rstest] -fn counters_increment_under_one_lock() { - let cache = InMemoryCache::::default(); - assert_eq!( - CounterCache::increment_cache(&cache, "counter", 1.5, ExactCacheContext::default()) - .unwrap(), - 1.5 - ); - assert_eq!( - CounterCache::increment_cache(&cache, "counter", 2.0, ExactCacheContext::default()) - .unwrap(), - 3.5 - ); -} - -#[rstest] -fn concurrent_increments_are_atomic() { - let cache = Arc::new(InMemoryCache::::default()); - cache.set_cache("counter", 1000.0, None).unwrap(); - let threads = (0..8) - .map(|_| { - let cache = cache.clone(); - std::thread::spawn(move || { - cache - .increment_cache("counter", 1.0, ExactCacheContext::default()) - .unwrap() - }) - }) - .collect::>(); - for thread in threads { - thread.join().unwrap(); - } - assert_eq!(cache.get_cache("counter").unwrap(), Some(1008.0)); -} - -#[rstest] -#[case::window_semantics(false)] -#[case::refresh_ttl_is_ignored(true)] -#[tokio::test] -async fn async_increment_delegates_to_the_locked_sync_path( - clock: Clock, - #[case] refresh_ttl: bool, -) { - let cache = cache_with::(&clock, 4); - assert_eq!( - cache - .async_increment("counter", 2.0, ttl(10), refresh_ttl) - .await, - Ok(2.0) - ); - at(&clock, 105); - assert_eq!( - cache - .async_increment("counter", 3.0, ttl(10), refresh_ttl) - .await, - Ok(5.0) - ); - assert_eq!(cache.get_cache("counter").unwrap(), Some(5.0)); - assert_eq!(cache.expires_at("counter").unwrap(), secs(110)); -} - -#[rstest] -fn expired_counters_restart_from_zero_with_a_new_ttl(clock: Clock) { - let cache = cache_with::(&clock, 4); - cache.increment_cache("counter", 2.0, ttl(10)).unwrap(); - at(&clock, 111); - assert_eq!(cache.increment_cache("counter", 1.0, ttl(10)), Ok(1.0)); - assert_eq!(cache.expires_at("counter").unwrap(), secs(121)); -} - -/// Python `InMemoryCache.set_cache` runs `evict_cache()` before every insert, and step 2 evicts -/// the earliest expiry while `len(cache_dict) >= max_size_in_memory`, even when the key being -/// written already exists. -#[rstest] -fn overwriting_an_existing_key_at_capacity_evicts_the_earliest_expiry_like_python(clock: Clock) { - let cache = cache(&clock, 2); - cache.set_cache("hot", "1".into(), secs(10)).unwrap(); - cache.set_cache("cold", "2".into(), secs(20)).unwrap(); - - cache.set_cache("cold", "3".into(), None).unwrap(); - - assert_eq!(cache.get_cache("hot").unwrap(), None); - assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into())); -} - -/// `claim_cache` has no Python counterpart; it never evicts another entry for a key it holds. -#[rstest] -fn claiming_an_existing_key_at_capacity_keeps_other_entries(clock: Clock) { - let cache = cache(&clock, 2); - cache.set_cache("hot", "1".into(), secs(10)).unwrap(); - cache.set_cache("cold", "2".into(), secs(20)).unwrap(); - - cache - .claim_cache("cold", "4".into(), &[], ExactCacheContext::default()) - .unwrap(); - assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into())); - assert_eq!(cache.get_cache("cold").unwrap(), Some("2".into())); -} - -/// Python `increment_cache` is `get_cache` then `set_cache`, so at capacity the write evicts -/// the earliest expiry first: equal expiries tie-break on the key, and the value read before -/// eviction is the one written back. -#[rstest] -fn incrementing_at_capacity_evicts_the_earliest_expiry_like_python(clock: Clock) { - let cache = cache_with::(&clock, 2); - for key in ["a", "b", "a", "b"] { - cache - .increment_cache(key, 1.0, ExactCacheContext::default()) - .unwrap(); - } - assert_eq!(cache.get_cache("a").unwrap(), None); - assert_eq!(cache.get_cache("b").unwrap(), Some(2.0)); -} - -#[rstest] -#[tokio::test] -async fn disabled_cache_does_not_retain_claims_counters_or_sets() { - let claims = InMemoryCache::::new(Some(0), None); - assert_eq!( - claims - .claim_cache("key", "first".into(), &[], ExactCacheContext::default()) - .unwrap(), - "first" - ); - assert_eq!(claims.get_cache("key").unwrap(), None); - - let counters = InMemoryCache::::new(Some(0), None); - assert_eq!( - counters - .increment_cache("key", 2.0, ExactCacheContext::default()) - .unwrap(), - 2.0 - ); - assert_eq!(counters.get_cache("key").unwrap(), None); - - let sets = InMemoryCache::>::new(Some(0), None); - assert_eq!( - sets.async_set_cache_sadd("key", vec!["a".into()], None) - .await - .unwrap(), - ["a"] - ); - assert_eq!(sets.get_cache("key").unwrap(), None); -} - -#[rstest] -#[tokio::test] -async fn ttl_and_oldest_key_operations_use_the_stored_expirations(clock: Clock) { - let cache = cache(&clock, 3); - cache.set_cache("later", "2".into(), secs(20)).unwrap(); - cache.set_cache("first", "1".into(), secs(10)).unwrap(); - cache.set_cache("latest", "3".into(), secs(30)).unwrap(); - - assert_eq!(cache.async_get_ttl("first").await.unwrap(), secs(110)); - assert_eq!( - TtlCache::async_get_ttl(&cache, "later").await.unwrap(), - secs(120) - ); - assert_eq!(cache.async_get_oldest_n_keys(1).await.unwrap(), ["first"]); - assert_eq!( - cache.async_get_oldest_n_keys(10).await.unwrap(), - ["first", "later", "latest"] - ); - assert_eq!( - cache.async_get_oldest_n_keys(0).await.unwrap(), - Vec::::new() - ); - assert_eq!(cache.async_get_ttl("missing").await.unwrap(), None); -} - -#[rstest] -#[tokio::test] -async fn increment_pipeline_preserves_operation_order(clock: Clock) { - let cache = cache_with::(&clock, 3); - let operation = |key: &str, amount, ttl| IncrementOperation { - key: key.into(), - amount, - ttl: secs(ttl), - }; - assert_eq!( - cache - .async_increment_pipeline(vec![ - operation("a", 1.0, 10), - operation("b", 5.0, 30), - operation("a", 2.0, 20), - ]) - .await - .unwrap(), - [1.0, 5.0, 3.0] - ); - assert_eq!(cache.get_cache("a").unwrap(), Some(3.0)); - assert_eq!(cache.expires_at("a").unwrap(), secs(110)); - assert_eq!(cache.expires_at("b").unwrap(), secs(130)); - assert_eq!( - cache.async_increment_pipeline(Vec::new()).await.unwrap(), - Vec::::new() - ); -} - -#[rstest] -#[tokio::test] -async fn set_capability_preserves_python_result_and_deduplicates_storage(clock: Clock) { - let cache = cache_with::>(&clock, 4); - let inserted = vec!["a".to_string(), "a".into(), "b".into()]; - assert_eq!( - cache - .async_set_cache_sadd("members", inserted.clone(), secs(10)) - .await - .unwrap(), - inserted - ); - assert_eq!( - cache - .async_set_cache_sadd("members", vec!["c".into()], secs(99)) - .await - .unwrap(), - ["c"] - ); - assert_eq!( - cache.get_cache("members").unwrap(), - Some(HashSet::from(["a".into(), "b".into(), "c".into()])) - ); - assert_eq!(cache.expires_at("members").unwrap(), secs(110)); - at(&clock, 111); - cache - .async_set_cache_sadd("members", vec!["d".into()], None) - .await - .unwrap(); - assert_eq!( - cache.get_cache("members").unwrap(), - Some(HashSet::from(["d".into()])) - ); - assert_eq!(cache.expires_at("members").unwrap(), secs(171)); -} - -#[rstest] -#[tokio::test] -async fn oversized_set_additions_are_not_stored() { - let cache = InMemoryCache::>::with_clock_and_size_measurement( - Some(4), - None, - Some(2), - Some(Arc::new(|value: &HashSet| Ok(value.len()))), - || Duration::ZERO, - ); - cache - .async_set_cache_sadd("members", vec!["a".into(), "b".into()], None) - .await - .unwrap(); - assert_eq!( - cache - .async_set_cache_sadd("members", vec!["c".into()], None) - .await - .unwrap(), - ["c"] - ); - assert_eq!( - cache.get_cache("members").unwrap(), - Some(HashSet::from(["a".into(), "b".into()])) - ); -} diff --git a/litellm/rust/crates/callbacks-legacy-python/Cargo.toml b/litellm/rust/crates/callbacks-legacy-python/Cargo.toml index 0133af16b80..7819979029c 100644 --- a/litellm/rust/crates/callbacks-legacy-python/Cargo.toml +++ b/litellm/rust/crates/callbacks-legacy-python/Cargo.toml @@ -11,6 +11,7 @@ litellm-host.workspace = true litellm-host-python.workspace = true pyo3.workspace = true +serde.workspace = true strum.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/litellm/rust/crates/callbacks-legacy-python/python_contract.json b/litellm/rust/crates/callbacks-legacy-python/python_contract.json index a96e9f04bdc..505651b5f16 100644 --- a/litellm/rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm/rust/crates/callbacks-legacy-python/python_contract.json @@ -28,7 +28,7 @@ "api_key", "additional_args" ], - "messages_input": [ + "messages_input_order": [ "body", "kwargs" ], @@ -43,6 +43,7 @@ "status", "headers", "body", + "streamed", "sent" ], "retried_request": [ diff --git a/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs index 71ca89b0790..33dc9996bb9 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/adapter.rs @@ -21,7 +21,7 @@ use serde_json::Value; use crate::{ DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, SentRequest, deferred::{PendingLogging, PendingSuccess}, - finalize, is_internal_call, + finalize, is_internal_call, messages_input, python::{Logging, Streaming}, setup, }; @@ -216,12 +216,18 @@ impl LegacyLogging { &self, py: Python<'py>, body: &Bound<'py, PyDict>, + sent: &Value, + as_called: &serde_json::Map, ) -> PyResult> { let description = match self.operation { LoggingOperation::Completion => "Chat completions", LoggingOperation::Responses => "Responses", LoggingOperation::Messages => { - return Logging::MessagesInput.call(py, (body, self.call.kwargs().bind(py))); + let order: Vec = Logging::MessagesInputOrder + .call(py, (body, self.call.kwargs().bind(py)))? + .extract()?; + let content = messages_input::serialized(&order, sent, as_called)?; + return messages_input::logged(py, &content); } LoggingOperation::Ocr => "OCR document processing", }; @@ -598,8 +604,12 @@ impl LegacyLogging { to_py(py, &wire.body)?.into_bound(py) } .cast_into::()?; + let mut as_called = serde_json::Map::new(); for (name, sent) in wire.body.as_object().into_iter().flatten() { if let Some(value) = self.caller_value(py, name, sent)? { + if lossless && messages_input::is_caller_scalar(&value) { + as_called.insert(name.clone(), json_from_py(&value)?); + } body.set_item(name, value)?; } } @@ -620,7 +630,7 @@ impl LegacyLogging { url: wire.url.clone(), context: context.clone(), }); - let input = self.logged_input(py, &body)?; + let input = self.logged_input(py, &body, &wire.body, &as_called)?; self.logger()?.pre_call( py, &input, @@ -752,10 +762,16 @@ impl LegacyLogging { let sent = self.request.as_ref().map(|request| SentRequest { url: &request.url, headers: &request.headers, - body: &request.body, + body: &head.sent_body, }); - self.logger()? - .record_response(py, head.status, &head.headers, &raw.body, sent)?; + self.logger()?.record_response( + py, + head.status, + &head.headers, + &raw.body, + head.streamed, + sent, + )?; } return Ok(HookStep::Ready(())); } @@ -1734,7 +1750,7 @@ assert reasons == [{'kind': 'thinking_signature'}, {'kind': 'strict_tools', 'dro #[case::responses(crate::LoggingOperation::Responses, "Responses")] #[case::messages( crate::LoggingOperation::Messages, - r#"[{'role': 'user', 'content': '{"pages": [1]}'}]"# + r#"[{'role': 'user', 'content': '{"pages":[1]}'}]"# )] #[case::ocr(crate::LoggingOperation::Ocr, "OCR document processing")] fn prepared_arguments_replace_the_legacy_view_without_losing_callback_aliases( @@ -1783,6 +1799,48 @@ assert original == [0] }); } + #[rstest] + fn the_messages_input_is_the_body_before_pre_call_edits_with_the_callers_scalars() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, PAYLOAD_LOGGER); + run( + py, + &locals, + c" +kwargs['temperature'] = 1 +def on_pre_call(additional_args): + additional_args['complete_input_dict']['system'] = 'edited' +", + ); + let mut logging = LegacyLogging { + operation: crate::LoggingOperation::Messages, + logger: Some(PythonLogger::new(local(&locals, "logger").unbind())), + ..legacy_call(py, &locals, false) + }; + let wire = WireRequest { + body: json!({"model": "m", "system": "caf\u{e9} \u{1f600}", "temperature": 1.0}), + ..route_wire() + }; + + let (_, step) = send_and_receive(py, &mut logging, wire, &route_context()); + + let HookStep::Ready(sent) = step else { + panic!("pre_call answers at once"); + }; + assert_eq!(sent.body["system"], "edited"); + run( + py, + &locals, + c" +assert logger.pre_input == [ + {'role': 'user', 'content': '{\"model\":\"m\",\"system\":\"caf\u{e9} \u{1f600}\",\"temperature\":1}'} +], logger.pre_input +", + ); + }); + } + #[rstest::rstest] fn a_cycle_through_the_retained_headers_is_collected() { Python::initialize(); diff --git a/litellm/rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm/rust/crates/callbacks-legacy-python/src/callbacks.rs index 9eed008910c..ff31a686a8a 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -4,7 +4,11 @@ use litellm_host::interceptors::{RequestContext, WireRequest}; use litellm_host_python::json_to_py; -use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; +use pyo3::{ + exceptions::PyBaseException, + prelude::*, + types::{PyBytes, PyDict}, +}; use serde_json::Value; use crate::logger::PythonLogger; @@ -13,7 +17,7 @@ use crate::python::{Logging, Wrapper}; pub struct SentRequest<'a> { pub url: &'a str, pub headers: &'a Py, - pub body: &'a Py, + pub body: &'a [u8], } pub trait LegacyCallbacks { @@ -47,6 +51,7 @@ pub trait LegacyCallbacks { status: u16, headers: &[(String, String)], body: &str, + streamed: bool, request: Option>, ) -> PyResult<()>; @@ -182,12 +187,21 @@ impl LegacyCallbacks for PythonLogger { status: u16, headers: &[(String, String)], body: &str, + streamed: bool, request: Option>, ) -> PyResult<()> { - let request = request.map(|sent| (sent.url, sent.headers.bind(py), sent.body.bind(py))); + let request = + request.map(|sent| (sent.url, sent.headers.bind(py), PyBytes::new(py, sent.body))); Logging::RecordResponse.call( py, - (self.object(py), status, headers.to_vec(), body, request), + ( + self.object(py), + status, + headers.to_vec(), + body, + streamed, + request, + ), )?; Ok(()) } diff --git a/litellm/rust/crates/callbacks-legacy-python/src/error.rs b/litellm/rust/crates/callbacks-legacy-python/src/error.rs index 97ce62ae99c..4be7e31b81f 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/error.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/error.rs @@ -5,13 +5,17 @@ use pyo3::{PyErr, exceptions::PyRuntimeError}; pub(crate) enum Error { #[error("call logging is not initialized")] LoggingNotInitialized, + #[error("the Messages input could not be serialized for logging")] + LoggedInput(#[source] serde_json::Error), } impl From for PyErr { fn from(error: Error) -> Self { let message = Report::new(&error).to_string(); match error { - Error::LoggingNotInitialized => PyRuntimeError::new_err(message), + Error::LoggingNotInitialized | Error::LoggedInput(_) => { + PyRuntimeError::new_err(message) + } } } } diff --git a/litellm/rust/crates/callbacks-legacy-python/src/lib.rs b/litellm/rust/crates/callbacks-legacy-python/src/lib.rs index 3a8c15ae54c..8ee096bd189 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/lib.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/lib.rs @@ -14,6 +14,7 @@ mod deferred; mod error; mod logger; mod mapping; +mod messages_input; mod python; pub use adapter::LegacyLogging; pub use call::PublicCall; diff --git a/litellm/rust/crates/callbacks-legacy-python/src/messages_input.rs b/litellm/rust/crates/callbacks-legacy-python/src/messages_input.rs new file mode 100644 index 00000000000..3cf42205cb5 --- /dev/null +++ b/litellm/rust/crates/callbacks-legacy-python/src/messages_input.rs @@ -0,0 +1,135 @@ +use pyo3::{ + prelude::*, + types::{PyBool, PyDict, PyFloat, PyInt, PyList}, +}; +use serde::ser::{Error as _, Serialize, SerializeMap, Serializer}; +use serde_json::{Map, Value}; + +use crate::error::Error; + +struct Ordered<'a> { + order: &'a [String], + sent: &'a Map, + as_called: &'a Map, +} + +impl Serialize for Ordered<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + for name in self.order { + if let Some(value) = self.as_called.get(name).or_else(|| self.sent.get(name)) { + map.serialize_entry(name, value)?; + } + } + for (name, value) in self.sent { + if !self.order.contains(name) { + map.serialize_entry(name, value)?; + } + } + map.end() + } +} + +pub(crate) fn is_caller_scalar(value: &Bound<'_, PyAny>) -> bool { + value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() +} + +pub(crate) fn serialized( + order: &[String], + sent: &Value, + as_called: &Map, +) -> Result { + let bytes = match sent { + Value::Object(sent) => litellm_host::raw_json::to_vec(&Ordered { + order, + sent, + as_called, + }), + other => litellm_host::raw_json::to_vec(other), + } + .map_err(Error::LoggedInput)?; + String::from_utf8(bytes).map_err(|error| Error::LoggedInput(serde_json::Error::custom(error))) +} + +pub(crate) fn logged<'py>(py: Python<'py>, content: &str) -> PyResult> { + let entry = PyDict::new(py); + entry.set_item("role", "user")?; + entry.set_item("content", content)?; + Ok(PyList::new(py, [entry])?.into_any()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn names(order: &[&str]) -> Vec { + order.iter().map(|name| (*name).to_owned()).collect() + } + + #[rstest] + #[case::caller_order( + &["messages", "max_tokens", "model"], + r#"{"messages":[],"max_tokens":8,"model":"m"}"# + )] + #[case::unlisted_keys_follow_in_sent_order( + &["model"], + r#"{"model":"m","max_tokens":8,"messages":[]}"# + )] + #[case::unknown_name_skipped( + &["absent", "model", "max_tokens", "messages"], + r#"{"model":"m","max_tokens":8,"messages":[]}"# + )] + fn top_level_keys_follow_the_given_order(#[case] order: &[&str], #[case] expected: &str) { + let sent = json!({"model": "m", "max_tokens": 8, "messages": []}); + + assert_eq!( + serialized(&names(order), &sent, &Map::new()).unwrap(), + expected + ); + } + + #[rstest] + #[case::int_for_a_float(json!(1.0), json!(1), r#"{"temperature":1}"#)] + #[case::bool_for_an_int(json!(1), json!(true), r#"{"temperature":true}"#)] + fn a_caller_scalar_keeps_its_spelling( + #[case] sent: Value, + #[case] called: Value, + #[case] expected: &str, + ) { + let as_called = Map::from_iter([("temperature".to_owned(), called)]); + + assert_eq!( + serialized( + &names(&["temperature"]), + &json!({"temperature": sent}), + &as_called + ) + .unwrap(), + expected + ); + } + + #[rstest] + #[case::non_ascii(json!("caf\u{e9} \u{1f600}"), "\"caf\u{e9} \u{1f600}\"")] + #[case::small_float(json!(1e-7), "1e-7")] + #[case::wide_integer( + json!(litellm_host::raw_json::encode("1180591620717411303424")), + "1180591620717411303424" + )] + fn values_are_written_as_the_wire_writes_them(#[case] leaf: Value, #[case] spelled: &str) { + let sent = json!({"system": leaf}); + + let logged = serialized(&names(&["system"]), &sent, &Map::new()).unwrap(); + + assert_eq!(logged, format!(r#"{{"system":{spelled}}}"#)); + assert_eq!( + logged.as_bytes(), + litellm_host::raw_json::to_vec(&sent).unwrap() + ); + } +} diff --git a/litellm/rust/crates/callbacks-legacy-python/src/python.rs b/litellm/rust/crates/callbacks-legacy-python/src/python.rs index 58ddf7c708a..85ee3b831ad 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/python.rs @@ -42,8 +42,8 @@ pub(crate) enum Logging { Update, #[strum(serialize = "pre_call")] PreCall, - #[strum(serialize = "messages_input")] - MessagesInput, + #[strum(serialize = "messages_input_order")] + MessagesInputOrder, #[strum(serialize = "post_call")] PostCall, #[strum(serialize = "record_response")] diff --git a/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs index 5b8b2bc4f96..ab1a4ab9cdb 100644 --- a/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm/rust/crates/callbacks-legacy-python/src/test_support.rs @@ -62,11 +62,11 @@ FAKES = { custom_llm_provider=provider, ), 'pre_call': lambda logger, input, api_key, additional_args: logger.pre_call(input, api_key, additional_args), - 'messages_input': lambda body, kwargs: [{'role': 'user', 'content': json.dumps(body)}], + 'messages_input_order': lambda body, kwargs: tuple(body), 'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call( original_response, api_key, additional_args ), - 'record_response': lambda logger, status, headers, body, sent: logger.model_call_details.update( + 'record_response': lambda logger, status, headers, body, streamed, sent: logger.model_call_details.update( response_headers=dict(headers) ), 'retried_request': lambda logger, logged_body, body, reasons: ( diff --git a/litellm/rust/crates/cost/src/price/meter.rs b/litellm/rust/crates/cost/src/price/meter.rs index 09fd23e42ed..887711fe1ed 100644 --- a/litellm/rust/crates/cost/src/price/meter.rs +++ b/litellm/rust/crates/cost/src/price/meter.rs @@ -50,6 +50,8 @@ pub enum OutputUnit { Pixel, Second(Option), VideoSecond, + Video, + BillableUnit, } /// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read diff --git a/litellm/rust/crates/host-python/src/gil.rs b/litellm/rust/crates/host-python/src/gil.rs index 5f54fae81ed..886ed848399 100644 --- a/litellm/rust/crates/host-python/src/gil.rs +++ b/litellm/rust/crates/host-python/src/gil.rs @@ -22,6 +22,18 @@ impl PythonContext { ))) } + /// A context with no request-local values, for work no Python caller is waiting on. + pub fn empty(py: Python<'_>) -> PyResult { + Ok(Self(Arc::new( + py.import("contextvars")?.call_method0("Context")?.unbind(), + ))) + } + + /// A `contextvars.Context` the Python host built for work that outlives its caller. + pub fn built(context: Bound<'_, PyAny>) -> Self { + Self(Arc::new(context.unbind())) + } + /// Runs `f` inside a fresh copy of the captured context. The copy is what lets two blocking /// calls run concurrently: a `contextvars.Context` cannot be entered twice at once. pub fn enter(&self, py: Python<'_>, f: F) -> PyResult diff --git a/litellm/rust/crates/host/Cargo.toml b/litellm/rust/crates/host/Cargo.toml index 2c113bc644c..73392827f31 100644 --- a/litellm/rust/crates/host/Cargo.toml +++ b/litellm/rust/crates/host/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +bytes.workspace = true litellm-secret-value.workspace = true futures-util.workspace = true litellm-coroutine.workspace = true diff --git a/litellm/rust/crates/host/src/interceptors.rs b/litellm/rust/crates/host/src/interceptors.rs index 8146448cd98..40591ec9ccd 100644 --- a/litellm/rust/crates/host/src/interceptors.rs +++ b/litellm/rust/crates/host/src/interceptors.rs @@ -1,5 +1,6 @@ use std::future::Future; +use bytes::Bytes; use serde_json::Value; /// The provider request as it is about to leave, offered to the host for rewriting. @@ -40,7 +41,7 @@ pub struct RequestContext { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RawResponse { pub body: String, - /// The provider's HTTP status and headers, when the route forwards them to logging. + /// The provider's HTTP exchange, when the route forwards it to logging. pub head: Option, } @@ -48,6 +49,8 @@ pub struct RawResponse { pub struct ResponseHead { pub status: u16, pub headers: Vec<(String, String)>, + pub sent_body: Bytes, + pub streamed: bool, } impl RawResponse { diff --git a/litellm/rust/crates/http/src/outbound.rs b/litellm/rust/crates/http/src/outbound.rs index b11041151af..49ff7fa6df0 100644 --- a/litellm/rust/crates/http/src/outbound.rs +++ b/litellm/rust/crates/http/src/outbound.rs @@ -38,7 +38,7 @@ pub fn json( pub fn json_bytes( url: &str, headers: &[(String, String)], - body: Vec, + body: impl Into, timeout: Option, ) -> Result { let url = Url::parse(url).map_err(Repr::ParseUrl)?; diff --git a/litellm/rust/crates/inference-messages/src/handler.rs b/litellm/rust/crates/inference-messages/src/handler.rs index 4796a640e25..d54d4f5f8d1 100644 --- a/litellm/rust/crates/inference-messages/src/handler.rs +++ b/litellm/rust/crates/inference-messages/src/handler.rs @@ -30,8 +30,7 @@ use serde_json::{Map, Value}; use super::{ Delivered, Error, HostAnswer, MessagesBody, MessagesCall, MessagesCallResponse, MessagesRoute, - Transport, TransportBody, TransportRequest, TransportResponse, UnencodableHeader, - WorkloadIdentity, + Transport, TransportRequest, TransportResponse, UnencodableHeader, WorkloadIdentity, common_utils::{MessagesProvider, merge_messages_header_sources, string_headers}, constants::{ MESSAGES_HTTP_ERROR_ATTEMPTS, MESSAGES_STREAM_ERROR_BODY_READ_SECS, MESSAGES_TIMEOUT_SECS, @@ -42,7 +41,7 @@ use super::{ use litellm_inference::{ caching::{CacheRequest, CallCache}, context::CallContext, - outbound::raw_leaf_outbound_request, + outbound::{raw_leaf_body, serialized_outbound_request}, }; struct ProviderCall { @@ -505,9 +504,13 @@ async fn call_provider( let timeout = request .timeout .unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); - let response = match send_recovering(http, transport, &request, timeout, context).await? { + let (upstream, sent_body) = + send_recovering(http, transport, &request, timeout, context).await?; + let response = match upstream { Upstream::Native(response) => response, - Upstream::Hosted(response) => return hosted_response(response, request, context).await, + Upstream::Hosted(response) => { + return hosted_response(response, sent_body, request, context).await; + } }; let ProviderCall { identity, @@ -522,6 +525,8 @@ async fn call_provider( let head = ResponseHead { status: response.status().as_u16(), headers: super::encoding::header_pairs(response.headers()), + sent_body, + streamed: stream, }; if stream { context @@ -556,6 +561,7 @@ async fn call_provider( async fn hosted_response( response: TransportResponse, + sent_body: Bytes, request: ProviderCall, context: &CallContext<'_, impl Interceptors>, ) -> Result { @@ -579,6 +585,8 @@ async fn hosted_response( .raw_response_received(RawResponse::new(String::new()).with_head(ResponseHead { status, headers: headers.clone(), + sent_body, + streamed: true, })) .await?; let streamed = @@ -593,9 +601,12 @@ async fn hosted_response( }; log_response_body(&text); context - .raw_response_received( - RawResponse::new(text.clone()).with_head(ResponseHead { status, headers }), - ) + .raw_response_received(RawResponse::new(text.clone()).with_head(ResponseHead { + status, + headers, + sent_body, + streamed: false, + })) .await?; if serde_json::from_str::(&text).is_err() { return Err(litellm_http::Error::upstream(status, text).into()); @@ -606,14 +617,14 @@ async fn hosted_response( fn hosted_request( authenticated: Authenticated, url: &str, - body: &Value, + body: Bytes, stream: bool, ) -> Result { if authenticated.signer.is_none() { return Ok(TransportRequest { url: url.to_owned(), headers: authenticated.headers, - body: TransportBody::Json(body.clone()), + body, stream, }); } @@ -622,7 +633,7 @@ fn hosted_request( .iter() .map(|(name, _)| name.clone()) .collect(); - let signed = raw_leaf_outbound_request(authenticated, url, body, None)?; + let signed = serialized_outbound_request(authenticated, url, body.clone(), None)?; let headers = signed .headers() .iter() @@ -635,15 +646,10 @@ fn hosted_request( Some((spelled, value.to_str().ok()?.to_owned())) }) .collect(); - let bytes = signed - .body() - .and_then(reqwest::Body::as_bytes) - .map(Bytes::copy_from_slice) - .unwrap_or_default(); Ok(TransportRequest { url: signed.url().to_string(), headers, - body: TransportBody::Signed(bytes), + body, stream, }) } @@ -654,7 +660,7 @@ async fn send_recovering( request: &ProviderCall, timeout: Duration, context: &CallContext<'_, impl Interceptors>, -) -> Result { +) -> Result<(Upstream, Bytes), Error> { let config = request.provider.config(); let mut body = std::borrow::Cow::Borrowed(&request.wire.body); let mut attempt = 1; @@ -680,16 +686,24 @@ async fn send_recovering( headers: request.wire.headers.clone(), signer: request.signer.clone(), }; + let sent_body = raw_leaf_body(&body)?; let response = match transport { None => Upstream::Native( - send(http, authenticated, &request.wire.url, &body, timeout).await?, + send( + http, + authenticated, + &request.wire.url, + sent_body.clone(), + timeout, + ) + .await?, ), Some(transport) => Upstream::Hosted( transport .send(hosted_request( authenticated, &request.wire.url, - &body, + sent_body.clone(), request.stream, )?) .await?, @@ -703,7 +717,7 @@ async fn send_recovering( } let failure = match response { Upstream::Native(response) if response.status().is_success() => { - return Ok(Upstream::Native(response)); + return Ok((Upstream::Native(response), sent_body)); } Upstream::Native(response) => { provider_failure(response, timeout, request.stream).await? @@ -721,7 +735,7 @@ async fn send_recovering( raised: Some(error), } } - hosted @ Upstream::Hosted(_) => return Ok(hosted), + hosted @ Upstream::Hosted(_) => return Ok((hosted, sent_body)), }; let retried = if attempt == MESSAGES_HTTP_ERROR_ATTEMPTS { None @@ -756,10 +770,10 @@ async fn send( http: &litellm_http::Client, authenticated: Authenticated, url: &str, - body: &Value, + body: Bytes, timeout: Duration, ) -> Result { - let request = raw_leaf_outbound_request(authenticated, url, body, None)?; + let request = serialized_outbound_request(authenticated, url, body, None)?; tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http)) .await .map_err(litellm_http::Error::read_timeout)? @@ -1746,13 +1760,13 @@ mod tests { .await .unwrap(); assert_eq!(call.wire.url, format!("{base}{path}")); - let signed = raw_leaf_outbound_request( + let signed = serialized_outbound_request( Authenticated { headers: call.wire.headers, signer: call.signer, }, &call.wire.url, - &call.wire.body, + raw_leaf_body(&call.wire.body).unwrap(), None, ) .unwrap(); diff --git a/litellm/rust/crates/inference-messages/src/lib.rs b/litellm/rust/crates/inference-messages/src/lib.rs index cd312ee61b1..e65804ce108 100644 --- a/litellm/rust/crates/inference-messages/src/lib.rs +++ b/litellm/rust/crates/inference-messages/src/lib.rs @@ -28,9 +28,7 @@ pub use host_answer::HostAnswer; pub use litellm_inference::Error; pub use pre_request::PreRequestHooks; pub use safeguards::Safeguards; -pub use transport::{ - Delivered, Transport, TransportBody, TransportRequest, TransportResponse, UnencodableHeader, -}; +pub use transport::{Delivered, Transport, TransportRequest, TransportResponse, UnencodableHeader}; pub use types::{ MessagesBody, MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, messages_body, diff --git a/litellm/rust/crates/inference-messages/src/transport.rs b/litellm/rust/crates/inference-messages/src/transport.rs index 1353592447b..7d26cc81348 100644 --- a/litellm/rust/crates/inference-messages/src/transport.rs +++ b/litellm/rust/crates/inference-messages/src/transport.rs @@ -1,18 +1,12 @@ use bytes::Bytes; use futures_util::{future::BoxFuture, stream::BoxStream}; -use serde_json::Value; use crate::Error; -pub enum TransportBody { - Json(Value), - Signed(Bytes), -} - pub struct TransportRequest { pub url: String, pub headers: Vec<(String, String)>, - pub body: TransportBody, + pub body: Bytes, pub stream: bool, } diff --git a/litellm/rust/crates/inference-messages/tests/messages/host.rs b/litellm/rust/crates/inference-messages/tests/messages/host.rs index 7254830a2b0..8667616f279 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/host.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/host.rs @@ -1,10 +1,8 @@ use litellm_core_utils::decline::Decline; -use litellm_host::lifecycle::ExecutionEvent; use std::sync::Mutex; -use litellm_host::{ - interceptors::{ExecutionFacts, RawResponse, RequestContext, ResultSource, WireRequest}, - lifecycle::CallEvent, +use litellm_host::interceptors::{ + ExecutionFacts, RawResponse, RequestContext, ResultSource, WireRequest, }; use litellm_inference_messages::{MessagesBody, MessagesCallResponse, route::Messages}; use litellm_inference_testing::{RecordingSecrets, no_secrets}; @@ -47,17 +45,7 @@ impl RecordingHost { } fn raw_heads(&self) -> Vec { - self.events - .lock() - .unwrap() - .iter() - .filter_map(|event| match event { - CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { - Some(raw.clone()) - } - _ => None, - }) - .collect() + self.events.raw_heads() } fn raw_responses(&self) -> Vec { @@ -444,6 +432,38 @@ async fn a_stream_emits_its_head_without_a_body_before_any_chunk(call: MessagesC ); } +#[rstest] +#[tokio::test] +async fn the_raw_head_carries_the_exact_bytes_the_provider_received( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let response = if stream { + ResponseTemplate::new(200).set_body_raw( + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "text/event-stream", + ) + } else { + message_response() + }; + let upstream = upstream([response]).await; + let host = RecordingHost::passthrough(authenticated( + with_fields( + call, + json!({"stream": stream, "temperature": 1e-7, "system": "caf\u{e9} \u{1f600}"}), + ), + upstream.uri(), + )); + + let _ = run_through(&host).await.expect("messages call succeeds"); + + let [raw] = <[RawResponse; 1]>::try_from(host.raw_heads()) + .unwrap_or_else(|raws| panic!("expected one raw response, got {}", raws.len())); + let head = raw.head.expect("the head is forwarded"); + assert_eq!(head.sent_body, only_request(&upstream).await.body); + assert_eq!(head.streamed, stream); +} + /// Python logs `optional_params` after its requested-param gate, so a dropped param must /// not resurface in callbacks. #[rstest] diff --git a/litellm/rust/crates/inference-messages/tests/messages/retry.rs b/litellm/rust/crates/inference-messages/tests/messages/retry.rs index e48482dc8b0..7eac27b0b10 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/retry.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/retry.rs @@ -149,6 +149,30 @@ async fn a_rejected_thinking_replay_is_sent_once_more_without_thinking( assert_eq!(chunks.is_empty(), !stream); } +#[rstest] +#[tokio::test] +async fn the_raw_head_of_a_retried_call_carries_the_retried_bytes( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let success = if stream { + sse_response() + } else { + message_response() + }; + let upstream = upstream([rejection(400, INVALID_SIGNATURE), success]).await; + let host = RecordingCall::::new(replay(call, upstream.uri(), stream, enabled())); + + execute(&host).await.expect("the retry succeeds"); + + let [_, retried] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + let [raw] = <[_; 1]>::try_from(host.events.0.raw_heads()).unwrap(); + assert_eq!( + raw.head.expect("the head is forwarded").sent_body, + retried.body + ); +} + #[rstest] #[tokio::test] async fn a_second_rejection_is_the_error_the_caller_sees( diff --git a/litellm/rust/crates/inference-messages/tests/messages/transport.rs b/litellm/rust/crates/inference-messages/tests/messages/transport.rs index f561b09e55f..bb566e2ee1f 100644 --- a/litellm/rust/crates/inference-messages/tests/messages/transport.rs +++ b/litellm/rust/crates/inference-messages/tests/messages/transport.rs @@ -3,7 +3,7 @@ use std::collections::VecDeque; use bytes::Bytes; use futures_util::{StreamExt, TryStreamExt, future::BoxFuture}; use litellm_inference_messages::{ - Delivered, MessagesCallResponse, Transport, TransportBody, TransportRequest, TransportResponse, + Delivered, MessagesCallResponse, Transport, TransportRequest, TransportResponse, }; use litellm_inference_testing::RecordingSecrets; use rstest::rstest; @@ -28,6 +28,7 @@ struct Sent { url: String, headers: Vec<(String, String)>, body: Value, + raw_body: Bytes, stream: bool, } @@ -55,13 +56,11 @@ impl Transport for ScriptedTransport { &self, request: TransportRequest, ) -> BoxFuture<'static, Result> { - let TransportBody::Json(body) = request.body else { - panic!("an unsigned request hands the host its JSON body"); - }; self.sent.lock().unwrap().push(Sent { url: request.url, headers: request.headers, - body, + body: serde_json::from_slice(&request.body).expect("the host is handed JSON bytes"), + raw_body: request.body, stream: request.stream, }); let next = self @@ -113,10 +112,16 @@ async fn execute( transport: Arc, call: MessagesCall, ) -> Result<(MessagesCallResponse, Vec), Error> { - let host = RecordingCall::::new(call); + execute_recorded(transport, &RecordingCall::::new(call)).await +} + +async fn execute_recorded( + transport: Arc, + host: &RecordingCall, +) -> Result<(MessagesCallResponse, Vec), Error> { let route = messages_route(Arc::new(RecordingSecrets::empty())).with_transport(transport); match route - .execute(host.request()?, &host, Some(host.events.0.sender.clone())) + .execute(host.request()?, host, Some(host.events.0.sender.clone())) .await? { MessagesCallResponse::Stream { head, chunks } => Ok(( @@ -230,6 +235,38 @@ async fn a_recoverable_rejection_is_retried_through_the_host_transport(call: Mes assert!(second.body.get("thinking").is_none()); } +#[rstest] +#[tokio::test] +async fn the_raw_head_carries_the_bytes_the_host_transport_last_sent( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let frames: &'static [&'static str] = + &["event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"]; + let success = if stream { + Scripted::Stream(frames) + } else { + Scripted::Complete(message_body()) + }; + let transport = ScriptedTransport::new([Scripted::Rejected(400, INVALID_SIGNATURE), success]); + let fields = json!({"stream": stream, "temperature": 1e-7, "system": "caf\u{e9}"}); + let host = + RecordingCall::::new(hosted_call(with_fields(call, replayed_thinking()), fields)); + + execute_recorded(transport.clone(), &host) + .await + .expect("the retry succeeds"); + + let sent = transport.sent(); + let [_, retried] = sent.as_slice() else { + panic!("two requests"); + }; + let [raw] = <[_; 1]>::try_from(host.events.0.raw_heads()).unwrap(); + let head = raw.head.expect("the head is forwarded"); + assert_eq!(head.sent_body, retried.raw_body); + assert_eq!(head.streamed, stream); +} + #[rstest] #[case::final_rejection(vec![Scripted::Rejected(429, "slow down")])] #[case::spent_retry(vec![ diff --git a/litellm/rust/crates/inference-messages/tests/support/mod.rs b/litellm/rust/crates/inference-messages/tests/support/mod.rs index 9115b6e89bb..fc1545796ae 100644 --- a/litellm/rust/crates/inference-messages/tests/support/mod.rs +++ b/litellm/rust/crates/inference-messages/tests/support/mod.rs @@ -153,6 +153,19 @@ impl Observations { } Ok(events) } + + pub fn raw_heads(&self) -> Vec { + self.lock() + .unwrap() + .iter() + .filter_map(|event| match event { + litellm_host::lifecycle::CallEvent::Execution( + litellm_host::lifecycle::ExecutionEvent::ProviderResponseReceived { raw }, + ) => Some(raw.clone()), + _ => None, + }) + .collect() + } } impl litellm_host::lifecycle::CallObserver for CallEvents { diff --git a/litellm/rust/crates/inference/src/outbound.rs b/litellm/rust/crates/inference/src/outbound.rs index 87fae0bbe9e..51636cf550c 100644 --- a/litellm/rust/crates/inference/src/outbound.rs +++ b/litellm/rust/crates/inference/src/outbound.rs @@ -1,5 +1,7 @@ use std::time::Duration; +use bytes::Bytes; + use litellm_http::outbound::{json, json_bytes, signed}; use litellm_llms::base_llm::auth::Authenticated; use serde_json::Value; @@ -35,15 +37,20 @@ pub fn outbound_request( } } -pub fn raw_leaf_outbound_request( +pub fn raw_leaf_body(body: &Value) -> Result { + litellm_host::raw_json::to_vec(body) + .map(Bytes::from) + .map_err(litellm_http::Error::request_body) +} + +pub fn serialized_outbound_request( authenticated: Authenticated, url: &str, - body: &Value, + body: Bytes, timeout: Option, ) -> Result { let Authenticated { headers, signer } = authenticated; - let bytes = litellm_host::raw_json::to_vec(body).map_err(litellm_http::Error::request_body)?; - let request = json_bytes(url, &headers, bytes, timeout)?; + let request = json_bytes(url, &headers, body, timeout)?; match signer { None => Ok(request), Some(signer) => signed(request, &signer), diff --git a/litellm/rust/crates/llms/src/anthropic/common_utils.rs b/litellm/rust/crates/llms/src/anthropic/common_utils.rs index cadc5711474..17150547c59 100644 --- a/litellm/rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm/rust/crates/llms/src/anthropic/common_utils.rs @@ -362,9 +362,16 @@ fn retain_blocks(messages: Vec, keep: impl Fn(&ContentBlock) -> bool) - .collect() } +fn is_empty_unsigned_thinking_block(block: &ContentBlock) -> bool { + // Anthropic sends redacted thinking blocks, and the signature is used to + // match them to the appropriate thinking block on their servers. If there + // is a signature, we need to keep it. + is_empty_thinking_block(block) && block.signature.as_deref().is_none_or(str::is_empty) +} + pub fn strip_empty_content_blocks(messages: Vec) -> Vec { retain_blocks(messages, |block| { - !is_empty_text_block(block) && !is_empty_thinking_block(block) + !is_empty_text_block(block) && !is_empty_unsigned_thinking_block(block) }) } @@ -1001,11 +1008,11 @@ mod tests { ]}]), json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "x", "content": "y"}]}]) )] - #[case::empty_signed_thinking_beside_tool_use( + #[case::empty_unsigned_thinking_beside_tool_use( json!([ {"role": "user", "content": "weather?"}, {"role": "assistant", "content": [ - {"type": "thinking", "thinking": "", "signature": "sig_abc"}, + {"type": "thinking", "thinking": ""}, {"type": "tool_use", "id": "toolu_01A", "name": "get_weather", "input": {"city": "Paris"}} ]} ]), @@ -1016,6 +1023,13 @@ mod tests { ]} ]) )] + #[case::empty_thinking_with_empty_signature( + json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "", "signature": ""}, + {"type": "redacted_thinking", "data": "opaque"} + ]}]), + json!([{"role": "assistant", "content": [{"type": "redacted_thinking", "data": "opaque"}]}]) + )] #[case::whitespace_thinking_beside_real_and_redacted_thinking( json!([{"role": "assistant", "content": [ {"type": "thinking", "thinking": " \n "}, @@ -1053,6 +1067,11 @@ mod tests { #[case::textless_non_text_block(json!([{"role": "user", "content": [ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AA=="}} ]}]))] + #[case::empty_signed_thinking_beside_tool_use(json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "", "signature": "sig_abc"}, + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "tool_use", "id": "toolu_01A", "name": "get_weather", "input": {"city": "Paris"}} + ]}]))] #[case::encrypted_reasoning_left_for_the_responses_bridge(json!([{"role": "assistant", "content": [ {"type": "thinking", "thinking": "plan", "signature": tagged("gAAAA_1")}, {"type": "redacted_thinking", "data": tagged("gAAAA_2")}, diff --git a/litellm/rust/crates/llms/src/anthropic/messages/handler.rs b/litellm/rust/crates/llms/src/anthropic/messages/handler.rs index 46a16e7d546..89c31d61f90 100644 --- a/litellm/rust/crates/llms/src/anthropic/messages/handler.rs +++ b/litellm/rust/crates/llms/src/anthropic/messages/handler.rs @@ -192,6 +192,7 @@ mod tests { json!([ {"role": "user", "content": "run it"}, {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "", "signature": "sig"}, {"type": "tool_use", "id": "functions_Bash_0", "name": "Bash", "input": {}}, {"type": "server_tool_use", "id": "srv_1", "name": "web_search", "input": {"query": "q"}}, {"type": "text", "text": "Web search results:\n\nURL: u"} diff --git a/litellm/rust/crates/model-catalog-refresh/AGENTS.md b/litellm/rust/crates/model-catalog-refresh/AGENTS.md index 96228716abf..057cada924d 100644 --- a/litellm/rust/crates/model-catalog-refresh/AGENTS.md +++ b/litellm/rust/crates/model-catalog-refresh/AGENTS.md @@ -1,7 +1,8 @@ # Model catalog refresh -- `fetch` fetches one document over http(s) or `s3://` and never parses it +- `fetch` fetches one document over http(s), `s3://` or `gs://` and never parses it - Only 429, 500, 502, 503, 504 and transport errors are retried. S3 errors are final +- `gs://` reuses the http attempt with a Google application default credentials bearer token, so its statuses retry like http. A token that cannot be acquired in time retries too, since a new GKE pod's metadata server can refuse the first requests. With `STORAGE_EMULATOR_HOST` set it sends no credentials, like Google's SDKs. The token never reaches a reason or a log line - `Failed.retryable` means only the attempt limit ended the fetch - A server's Retry-After overrides the backoff, capped at `max_delay` - `Start::After(failed)` resumes the backoff schedule where another caller's first attempt left off diff --git a/litellm/rust/crates/model-catalog-refresh/Cargo.toml b/litellm/rust/crates/model-catalog-refresh/Cargo.toml index 76f4a9fb969..f43f04411d5 100644 --- a/litellm/rust/crates/model-catalog-refresh/Cargo.toml +++ b/litellm/rust/crates/model-catalog-refresh/Cargo.toml @@ -8,10 +8,13 @@ repository.workspace = true [dependencies] backon.workspace = true bytes.workspace = true +litellm-auth-gcp.workspace = true +litellm-auth-types.workspace = true litellm-error.workspace = true litellm-http.workspace = true litellm-s3.workspace = true litellm-tracing.workspace = true +percent-encoding.workspace = true reqwest.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["time"] } @@ -20,5 +23,6 @@ url.workspace = true [dev-dependencies] aws-credential-types = "1.3.0" litellm-http = { workspace = true, features = ["test-support"] } +litellm-secret-value.workspace = true rstest.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/litellm/rust/crates/model-catalog-refresh/src/attempt.rs b/litellm/rust/crates/model-catalog-refresh/src/attempt.rs index 45d80e36a18..190c5069b70 100644 --- a/litellm/rust/crates/model-catalog-refresh/src/attempt.rs +++ b/litellm/rust/crates/model-catalog-refresh/src/attempt.rs @@ -2,14 +2,24 @@ use std::time::Duration; use bytes::Bytes; +use litellm_auth_gcp::{CredentialSource, GcpAuth}; use litellm_http::Client; use litellm_s3::S3Uri; use litellm_s3::sdk::config::retry::RetryConfig; use litellm_s3::sdk::config::timeout::TimeoutConfig; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; use reqwest::header::{ETAG, HeaderMap, RETRY_AFTER}; use url::Url; -use crate::{Error, RETRYABLE_STATUSES, S3Access}; +use crate::{Error, GcsAccess, GcsUri, RETRYABLE_STATUSES, S3Access}; + +const GCS_ENDPOINT: &str = "https://storage.googleapis.com"; + +const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'_') + .remove(b'.') + .remove(b'~'); pub(crate) const MAX_BODY_BYTES: u64 = 100 * 1024 * 1024; @@ -46,14 +56,19 @@ impl Rejected { pub(crate) async fn http( client: &Client, url: Url, + bearer: Option<&str>, timeout: Duration, ) -> Result { + let request = match bearer { + Some(token) => client.get(url).bearer_auth(token), + None => client.get(url), + }; let transport = |error: litellm_http::Error| Rejected { cause: Cause::Transport(Error::from(error)), retryable: true, retry_after: None, }; - let mut response = tokio::time::timeout(timeout, client.get(url).send()) + let mut response = tokio::time::timeout(timeout, request.send()) .await .map_err(litellm_http::Error::read_timeout) .and_then(|sent| sent.map_err(litellm_http::Error::from)) @@ -135,6 +150,78 @@ pub(crate) async fn s3( fetched.await.map_err(Rejected::fatal) } +pub(crate) async fn gcs( + client: &Client, + gcs: &GcsAccess, + object: &GcsUri, + timeout: Duration, +) -> Result { + let started = tokio::time::Instant::now(); + let (endpoint, token) = match gcs { + GcsAccess::Environment => { + match emulator_endpoint(std::env::var("STORAGE_EMULATOR_HOST").ok()) { + Some(emulator) => (emulator, None), + None => ( + GCS_ENDPOINT.to_owned(), + Some( + bounded_token( + timeout, + GcpAuth::default().access_token(CredentialSource::application_default( + &|name| std::env::var(name).ok(), + )), + ) + .await?, + ), + ), + } + } + GcsAccess::Config { + endpoint, + token_source, + } => ( + endpoint.clone(), + Some(bounded_token(timeout, token_source.token()).await?), + ), + }; + let url = Url::parse(&format!( + "{}/storage/v1/b/{}/o/{}?alt=media", + endpoint.trim_end_matches('/'), + object.bucket, + utf8_percent_encode(&object.object, OBJECT_NAME_ENCODE_SET), + )) + .map_err(|error| Rejected::fatal(Error::from(error)))?; + http( + client, + url, + token.as_ref().map(|token| token.expose()), + timeout.saturating_sub(started.elapsed()), + ) + .await +} + +async fn bounded_token( + timeout: Duration, + token: impl Future>, +) -> Result { + let retryable = |error| Rejected { + cause: Cause::Transport(error), + retryable: true, + retry_after: None, + }; + tokio::time::timeout(timeout, token) + .await + .map_err(|elapsed| retryable(Error::gcs_token_timeout(elapsed)))? + .map_err(|error| retryable(Error::gcs_token(error))) +} + +fn emulator_endpoint(host: Option) -> Option { + match host { + Some(host) if host.contains("://") => Some(host), + Some(host) if !host.is_empty() => Some(format!("http://{host}")), + _ => None, + } +} + fn append(body: &mut Vec, chunk: &Bytes) -> Result<(), Error> { if (body.len() + chunk.len()) as u64 > MAX_BODY_BYTES { return Err(Error::body_too_large(MAX_BODY_BYTES)); diff --git a/litellm/rust/crates/model-catalog-refresh/src/error.rs b/litellm/rust/crates/model-catalog-refresh/src/error.rs index c79daeb4777..ea12eac2f03 100644 --- a/litellm/rust/crates/model-catalog-refresh/src/error.rs +++ b/litellm/rust/crates/model-catalog-refresh/src/error.rs @@ -25,6 +25,12 @@ enum Repr { S3Body(#[source] ByteStreamError), #[error("S3 GetObject timed out")] S3Timeout(#[source] Elapsed), + #[error("invalid GCS URL")] + GcsUri, + #[error("Google access token unavailable")] + GcsToken(#[source] litellm_auth_types::Error), + #[error("Google access token timed out")] + GcsTokenTimeout(#[source] Elapsed), #[error("response body is over the {limit} byte limit")] BodyTooLarge { limit: u64 }, } @@ -34,6 +40,18 @@ impl Error { Self(Box::new(Repr::UnsupportedScheme { scheme })) } + pub(crate) fn invalid_gcs_url() -> Self { + Self(Box::new(Repr::GcsUri)) + } + + pub(crate) fn gcs_token(cause: litellm_auth_types::Error) -> Self { + Self(Box::new(Repr::GcsToken(cause))) + } + + pub(crate) fn gcs_token_timeout(cause: Elapsed) -> Self { + Self(Box::new(Repr::GcsTokenTimeout(cause))) + } + pub(crate) fn body_too_large(limit: u64) -> Self { Self(Box::new(Repr::BodyTooLarge { limit })) } diff --git a/litellm/rust/crates/model-catalog-refresh/src/fetch.rs b/litellm/rust/crates/model-catalog-refresh/src/fetch.rs index bd58827b546..f1fd62d2f46 100644 --- a/litellm/rust/crates/model-catalog-refresh/src/fetch.rs +++ b/litellm/rust/crates/model-catalog-refresh/src/fetch.rs @@ -1,8 +1,10 @@ use std::borrow::Cow; +use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use backon::Retryable; +use litellm_auth_gcp::GcpTokenSource; use litellm_error::Report; use litellm_http::Client; use url::Url; @@ -20,6 +22,15 @@ pub enum S3Access { Config(Box), } +#[derive(Clone)] +pub enum GcsAccess { + Environment, + Config { + endpoint: String, + token_source: Arc, + }, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct Failed { pub reason: String, @@ -38,6 +49,7 @@ pub enum Start { pub async fn fetch( client: &Client, s3: &S3Access, + gcs: &GcsAccess, url: &str, policy: RetryPolicy, start: Start, @@ -58,8 +70,11 @@ pub async fn fetch( made.fetch_add(1, Ordering::Relaxed); async { match &source { - Source::Http(http) => attempt::http(client, http.clone(), policy.timeout).await, + Source::Http(http) => { + attempt::http(client, http.clone(), None, policy.timeout).await + } Source::S3(object) => attempt::s3(client, s3, object, policy.timeout).await, + Source::Gcs(object) => attempt::gcs(client, gcs, object, policy.timeout).await, } } }; diff --git a/litellm/rust/crates/model-catalog-refresh/src/source.rs b/litellm/rust/crates/model-catalog-refresh/src/source.rs index 28849ba2841..0b26db0620d 100644 --- a/litellm/rust/crates/model-catalog-refresh/src/source.rs +++ b/litellm/rust/crates/model-catalog-refresh/src/source.rs @@ -8,6 +8,34 @@ use crate::Error; pub enum Source { Http(Url), S3(S3Uri), + Gcs(GcsUri), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GcsUri { + pub bucket: String, + pub object: String, +} + +impl TryFrom<&str> for GcsUri { + type Error = Error; + + fn try_from(uri: &str) -> Result { + let rest = uri.split_once("://").map_or(uri, |(_, rest)| rest); + let (bucket, path) = rest.split_at(rest.find(['/', '?', '#']).unwrap_or(rest.len())); + let object = path + .split(['?', '#']) + .next() + .unwrap_or_default() + .trim_start_matches('/'); + if bucket.is_empty() || object.is_empty() { + return Err(Error::invalid_gcs_url()); + } + Ok(Self { + bucket: bucket.to_owned(), + object: object.to_owned(), + }) + } } impl Source { @@ -16,6 +44,7 @@ impl Source { match parsed.scheme() { "http" | "https" => Ok(Self::Http(parsed)), "s3" => Ok(Self::S3(S3Uri::try_from(url)?)), + "gs" => Ok(Self::Gcs(GcsUri::try_from(url)?)), scheme => Err(Error::unsupported_scheme(scheme.to_owned())), } } @@ -48,8 +77,30 @@ mod tests { ); } + #[rstest] + #[case::object("gs://bucket/cost.json", "bucket", "cost.json")] + #[case::nested_object("gs://bucket/prices/cost.json", "bucket", "prices/cost.json")] + #[case::space_kept_raw("gs://bucket/my prices.json", "bucket", "my prices.json")] + #[case::query_dropped("gs://bucket/cost.json?generation=1", "bucket", "cost.json")] + fn gs_urls_name_a_bucket_and_object( + #[case] url: &str, + #[case] bucket: &str, + #[case] object: &str, + ) { + assert_eq!( + Source::parse(url).unwrap(), + Source::Gcs(GcsUri { + bucket: bucket.into(), + object: object.into(), + }) + ); + } + #[rstest] #[case::s3_without_a_key("s3://bucket/")] + #[case::gs_without_an_object("gs://bucket/")] + #[case::gs_without_a_path("gs://bucket")] + #[case::gs_without_a_bucket("gs:///cost.json")] #[case::unsupported("ftp://example.com/cost.json")] #[case::file("file:///tmp/cost.json")] #[case::not_a_url("not a url")] diff --git a/litellm/rust/crates/model-catalog-refresh/tests/fetch.rs b/litellm/rust/crates/model-catalog-refresh/tests/fetch.rs index 2a4ec996bd4..4509ee6f7d5 100644 --- a/litellm/rust/crates/model-catalog-refresh/tests/fetch.rs +++ b/litellm/rust/crates/model-catalog-refresh/tests/fetch.rs @@ -1,13 +1,19 @@ use std::io::{BufRead, BufReader, Write}; use std::net::TcpListener; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; use aws_credential_types::Credentials; use aws_credential_types::provider::error::CredentialsError; use aws_credential_types::provider::{ProvideCredentials, future}; +use litellm_auth_gcp::{GcpAuthFuture, GcpTokenSource}; +use litellm_auth_types::{Error, ErrorDetail}; use litellm_http::Client; -use litellm_model_catalog_refresh::{Failed, Fetched, RetryPolicy, S3Access, Start, fetch}; +use litellm_model_catalog_refresh::{ + Failed, Fetched, GcsAccess, RetryPolicy, S3Access, Start, fetch, +}; +use litellm_secret_value::SecretValue; use rstest::rstest; const POLICY: RetryPolicy = RetryPolicy { @@ -140,6 +146,7 @@ async fn fetched(url: &str, policy: RetryPolicy, start: Start) -> Result); + +impl GcpTokenSource for Token { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async { Ok("model-cost-project".into()) }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + self.0.map(SecretValue::new).map_err(|subject| { + Error::CredentialAcquisition(ErrorDetail::Unresolved { subject }) + }) + }) + } +} + +struct StalledToken; + +impl GcpTokenSource for StalledToken { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(std::future::pending()) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(std::future::pending()) + } +} + +async fn fetched_from_gcs( + url: &str, + upstream: &Upstream, + token: impl GcpTokenSource + 'static, +) -> Result { + fetch( + &Client::for_test(), + &S3Access::Environment { region: None }, + &GcsAccess::Config { + endpoint: format!("{}/", upstream.base), + token_source: Arc::new(token), + }, + url, + POLICY, + Start::Fresh, + ) + .await +} + +#[rstest] +#[tokio::test] +async fn a_gcs_fetch_sends_the_bearer_token_to_the_object_media_url() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + assert_eq!( + fetched_from_gcs( + "gs://prices/maps/cost.json", + &upstream, + Token(Ok(GCS_TOKEN)) + ) + .await, + Ok(Fetched { + body: b"{}".to_vec(), + etag: Some("\"v1\"".into()), + }) + ); + let heads = upstream.heads(); + let [head] = heads.as_slice() else { + panic!("expected one request, got {heads:?}"); + }; + assert!( + head.starts_with("GET /storage/v1/b/prices/o/maps%2Fcost.json?alt=media HTTP/1.1"), + "{head}" + ); + assert_eq!( + header(head, "authorization"), + Some(format!("Bearer {GCS_TOKEN}").as_str()) + ); +} + +#[rstest] +#[case::unavailable(UNAVAILABLE, " (after 3 attempts)", true, 3)] +#[case::rate_limited( + "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + " (after 3 attempts)", + true, + 3 +)] +#[case::unauthorized( + "HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "", + false, + 1 +)] +#[case::denied( + "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "", + false, + 1 +)] +#[case::missing(NOT_FOUND, "", false, 1)] +#[tokio::test] +async fn a_gcs_status_retries_only_when_an_http_status_would( + #[case] response: &'static str, + #[case] suffix: &str, + #[case] retryable: bool, + #[case] requests: usize, +) { + let upstream = Upstream::serve(vec![Answer::Raw(response)]); + let url = "gs://prices/cost.json"; + let failed = fetched_from_gcs(url, &upstream, Token(Ok(GCS_TOKEN))) + .await + .unwrap_err(); + let status = &response[9..12]; + assert_eq!(failed.reason, format!("HTTP {status} from {url}{suffix}")); + assert_eq!(failed.retryable, retryable); + assert_eq!(upstream.heads().len(), requests); +} + +#[rstest] +#[tokio::test] +async fn a_gcs_fetch_without_a_token_retries_without_a_request() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + let url = "gs://prices/cost.json"; + let failed = fetched_from_gcs(url, &upstream, Token(Err("Google access token"))) + .await + .unwrap_err(); + assert!( + failed.reason.starts_with(&format!( + "error fetching {url}: Google access token unavailable" + )), + "{}", + failed.reason + ); + assert!(failed.retryable); + assert!(upstream.heads().is_empty()); +} + +struct TokenAfterOneRefusal(AtomicU32); + +impl GcpTokenSource for TokenAfterOneRefusal { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async { Ok("model-cost-project".into()) }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + match self.0.fetch_add(1, Ordering::Relaxed) { + 0 => Err(Error::CredentialAcquisition(ErrorDetail::Unresolved { + subject: "Google access token", + })), + _ => Ok(SecretValue::new(GCS_TOKEN)), + } + }) + } +} + +#[rstest] +#[tokio::test] +async fn a_gcs_token_refused_once_is_retried_into_a_fetch() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + let fetched = fetched_from_gcs( + "gs://prices/cost.json", + &upstream, + TokenAfterOneRefusal(AtomicU32::new(0)), + ) + .await + .unwrap(); + assert_eq!(fetched.body, b"{}"); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_gcs_token_that_never_arrives_fails_within_the_attempt_timeout() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + let url = "gs://prices/cost.json"; + let once = RetryPolicy { + max_attempts: 1, + ..POLICY + }; + let failed = tokio::time::timeout( + once.timeout * 2, + fetch( + &Client::for_test(), + &S3Access::Environment { region: None }, + &GcsAccess::Config { + endpoint: format!("{}/", upstream.base), + token_source: Arc::new(StalledToken), + }, + url, + once, + Start::Fresh, + ), + ) + .await + .expect("the token wait outlived the attempt timeout") + .unwrap_err(); + assert!( + failed.reason.starts_with(&format!( + "error fetching {url}: Google access token timed out" + )), + "{}", + failed.reason + ); + assert!(failed.retryable); + assert!(upstream.heads().is_empty()); +} + +struct SlowToken(Duration); + +impl GcpTokenSource for SlowToken { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async { Ok("model-cost-project".into()) }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + tokio::time::sleep(self.0).await; + Ok(SecretValue::new(GCS_TOKEN)) + }) + } +} + +#[rstest] +#[tokio::test] +async fn a_gcs_attempt_spends_one_timeout_across_the_token_and_the_request() { + let upstream = Upstream::serve(vec![Answer::Stall]); + let once = RetryPolicy { + max_attempts: 1, + timeout: Duration::from_secs(2), + ..POLICY + }; + let started = std::time::Instant::now(); + let failed = fetch( + &Client::for_test(), + &S3Access::Environment { region: None }, + &GcsAccess::Config { + endpoint: format!("{}/", upstream.base), + token_source: Arc::new(SlowToken(once.timeout / 2)), + }, + "gs://prices/cost.json", + once, + Start::Fresh, + ) + .await + .unwrap_err(); + assert!(failed.retryable, "{}", failed.reason); + assert!( + started.elapsed() < once.timeout + once.timeout / 4, + "the attempt took {:?}, over its {:?} timeout", + started.elapsed(), + once.timeout + ); +} + const DECLARED_OVERSIZED: &str = "HTTP/1.1 200 OK\r\nContent-Length: 104857601\r\nConnection: close\r\n\r\n"; diff --git a/litellm/rust/crates/model-catalog/AGENTS.md b/litellm/rust/crates/model-catalog/AGENTS.md index d74c206813f..7f8d872940e 100644 --- a/litellm/rust/crates/model-catalog/AGENTS.md +++ b/litellm/rust/crates/model-catalog/AGENTS.md @@ -42,11 +42,11 @@ Every price in a row lowers to one `Price { meter, channel, when }` and its `Uni `lower.rs` is the only code that knows both the wire types and `litellm-cost`. It lowers an `EntryWire` into `Prices`, `Adjustments` and `Model` and checks the rules that span keys, such as one price per meter and condition, contiguous tiers, no suffix inside a block, and off-peak needing hours and prices. Lowering only goes one way. Nothing turns `Prices` back into JSON, since the raw fields are kept -`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations` +`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A document that starts with a UTF-8 or UTF-16 byte order mark is read as that text. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. An `alias_of` row that carries other fields is still served as its target until `without_rejected` drops it. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations` `store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md) -`cost_map.rs` holds a host's inputs, the installed document and the `register_model` overlay in `register.rs`. It serves the document in the key order and alias sharing Python's `litellm.model_cost` has, without the overlay, since where a registration lands is a lookup in the published table. It never holds the live table, which belongs to the host +`cost_map.rs` holds a host's inputs, the installed document and the `register_model` overlay in `register.rs`. It serves the document in the key order and alias sharing Python's `litellm.model_cost` has, without the overlay, since where a registration lands is a lookup in the published table. It never holds the live table, which belongs to the host. `CostMap::read` refuses a document with a top-level value that is not a row, since a host table has nowhere to serve one - `CostMap` is immutable behind an `Arc`. A new document is a new `CostMap` - `Registrations` is generic over the host's value type, because Python registrations can hold `datetime` and other values JSON cannot. The host keeps its own objects (`Py` in python-bridge) and does the merging diff --git a/litellm/rust/crates/model-catalog/Cargo.toml b/litellm/rust/crates/model-catalog/Cargo.toml index 8cd8ee5760b..d85bb799924 100644 --- a/litellm/rust/crates/model-catalog/Cargo.toml +++ b/litellm/rust/crates/model-catalog/Cargo.toml @@ -13,6 +13,7 @@ litellm-cost = { workspace = true, features = ["serde"] } litellm-error.workspace = true litellm-tracing.workspace = true +encoding_rs = "0.8.35" indexmap = { version = "2.14.0", features = ["serde"] } jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } macro_rules_attribute.workspace = true diff --git a/litellm/rust/crates/model-catalog/src/catalog.rs b/litellm/rust/crates/model-catalog/src/catalog.rs index 6c11a39b0eb..05725d06863 100644 --- a/litellm/rust/crates/model-catalog/src/catalog.rs +++ b/litellm/rust/crates/model-catalog/src/catalog.rs @@ -1,7 +1,8 @@ use std::borrow::Cow; use std::collections::HashSet; -use indexmap::IndexMap; +use encoding_rs::Encoding; +use indexmap::{IndexMap, IndexSet}; use serde::Deserialize; use serde_json::Value; @@ -69,6 +70,8 @@ pub struct Catalog { struct CatalogRows { entries: IndexMap, aliases: IndexMap, + /// The aliases an `alias_of` row declared, whose own row can be rejected + pointers: IndexSet, } impl CatalogRows { @@ -141,8 +144,11 @@ impl ModelEntry { impl Catalog { pub fn parse(body: &[u8]) -> Result, Error> { - let document = serde_json::from_slice::(body) - .map_err(|error| Error::new("", Repr::Json(error)))?; + let document = match text_after_byte_order_mark(body) { + Some(text) => serde_json::from_str::(&text), + None => serde_json::from_slice::(body), + } + .map_err(|error| Error::new("", Repr::Json(error)))?; let mut problems = Vec::new(); let mut sections = Vec::new(); @@ -165,16 +171,13 @@ impl Catalog { } }; let row = read_row(&fields); - let readable = row.problems.is_empty(); problems.extend( row.problems .into_iter() .map(|problem| problem.prepend(&name)), ); if let Some(target) = row.alias_of { - if readable { - pointer_rows.push((name.clone(), target)); - } + pointer_rows.push((name.clone(), target)); sections.push(Section { index, key: name, @@ -215,6 +218,7 @@ impl Catalog { } } } + let mut pointers = IndexSet::new(); for (alias, target) in pointer_rows { if !entries.contains_key(&target) { problems.push(Error::from_key(&alias, Repr::AliasTargetMissing { target })); @@ -227,6 +231,7 @@ impl Catalog { }, )); } else { + pointers.insert(alias.clone()); aliases.insert(alias, target); } } @@ -243,7 +248,11 @@ impl Catalog { }); Ok(Loaded { value: Self { - store: Store::new(CatalogRows { entries, aliases }), + store: Store::new(CatalogRows { + entries, + aliases, + pointers, + }), sections, fallback_generalizations, }, @@ -378,8 +387,16 @@ impl Catalog { .collect(); let mut catalog = self; catalog.store.edit(|rows| { - rows.entries.retain(|name, _| !rejected.contains(name)); - rows.aliases.retain(|_, owner| !rejected.contains(owner)); + let CatalogRows { + entries, + aliases, + pointers, + } = rows; + entries.retain(|name, _| !rejected.contains(name)); + aliases.retain(|alias, owner| { + !rejected.contains(owner) && !(pointers.contains(alias) && rejected.contains(alias)) + }); + pointers.retain(|alias| aliases.contains_key(alias)); }); catalog .sections @@ -504,6 +521,11 @@ fn read_row(fields: &Fields) -> RowRead { } } +fn text_after_byte_order_mark(body: &[u8]) -> Option> { + let (encoding, mark) = Encoding::for_bom(body)?; + encoding.decode_without_bom_handling_and_without_replacement(&body[mark..]) +} + fn is_section(key: &str) -> bool { key == SAMPLE_SPEC || key == FALLBACK_GENERALIZATIONS } diff --git a/litellm/rust/crates/model-catalog/src/cost_map.rs b/litellm/rust/crates/model-catalog/src/cost_map.rs index bec417acae7..91fb7d9e222 100644 --- a/litellm/rust/crates/model-catalog/src/cost_map.rs +++ b/litellm/rust/crates/model-catalog/src/cost_map.rs @@ -6,7 +6,7 @@ use sha1::{Digest, Sha1}; use crate::bundled::BUNDLED_REGISTRY; use crate::catalog::{Catalog, FALLBACK_GENERALIZATIONS, IntegrityLimits}; -use crate::error::Error; +use crate::error::{Error, ErrorKind, Repr}; /// Which document is installed: the `git_blob_id` of its bytes and the ETag it was served with #[derive(Clone, Debug, PartialEq, Eq)] @@ -62,10 +62,19 @@ impl CostMap { BUNDLED.clone() } - /// Reads a fetched document. One that does not parse, or that has fewer models than `limits` allow against the - /// bundled one, is refused. Every alias problem it has is logged + /// Reads a fetched document. One that does not parse, that has a top-level value a host table cannot serve as a + /// row, or that has fewer models than `limits` allow against the bundled one, is refused. Every alias problem it + /// has is logged pub fn read(body: &[u8], etag: Option, limits: IntegrityLimits) -> Result { let loaded = Catalog::parse(body)?; + if let Some(key) = loaded + .errors + .iter() + .find(|error| error.kind() == ErrorKind::NotAnObject) + .and_then(Error::key) + { + return Err(Error::from_key(&key, Repr::UnservableValue)); + } loaded .value .check_size(BUNDLED.base.catalog.model_count(), limits)?; diff --git a/litellm/rust/crates/model-catalog/src/error.rs b/litellm/rust/crates/model-catalog/src/error.rs index a4b54b5a551..47d0ca221c6 100644 --- a/litellm/rust/crates/model-catalog/src/error.rs +++ b/litellm/rust/crates/model-catalog/src/error.rs @@ -49,6 +49,9 @@ pub(crate) enum Repr { #[error("must be a JSON object")] #[strum_discriminants(strum(serialize = "not_an_object"))] NotAnObject, + #[error("a host table cannot serve a top-level value that is not a JSON object")] + #[strum_discriminants(strum(serialize = "unservable_value"))] + UnservableValue, #[error("invalid value")] #[strum_discriminants(strum(serialize = "invalid"))] Invalid(#[source] serde_json::Error), @@ -138,9 +141,11 @@ impl Error { pub fn rejects(&self) -> Rejects { match self.repr() { - Repr::Json(_) | Repr::Empty | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } => { - Rejects::Document - } + Repr::Json(_) + | Repr::Empty + | Repr::BelowMinimum { .. } + | Repr::Shrunk { .. } + | Repr::UnservableValue => Rejects::Document, Repr::NotAnObject | Repr::Invalid(_) | Repr::UnknownBlockKey @@ -174,6 +179,7 @@ impl Error { Repr::Empty | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } => true, Repr::Json(_) | Repr::NotAnObject + | Repr::UnservableValue | Repr::Invalid(_) | Repr::UnknownBlockKey | Repr::UnknownTimezone { .. } @@ -213,6 +219,7 @@ impl Error { | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } | Repr::NotAnObject + | Repr::UnservableValue | Repr::UnknownBlockKey | Repr::UnknownTimezone { .. } | Repr::DuplicatePrice { .. } @@ -397,6 +404,7 @@ mod tests { #[case::below_minimum(Repr::BelowMinimum { actual: 1, minimum: 2 }, Rejects::Document)] #[case::shrunk(Repr::Shrunk { actual: 1, reference: 3, ratio: 0.5 }, Rejects::Document)] #[case::not_an_object(Repr::NotAnObject, Rejects::Row)] + #[case::unservable_value(Repr::UnservableValue, Rejects::Document)] #[case::invalid(Repr::Invalid(serde_json::from_str::("x").unwrap_err()), Rejects::Row)] #[case::unknown_block_key(Repr::UnknownBlockKey, Rejects::Row)] #[case::missing_provider(Repr::MissingProvider, Rejects::Nothing)] diff --git a/litellm/rust/crates/model-catalog/src/model_info.rs b/litellm/rust/crates/model-catalog/src/model_info.rs index fcd0c027323..d0fd74e2cb7 100644 --- a/litellm/rust/crates/model-catalog/src/model_info.rs +++ b/litellm/rust/crates/model-catalog/src/model_info.rs @@ -115,13 +115,16 @@ pub struct ModelInfoFields { pub output_cost_per_token_above_512k_tokens: Option, pub output_cost_per_second: Option, pub output_cost_per_second_1080p: Option, + pub output_cost_per_video_clip: Option>>, pub output_cost_per_second_480p: Option, pub output_cost_per_second_720p: Option, pub output_cost_per_second_768p: Option, pub output_cost_per_second_2k: Option, pub output_cost_per_second_4k: Option, + pub output_cost_per_video: Option, pub output_cost_per_video_per_second: Option, pub output_cost_per_image: Option, + pub output_cost_per_billable_unit: Option, pub output_cost_per_pixel: Option, pub output_cost_per_image_token: Option, pub output_cost_per_image_token_batches: Option, diff --git a/litellm/rust/crates/model-catalog/src/schema.rs b/litellm/rust/crates/model-catalog/src/schema.rs index 60c755fe94f..bd25b6bcd79 100644 --- a/litellm/rust/crates/model-catalog/src/schema.rs +++ b/litellm/rust/crates/model-catalog/src/schema.rs @@ -19,6 +19,7 @@ struct ModelEntry { #[derive(JsonSchema)] #[allow(dead_code)] struct PriceBlocks { + output_cost_per_video_clip: Option>>, /// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits). guardrail_cost_per_unit: Option>, /// Rates that replace the same-named base fields while the request falls inside the stated UTC windows. @@ -64,7 +65,7 @@ struct RegistryDocument { pub(crate) const FLAT_KEY_PATTERN: &str = concat!( r"^(", r"input_cost_per_((audio_|image_|video_)?token|character|image|pixel|query|request|second|text_message|audio_per_second|video_per_second(_above_\d+s_interval)?)", - r"|output_cost_per_((audio_|image_|video_|reasoning_)?token|character|image(_\d+(\.\d+)?K?)?|pixel|second(_\d+[pk])?|video_per_second)", + r"|output_cost_per_((audio_|image_|video_|reasoning_)?token|character|image(_\d+(\.\d+)?K?)?|pixel|second(_\d+[pk])?|video|video_per_second|billable_unit)", r"|cache_read_input_(audio_|image_|video_)?token_cost", r"|explicit_cache_read_input_token_cost", r"|cache_creation_input_(audio_|image_|video_)?token_cost(_above_1hr|_above_12hr)?", @@ -202,6 +203,10 @@ mod tests { #[case::feature_batch("annotation_cost_per_page_batches")] #[case::video_interval("input_cost_per_video_per_second_above_8s_interval")] #[case::explicit_cache_read("explicit_cache_read_input_token_cost")] + #[case::billable_unit("output_cost_per_billable_unit")] + #[case::billable_unit_is_output_only("input_cost_per_billable_unit")] + #[case::video("output_cost_per_video")] + #[case::video_is_output_only("input_cost_per_video")] #[case::wall_clock("cost_per_second")] #[case::unused_meter("cost_per_token")] #[case::other_currency("input_dbu_cost_per_token")] diff --git a/litellm/rust/crates/model-catalog/src/wire/entry.rs b/litellm/rust/crates/model-catalog/src/wire/entry.rs index d0bdb394720..c8c704c2d1a 100644 --- a/litellm/rust/crates/model-catalog/src/wire/entry.rs +++ b/litellm/rust/crates/model-catalog/src/wire/entry.rs @@ -21,6 +21,7 @@ pub struct EntryWire { pub off_peak: Option, pub search_context: Option, pub guardrail: Option>, + pub output_cost_per_video_clip: Option>>, pub regional_endpoint_uplift: Option, pub residency_uplift_eu: Option, pub residency_uplift_us: Option, @@ -49,6 +50,9 @@ impl EntryWire { "off_peak_pricing" => self.off_peak = reader.off_peak(value), "search_context_cost_per_query" => self.search_context = reader.value(path(), value), "guardrail_cost_per_unit" => self.guardrail = reader.value(path(), value), + "output_cost_per_video_clip" => { + self.output_cost_per_video_clip = reader.value(path(), value) + } "regional_endpoint_uplift_multiplier" => { self.regional_endpoint_uplift = reader.value(path(), value) } @@ -108,6 +112,17 @@ mod tests { .collect() } + #[rstest] + #[case::clip_table(json!({"5s": {"720p": 0.15}}), false)] + #[case::empty(json!({}), false)] + #[case::invalid_price(json!({"5s": {"720p": "invalid"}}), true)] + fn video_clip_table_is_a_known_price_block(#[case] prices: Value, #[case] invalid: bool) { + let wire = read(json!({"output_cost_per_video_clip": prices})); + assert!(wire.unknown.is_empty()); + assert_eq!(!wire.problems.is_empty(), invalid); + assert_eq!(wire.output_cost_per_video_clip.is_some(), !invalid); + } + #[rstest] fn every_key_lands_in_exactly_one_place() { let wire = read(json!({ diff --git a/litellm/rust/crates/model-catalog/src/wire/flat_key.rs b/litellm/rust/crates/model-catalog/src/wire/flat_key.rs index 906d917b487..b3a2c39acba 100644 --- a/litellm/rust/crates/model-catalog/src/wire/flat_key.rs +++ b/litellm/rust/crates/model-catalog/src/wire/flat_key.rs @@ -210,6 +210,8 @@ fn output_unit(unit: &str) -> Option { "pixel" => Some(OutputUnit::Pixel), "second" => Some(OutputUnit::Second(None)), "video_per_second" => Some(OutputUnit::VideoSecond), + "video" => Some(OutputUnit::Video), + "billable_unit" => Some(OutputUnit::BillableUnit), _ => None, } } @@ -285,6 +287,10 @@ mod tests { None ) )] + #[case::billable_unit( + "output_cost_per_billable_unit", + key(Meter::Output(OutputUnit::BillableUnit), Band::Base, None) + )] #[case::image_token_is_a_token( "output_cost_per_image_token", key(Meter::Output(OutputUnit::Token(TokenKind::Image)), Band::Base, None) @@ -309,6 +315,10 @@ mod tests { Some(TierSuffix::Batches) ) )] + #[case::video( + "output_cost_per_video", + key(Meter::Output(OutputUnit::Video), Band::Base, None) + )] #[case::wall_clock("cost_per_second", key(Meter::WallClockSecond, Band::Base, None))] #[case::browser_open( "browser_open_cost_per_request", diff --git a/litellm/rust/crates/model-catalog/tests/catalog.rs b/litellm/rust/crates/model-catalog/tests/catalog.rs index 679c67412c8..9ce8fcccd85 100644 --- a/litellm/rust/crates/model-catalog/tests/catalog.rs +++ b/litellm/rust/crates/model-catalog/tests/catalog.rs @@ -26,6 +26,12 @@ fn catalog(document: Value) -> Catalog { loaded(document).value } +fn utf16(text: &[u8], mark: [u8; 2], bytes: fn(u16) -> [u8; 2]) -> Vec { + mark.into_iter() + .chain(text.iter().flat_map(|byte| bytes(u16::from(*byte)))) + .collect() +} + fn written(catalog: &Catalog) -> String { let document: Map = catalog .written() @@ -452,16 +458,39 @@ fn a_dangling_alias_of_row_is_rejected_and_dropped_from_the_written_document() { } #[rstest] -fn an_alias_of_row_with_other_fields_is_never_served() { +fn an_alias_of_row_with_other_fields_is_served_as_its_target_until_its_row_is_rejected() { let parsed = loaded(json!({ - "Real": {"litellm_provider": "t"}, - "real-latest": {"alias_of": "Real", "mode": "chat"} + "Real": {"litellm_provider": "t", "input_cost_per_token": 1e-6}, + "real-latest": {"alias_of": "Real", "input_cost_per_token": 9e-6} })); - assert!(parsed.value.lookup("real-latest").is_none()); + let found = parsed.value.lookup("real-latest").unwrap(); + assert_eq!(found.canonical_key, "Real"); + assert_eq!( + found.entry.field("input_cost_per_token"), + Some(&json!(1e-6)) + ); let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); assert_eq!(removed.len(), 1); assert!(catalog.lookup("real-latest").is_none()); - assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#); + assert_eq!( + written(&catalog), + r#"{"Real":{"litellm_provider":"t","input_cost_per_token":1e-6}}"# + ); +} + +#[rstest] +#[case::utf8_with_a_byte_order_mark([b"\xEF\xBB\xBF".as_slice(), br#"{"m":{"litellm_provider":"p"}}"#].concat())] +#[case::utf16_little_endian(utf16(br#"{"m":{"litellm_provider":"p"}}"#, [0xFF, 0xFE], u16::to_le_bytes))] +#[case::utf16_big_endian(utf16(br#"{"m":{"litellm_provider":"p"}}"#, [0xFE, 0xFF], u16::to_be_bytes))] +fn a_document_that_starts_with_a_byte_order_mark_parses_as_its_text(#[case] body: Vec) { + let catalog = Catalog::parse(&body).unwrap().value; + assert_eq!(catalog.model_names().collect::>(), ["m"]); +} + +#[rstest] +fn a_byte_order_mark_before_bytes_that_are_not_its_encoding_is_invalid_json() { + let error = Catalog::parse(&[0xFF, 0xFE, b'{', b'}', b'x']).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Json); } #[rstest] diff --git a/litellm/rust/crates/model-catalog/tests/cost_map.rs b/litellm/rust/crates/model-catalog/tests/cost_map.rs index 04a04170df6..7609b754595 100644 --- a/litellm/rust/crates/model-catalog/tests/cost_map.rs +++ b/litellm/rust/crates/model-catalog/tests/cost_map.rs @@ -1,5 +1,5 @@ use litellm_model_catalog::{ - BUNDLED_REGISTRY, CostMap, ErrorKind, IntegrityLimits, Provenance, git_blob_id, + BUNDLED_REGISTRY, CostMap, ErrorKind, IntegrityLimits, Provenance, Rejects, git_blob_id, }; use rstest::rstest; use serde_json::json; @@ -20,6 +20,42 @@ fn a_refused_document_says_whether_it_failed_the_integrity_limits( assert_eq!((error.is_integrity(), error.kind()), (integrity, kind)); } +#[rstest] +#[case::string(json!("mirrored 2026-11-30"))] +#[case::null(json!(null))] +#[case::list(json!([1, 2]))] +#[case::number(json!(5))] +fn a_document_with_a_top_level_value_that_is_not_a_row_is_refused( + #[case] value: serde_json::Value, +) { + let body = json!({"m": {"litellm_provider": "p"}, "_comment": value}).to_string(); + let error = CostMap::read(body.as_bytes(), None, any_size()).unwrap_err(); + assert_eq!( + (error.kind(), error.key().as_deref(), error.is_integrity()), + (ErrorKind::UnservableValue, Some("_comment"), false) + ); + assert_eq!(error.rejects(), Rejects::Document); +} + +#[rstest] +#[case::sample_spec("sample_spec")] +#[case::fallback_generalizations("fallback_generalizations")] +fn a_reserved_top_level_key_may_hold_any_value(#[case] key: &str) { + let body = json!({"m": {"litellm_provider": "p"}, key: "not a row"}).to_string(); + assert!(CostMap::read(body.as_bytes(), None, any_size()).is_ok()); +} + +#[rstest] +fn provenance_of_a_document_with_a_byte_order_mark_is_its_served_bytes() { + let body = [ + b"\xEF\xBB\xBF".as_slice(), + br#"{"m":{"litellm_provider":"p"}}"#, + ] + .concat(); + let cost_map = CostMap::read(&body, None, any_size()).unwrap(); + assert_eq!(cost_map.provenance().revision, git_blob_id(&body)); +} + #[rstest] fn provenance_is_the_read_bytes_and_etag() { let body = br#"{"m":{"litellm_provider":"p"}}"#; diff --git a/litellm/rust/crates/python-bridge/benches/serialization.rs b/litellm/rust/crates/python-bridge/benches/serialization.rs index d398d9fdbfc..e2a0d18e9d3 100644 --- a/litellm/rust/crates/python-bridge/benches/serialization.rs +++ b/litellm/rust/crates/python-bridge/benches/serialization.rs @@ -1,8 +1,11 @@ use std::{hint::black_box, time::Duration}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use litellm_host_python::{from_py, to_py}; -use pyo3::{prelude::*, types::PyDict}; +use litellm_host_python::{from_py, json_to_py, to_py}; +use pyo3::{ + prelude::*, + types::{PyDict, PyString}, +}; use serde_json::{Value, json}; const PAYLOAD_SIZES: &[(&str, usize)] = &[ @@ -89,12 +92,81 @@ fn bridge_serialization(c: &mut Criterion) { }); } +const MESSAGES_INPUT_TEXTS: &[(&str, &str)] = &[ + ( + "ascii", + "Here is some ordinary English text about the code.\n", + ), + ( + "mixed", + "Here is some ordinary English text about the code. Done \u{2705} caf\u{e9} \u{1f600}\n", + ), + ( + "unicode", + "\u{65e5}\u{672c}\u{8a9e}\u{306e}\u{30c6}\u{30ad}\u{30b9}\u{30c8} \u{1f600}\u{1f680} caf\u{e9}\n", + ), +]; + +fn messages_body(text: &str) -> Value { + let turn = text.repeat(6); + let messages: Vec = (0..) + .map(|index| { + json!([ + {"role": "user", "content": [{"type": "text", "text": turn}]}, + {"role": "assistant", "content": [ + {"type": "text", "text": turn}, + {"type": "tool_use", "id": format!("toolu_{index:06}"), "name": "bash", "input": {"command": "ls"}} + ]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": format!("toolu_{index:06}"), "content": turn}]} + ]) + }) + .scan(0, |size, turns| { + *size += turns.to_string().len(); + (*size < 210 * 1024).then_some(turns) + }) + .flat_map(|turns| turns.as_array().cloned().unwrap_or_default()) + .collect(); + json!({"model": "claude-sonnet-5", "max_tokens": 4096, "messages": messages, "stream": true}) +} + +fn messages_input(c: &mut Criterion) { + Python::initialize(); + Python::attach(|py| { + let json = py.import("json").expect("Python json module should import"); + for &(label, text) in MESSAGES_INPUT_TEXTS { + let body = messages_body(text); + let dict = json_to_py(py, &body).expect("body should convert"); + c.bench_with_input( + BenchmarkId::new("messages_input_python_json_dumps", label), + &dict, + |b, dict| { + b.iter(|| { + json.call_method1("dumps", (black_box(dict),)) + .expect("body should serialize") + }) + }, + ); + c.bench_with_input( + BenchmarkId::new("messages_input_rust_raw_json", label), + &body, + |b, body| { + b.iter(|| { + let bytes = litellm_host::raw_json::to_vec(black_box(body)) + .expect("body should serialize"); + PyString::new(py, std::str::from_utf8(&bytes).expect("JSON is UTF-8")) + }) + }, + ); + } + }); +} + criterion_group! { name = benches; config = Criterion::default() .sample_size(20) .warm_up_time(Duration::from_secs(1)) .measurement_time(Duration::from_secs(4)); - targets = bridge_serialization + targets = bridge_serialization, messages_input } criterion_main!(benches); diff --git a/litellm/rust/crates/python-bridge/src/lib.rs b/litellm/rust/crates/python-bridge/src/lib.rs index 45ea95560e1..a7acb35abb4 100644 --- a/litellm/rust/crates/python-bridge/src/lib.rs +++ b/litellm/rust/crates/python-bridge/src/lib.rs @@ -53,6 +53,8 @@ mod _native { #[pymodule_export] use crate::redis::store::auth_cache::{NativeAuthCache, PyLocalSizes}; #[pymodule_export] + use crate::redis::store::deployment_health::NativeDeploymentHealth; + #[pymodule_export] use crate::redis::store::rate_limits::{ NativeRateLimits, PyAtomicApplied, PyAtomicOverLimit, PyGaugeOverLimit, PyGaugesAcquired, PyGuardedReply, PyWindowCount, @@ -60,7 +62,9 @@ mod _native { #[pymodule_export] use crate::redis::store::request::{NativeRequestRedis, NativeSend}; #[pymodule_export] - use crate::redis::store::router_usage::NativeRouterUsage; + use crate::redis::store::router_session_pins::NativeRouterSessionPins; + #[pymodule_export] + use crate::redis::store::router_usage::{NativeRouterUsage, NativeUsageRead}; #[pymodule_export] use crate::redis::store::sensitive_route_pins::NativeSensitiveRoutePins; #[pymodule_export] @@ -153,6 +157,8 @@ mod tests { "NativeRequestRedis", "NativeSend", "NativeAuthCache", + "NativeDeploymentHealth", + "NativeRouterSessionPins", "NativeRateLimits", "WindowCount", "AtomicApplied", @@ -163,6 +169,7 @@ mod tests { "NativePending", "NativeRouterCooldowns", "NativeRouterUsage", + "NativeUsageRead", "NativeSensitiveRoutePins", "NativeRedisStoreError", "NativeSpendCounters", diff --git a/litellm/rust/crates/python-bridge/src/model_cost/fetch.rs b/litellm/rust/crates/python-bridge/src/model_cost/fetch.rs index dff6296cf91..2444546810e 100644 --- a/litellm/rust/crates/python-bridge/src/model_cost/fetch.rs +++ b/litellm/rust/crates/python-bridge/src/model_cost/fetch.rs @@ -1,13 +1,14 @@ use std::time::Duration; use litellm_http::{ClientVariant, HttpClientConfig}; -use litellm_model_catalog_refresh::{Failed, Fetched, RetryPolicy, S3Access, Start}; +use litellm_model_catalog_refresh::{Failed, Fetched, GcsAccess, RetryPolicy, S3Access, Start}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyBytes; /// A cost map URL and how to connect to it. An `s3://` URL reaches S3 as an AWS SDK finds it in the environment, -/// with LiteLLM's `AWS_REGION_NAME` winning over the region it finds +/// with LiteLLM's `AWS_REGION_NAME` winning over the region it finds. A `gs://` URL reaches Cloud Storage with Google +/// application default credentials #[pyclass( frozen, name = "NativeCostMapSource", @@ -42,8 +43,15 @@ impl NativeCostMapSource { let policy = policy.0; crate::execution::run_async_value(py, async move { Ok(Outcome( - litellm_model_catalog_refresh::fetch(&client, &s3(), &url, policy, Start::Fresh) - .await, + litellm_model_catalog_refresh::fetch( + &client, + &s3(), + &GcsAccess::Environment, + &url, + policy, + Start::Fresh, + ) + .await, )) }) } @@ -68,6 +76,7 @@ impl NativeCostMapSource { litellm_host_python::run_isolated(litellm_model_catalog_refresh::fetch( &client, &s3(), + &GcsAccess::Environment, &url, policy, start, diff --git a/litellm/rust/crates/python-bridge/src/redis/events.rs b/litellm/rust/crates/python-bridge/src/redis/events.rs index a281d78d5a6..b5d51442561 100644 --- a/litellm/rust/crates/python-bridge/src/redis/events.rs +++ b/litellm/rust/crates/python-bridge/src/redis/events.rs @@ -24,16 +24,52 @@ impl EventLoop { } } +/// The Python context a Redis event is reported in. Each constructor names whose context it is, +/// so no event is reported in the context of whoever happened to build the thing reporting it. +#[derive(Clone)] +pub(crate) struct ReportContext(PythonContext); + +impl ReportContext { + /// The caller of a store op, on the stack now: its `with_service_target` and the span its + /// request parents service events under. + pub(crate) fn of_caller(py: Python<'_>) -> PyResult { + Ok(Self(PythonContext::capture(py)?)) + } + + /// The request whose post-call batch its deadline sends, with no caller awaiting the flush. + fn of_deadline_flush(py: Python<'_>) -> PyResult { + let context = py + .import(ServiceLoggingEvents::MODULE)? + .call_method0("deadline_flush_context")?; + Ok(Self(PythonContext::built(context))) + } + + /// Nobody's, for breaker transitions and calls no store op scoped. + fn background(py: Python<'_>) -> PyResult { + Ok(Self(PythonContext::empty(py)?)) + } +} + /// The Python caller of one store operation. Its context carries `with_service_target` and the /// ambient OTel span, and its `call_type` is the label `RedisCache` reports, so dashboards keep /// their series. pub(crate) struct RedisCall { - pub(crate) context: PythonContext, + pub(crate) context: ReportContext, pub(crate) event_loop: Option, pub(crate) call_type: &'static str, } impl RedisCall { + /// The deadline's flush of a request's post-call batch. Built when a post-call declaration + /// arms the deadline, so it is the declaring caller's request the flush is reported under. + pub(crate) fn deadline_flush(py: Python<'_>) -> PyResult { + Ok(Self { + context: ReportContext::of_deadline_flush(py)?, + event_loop: EventLoop::running(py), + call_type: "post_call_redis_batch", + }) + } + pub(crate) fn tag(self) -> CallTag { Arc::new(self) } @@ -47,7 +83,7 @@ impl RedisCall { pub(crate) struct ServiceLoggingEvents { record_call: Arc>, record_breaker: Arc>, - context: PythonContext, + context: ReportContext, /// The loop the client was built on, for calls whose caller had none running. event_loop: Option, runtime: Handle, @@ -79,15 +115,15 @@ impl ServiceLoggingEvents { Ok(Self { record_call: Arc::new(module.getattr("record_redis_call")?.unbind()), record_breaker: Arc::new(module.getattr("record_redis_breaker")?.unbind()), - context: PythonContext::capture(py)?, + context: ReportContext::background(py)?, event_loop, runtime, }) } - fn report(&self, context: PythonContext, record: Arc>, args: ReportArgs) { + fn report(&self, context: ReportContext, record: Arc>, args: ReportArgs) { self.runtime.spawn(async move { - let reported = attach_blocking(context, move |py| match args { + let reported = attach_blocking(context.0, move |py| match args { ReportArgs::Call(call) => record .bind(py) .call1(( @@ -205,7 +241,6 @@ impl RedisEvents for ServiceLoggingEvents { mod tests { use std::{sync::Arc, time::Duration}; - use litellm_host_python::PythonContext; use litellm_redis_client::{CallEvent, FailureKind, Operation, Outcome, RedisEvents}; use pyo3::{ prelude::*, @@ -214,7 +249,7 @@ mod tests { use rstest::rstest; use tokio::runtime::Handle; - use super::{EventLoop, RedisCall, ServiceLoggingEvents}; + use super::{EventLoop, RedisCall, ReportContext, ServiceLoggingEvents}; const RECORDER: &str = " import contextvars, time @@ -272,7 +307,7 @@ def record_redis_breaker(state): .call_method1("set", ("spend_counter",)) .unwrap(); RedisCall { - context: PythonContext::capture(py).unwrap(), + context: ReportContext::of_caller(py).unwrap(), event_loop: caller_loop.map(|caller_loop| stand_in_loop(py, caller_loop)), call_type: "async_increment", } @@ -308,6 +343,41 @@ def record_redis_breaker(state): ); } + #[tokio::test(flavor = "multi_thread")] + async fn a_call_nobody_scoped_is_reported_outside_the_context_the_client_was_built_in() { + Python::initialize(); + let (events, namespace) = Python::attach(|py| { + let name = c"redis_events_recorder_built_inside_a_request"; + let module = + PyModule::from_code(py, &std::ffi::CString::new(RECORDER).unwrap(), name, name) + .unwrap(); + module + .getattr("target") + .unwrap() + .call_method1("set", ("the request that connected",)) + .unwrap(); + let events = ServiceLoggingEvents::from_module( + py, + &module, + Handle::current(), + Some(stand_in_loop(py, "client loop")), + ) + .unwrap(); + (events, module.unbind()) + }); + + events.call(&CallEvent { + operation: Operation::Command("INCRBYFLOAT"), + duration: Duration::from_millis(250), + outcome: Outcome::Succeeded, + }); + + assert_eq!( + seen(&namespace, 1).await, + "[('client loop', 'INCRBYFLOAT', 0.25, True, None, None)]" + ); + } + #[rstest] #[case::opened(litellm_circuit_breaker::State::Open { since: std::time::Instant::now() }, "open")] #[case::probing(litellm_circuit_breaker::State::HalfOpen, "half_open")] diff --git a/litellm/rust/crates/python-bridge/src/redis/store/auth_cache.rs b/litellm/rust/crates/python-bridge/src/redis/store/auth_cache.rs index 8d7788b136e..56277ad1d3a 100644 --- a/litellm/rust/crates/python-bridge/src/redis/store/auth_cache.rs +++ b/litellm/rust/crates/python-bridge/src/redis/store/auth_cache.rs @@ -109,6 +109,32 @@ impl NativeAuthCache { }) } + fn lease<'py>(&self, py: Python<'py>, key: String) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_get_cache", kind_error, async move { + Ok::<_, litellm_redis_store::Error>(store.lease(&key).await) + }) + } + + /// The Redis write alone, `False` when a delete on any pod voided `lease`. + #[pyo3(signature = (key, lease, value, ttl_seconds = None))] + fn set_leased<'py>( + &self, + py: Python<'py>, + key: String, + lease: Option, + value: PyValue, + ttl_seconds: Option, + ) -> PyResult> { + let store = self.0.clone(); + let ttl = ttl(ttl_seconds)?; + redis_async(py, "async_set_cache", kind_error, async move { + Ok::<_, litellm_redis_store::Error>( + store.set_leased(&key, lease.as_deref(), value.0, ttl).await, + ) + }) + } + /// Memory now and the `SET` on the batch `send` picks. #[pyo3(signature = (key, value, ttl_seconds, send))] fn declare_set( diff --git a/litellm/rust/crates/python-bridge/src/redis/store/deployment_health.rs b/litellm/rust/crates/python-bridge/src/redis/store/deployment_health.rs new file mode 100644 index 00000000000..4c5bb740233 --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/redis/store/deployment_health.rs @@ -0,0 +1,72 @@ +use std::sync::Arc; + +use litellm_redis_store::{ + Error, + deployment_health::{DeploymentHealth, HealthStates}, +}; +use pyo3::prelude::*; + +use super::{ + PyData, duration, kind_error, policy::router_tier_policies, redis_async, redis_sync, remote, +}; +use crate::redis::client::NativeRedisClient; + +/// The background health checks' states behind `DeploymentHealthCache`. The `_sync` methods block +/// with the GIL released, for the health check writer and the router's sync calls. +#[pyclass(frozen, name = "NativeDeploymentHealth")] +pub(crate) struct NativeDeploymentHealth(Arc); + +#[pymethods] +impl NativeDeploymentHealth { + #[new] + fn new(py: Python<'_>, client: Option>) -> PyResult { + Ok(Self(Arc::new(DeploymentHealth::new( + client.map(|client| remote(&client)), + router_tier_policies(py)?.default_in_memory_ttl, + )))) + } + + /// Adds the Redis tier and keeps memory; `false` when one was already attached. + fn attach(&self, client: PyRef<'_, NativeRedisClient>) -> bool { + self.0.attach(remote(&client)) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + fn get<'py>(&self, py: Python<'py>) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_get_cache", kind_error, async move { + store.get().await.map(|states| states.map(PyData)) + }) + } + + fn get_sync(&self, py: Python<'_>) -> PyResult>> { + let store = self.0.clone(); + redis_sync(py, "get_cache", kind_error, async move { + store.get().await.map(|states| states.map(PyData)) + }) + } + + fn get_shared_sync(&self, py: Python<'_>) -> PyResult>> { + let store = self.0.clone(); + redis_sync(py, "get_cache", kind_error, async move { + Ok::<_, Error>(store.get_shared().await.map(PyData)) + }) + } + + fn set_sync( + &self, + py: Python<'_>, + states: PyData, + ttl_seconds: f64, + ) -> PyResult<()> { + let store = self.0.clone(); + let ttl = duration(ttl_seconds)?; + redis_sync(py, "set_cache", kind_error, async move { + store.set(states.0, ttl).await + }) + } +} diff --git a/litellm/rust/crates/python-bridge/src/redis/store/mod.rs b/litellm/rust/crates/python-bridge/src/redis/store/mod.rs index d9c3aabb814..c788c92caa7 100644 --- a/litellm/rust/crates/python-bridge/src/redis/store/mod.rs +++ b/litellm/rust/crates/python-bridge/src/redis/store/mod.rs @@ -1,8 +1,10 @@ pub(crate) mod auth_cache; +pub(crate) mod deployment_health; mod policy; pub(crate) mod rate_limits; pub(crate) mod request; pub(crate) mod router_cooldowns; +pub(crate) mod router_session_pins; pub(crate) mod router_usage; pub(crate) mod sensitive_route_pins; pub(crate) mod spend_counters; @@ -16,7 +18,6 @@ use std::{ time::Duration, }; -use litellm_host_python::PythonContext; use litellm_python_compat::number::{SecondsError, seconds}; use litellm_redis_store::{Error, FromValue, IntoValue, Number, Pending, Remote}; use pyo3::{ @@ -29,14 +30,14 @@ use pyo3::{ use crate::execution::{run_async_value, run_sync_value}; use crate::redis::{ client::NativeRedisClient, - events::{EventLoop, RedisCall}, + events::{EventLoop, RedisCall, ReportContext}, }; /// The labels `RedisCache` reports for this operation, in the caller's context so its /// `service_target` reaches the service event, dispatched on the loop the caller runs on. fn call(py: Python<'_>, call_type: &'static str) -> PyResult { Ok(RedisCall { - context: PythonContext::capture(py)?, + context: ReportContext::of_caller(py)?, event_loop: EventLoop::running(py), call_type, }) @@ -236,6 +237,10 @@ impl<'py, T: IntoValue> IntoPyObject<'py> for PyData { } } +fn unwrap_values(values: Vec) -> Vec { + values.into_iter().map(|value| value.0).collect() +} + fn unwrap_all(values: Vec>) -> Vec { values.into_iter().map(|value| value.0).collect() } diff --git a/litellm/rust/crates/python-bridge/src/redis/store/policy.rs b/litellm/rust/crates/python-bridge/src/redis/store/policy.rs index ffdf7b97a2e..506c743fad4 100644 --- a/litellm/rust/crates/python-bridge/src/redis/store/policy.rs +++ b/litellm/rust/crates/python-bridge/src/redis/store/policy.rs @@ -20,6 +20,7 @@ const MEMORY_TTL: FieldSpec> = pub(crate) struct RouterTierPolicies { pub(crate) cooldowns: TierPolicy, pub(crate) usage: TierPolicy, + pub(crate) default_in_memory_ttl: Option, } pub(crate) fn router_tier_policies(py: Python<'_>) -> PyResult { @@ -40,6 +41,7 @@ fn project(snapshot: &Snapshot<'_>) -> Result Option { + Python::attach(RedisCall::deadline_flush) + .inspect_err(|error| { + tracing::warn!(%error, "a post-call deadline flush will be reported without its request"); + }) + .ok() + .map(RedisCall::tag) +} /// One request's native Redis batches. Python builds one per request scope and passes it to each /// request-aware store call. @@ -15,9 +28,8 @@ impl NativeRequestRedis { fn new(py: Python<'_>, post_call_deadline_seconds: f64) -> PyResult { let runtime = litellm_host_python::runtime_handle()?; let deadline = duration(post_call_deadline_seconds)?; - let deadline_tag = call(py, "post_call_redis_batch")?.tag(); Ok(Self(crate::logger::capture(py).scope(|| { - RequestRedis::with_deadline_tag(runtime, deadline, Some(deadline_tag)) + RequestRedis::with_deadline_tag(runtime, deadline, Arc::new(deadline_flush_tag)) }))) } diff --git a/litellm/rust/crates/python-bridge/src/redis/store/router_session_pins.rs b/litellm/rust/crates/python-bridge/src/redis/store/router_session_pins.rs new file mode 100644 index 00000000000..59dbcf853cb --- /dev/null +++ b/litellm/rust/crates/python-bridge/src/redis/store/router_session_pins.rs @@ -0,0 +1,234 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_redis_store::{ + Clock, + router_session_pins::{Eligible, PinKey, RouterSessionPins}, + system_clock, +}; +use pyo3::prelude::*; + +use super::{ + NativePending, duration, kind_error, pending, policy::router_tier_policies, redis_async, + remote, request::NativeSend, unwrap_values, value::PyValue, +}; +use crate::redis::client::NativeRedisClient; + +fn pin(value: Option) -> Option { + value.map(PyValue) +} + +/// The `PinKey` dataclasses in `litellm.redis.stores.router_session_pins.keys`. A key whose fields +/// are a subset of another's comes after it, so extraction never stops at the smaller one. +#[derive(FromPyObject)] +enum PyPinKey { + TierModel { + router: String, + tiers: String, + caller: Option, + session_id: String, + tier: String, + replay_fallback: Option, + }, + TierSession { + router: String, + caller: Option, + session_id: String, + }, + DeploymentSession { + model_group: String, + caller: Option, + session_id: String, + }, + Deployment { + model_group: String, + caller: String, + }, +} + +impl From for PinKey { + fn from(key: PyPinKey) -> Self { + match key { + PyPinKey::TierModel { + router, + tiers, + caller, + session_id, + tier, + replay_fallback, + } => Self::TierModel { + router, + tiers, + caller, + session_id, + tier, + replay_fallback, + }, + PyPinKey::TierSession { + router, + caller, + session_id, + } => Self::TierSession { + router, + caller, + session_id, + }, + PyPinKey::DeploymentSession { + model_group, + caller, + session_id, + } => Self::DeploymentSession { + model_group, + caller, + session_id, + }, + PyPinKey::Deployment { + model_group, + caller, + } => Self::Deployment { + model_group, + caller, + }, + } + } +} + +/// `InMemoryCache(clock=...)`: a callable returning epoch seconds, which tests use to move time. +fn python_clock(clock: Py) -> Clock { + Arc::new(move || { + Python::attach(|py| clock.call0(py)?.extract::(py)) + .ok() + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + .unwrap_or_default() + }) +} + +/// The router's session affinity pins, behind `RouterSessionPinStore`. +#[pyclass(frozen, name = "NativeRouterSessionPins")] +pub(crate) struct NativeRouterSessionPins(Arc); + +#[pymethods] +impl NativeRouterSessionPins { + #[new] + #[pyo3(signature = (client, clock=None))] + fn new( + py: Python<'_>, + client: Option>, + clock: Option>, + ) -> PyResult { + Ok(Self(Arc::new(RouterSessionPins::with_clock( + client.map(|client| remote(&client)), + router_tier_policies(py)?.default_in_memory_ttl, + clock.map_or_else(system_clock, python_clock), + )))) + } + + /// Adds the Redis tier and keeps memory; `false` when one was already attached. + fn attach(&self, client: PyRef<'_, NativeRedisClient>) -> bool { + self.0.attach(remote(&client)) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + fn get<'py>(&self, py: Python<'py>, key: PyPinKey) -> PyResult> { + let store = self.0.clone(); + let key = PinKey::from(key); + redis_async(py, "async_get_cache", kind_error, async move { + store.get(&key).await.map(pin) + }) + } + + fn get_local(&self, key: PyPinKey) -> PyResult> { + self.0.get_local(&key.into()).map(pin).map_err(kind_error) + } + + fn set_local(&self, key: PyPinKey, value: PyValue, ttl_seconds: f64) -> PyResult<()> { + self.0 + .set_local(&key.into(), value.0, duration(ttl_seconds)?) + .map_err(kind_error) + } + + fn delete_local(&self, key: PyPinKey) -> PyResult<()> { + self.0.delete_local(&key.into()).map_err(kind_error) + } + + fn set<'py>( + &self, + py: Python<'py>, + key: PyPinKey, + value: PyValue, + ttl_seconds: f64, + ) -> PyResult> { + let store = self.0.clone(); + let key = PinKey::from(key); + let ttl = duration(ttl_seconds)?; + redis_async(py, "async_set_cache", kind_error, async move { + store.set(&key, value.0, ttl).await + }) + } + + fn get_shared<'py>(&self, py: Python<'py>, key: PyPinKey) -> PyResult> { + let store = self.0.clone(); + let key = PinKey::from(key); + redis_async(py, "async_get_cache", kind_error, async move { + store.get_shared(&key).await.map(pin) + }) + } + + /// The read script, declared on the batch the caller's `NativeSend` picked. + fn read(&self, key: PyPinKey, ttl_seconds: f64, send: &NativeSend) -> PyResult { + let declared = self + .0 + .read(send.send(), &key.into(), duration(ttl_seconds)?) + .map_err(kind_error)?; + Ok(pending("async_register_script", kind_error, declared, pin)) + } + + #[pyo3(signature = (key, value, ttl_seconds, eligible_values, preserve_legacy_model_ids))] + fn claim<'py>( + &self, + py: Python<'py>, + key: PyPinKey, + value: PyValue, + ttl_seconds: f64, + eligible_values: Option>, + preserve_legacy_model_ids: bool, + ) -> PyResult> { + let store = self.0.clone(); + let key = PinKey::from(key); + let ttl = duration(ttl_seconds)?; + let eligible = eligible(eligible_values, preserve_legacy_model_ids); + redis_async(py, "async_register_script", kind_error, async move { + store.claim(&key, &value.0, ttl, &eligible).await.map(pin) + }) + } + + #[pyo3(signature = (key, value, ttl_seconds, eligible_values, preserve_legacy_model_ids))] + fn claim_local( + &self, + key: PyPinKey, + value: PyValue, + ttl_seconds: f64, + eligible_values: Option>, + preserve_legacy_model_ids: bool, + ) -> PyResult { + self.0 + .claim_local( + &key.into(), + &value.0, + duration(ttl_seconds)?, + &eligible(eligible_values, preserve_legacy_model_ids), + ) + .map(PyValue) + .map_err(kind_error) + } +} + +fn eligible(values: Option>, preserve_legacy_model_ids: bool) -> Eligible { + Eligible { + values: values.map(unwrap_values), + preserve_legacy_model_ids, + } +} diff --git a/litellm/rust/crates/python-bridge/src/redis/store/router_usage.rs b/litellm/rust/crates/python-bridge/src/redis/store/router_usage.rs index 48565028de3..dbe7ecfedb2 100644 --- a/litellm/rust/crates/python-bridge/src/redis/store/router_usage.rs +++ b/litellm/rust/crates/python-bridge/src/redis/store/router_usage.rs @@ -1,28 +1,40 @@ -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex, PoisonError}, + time::Duration, +}; -use litellm_redis_store::router_usage::{Counter, Kind, RouterUsage}; +use litellm_redis_store::router_usage::{ + Counter, Increment, Kind, RouterUsage, Scope, UsageReservation, +}; use pyo3::prelude::*; use super::{ - PyNumber, Spelled, kind_error, policy::router_tier_policies, redis_async, redis_sync, remote, + NativePending, PyNumber, Spelled, kind_error, pending, + policy::router_tier_policies, + redis_async, redis_sync, remote, + request::{NativeRequestRedis, NativeSend}, + ttl, }; use crate::redis::client::NativeRedisClient; /// `litellm.redis.stores.router_usage.UsageCounter`. #[derive(FromPyObject)] struct PyCounter { + scope: Spelled, deployment_id: String, model: Option, - minute: String, + window: String, kind: Spelled, } impl From for Counter { fn from(counter: PyCounter) -> Self { Self { + scope: counter.scope.0, deployment_id: counter.deployment_id, model: counter.model, - minute: counter.minute, + window: counter.window, kind: counter.kind.0, } } @@ -32,11 +44,66 @@ fn counters(counters: Vec) -> Vec { counters.into_iter().map(Counter::from).collect() } +/// `litellm.redis.stores.router_usage.CounterIncrement`. +#[derive(FromPyObject)] +struct PyIncrement { + counter: PyCounter, + amount: PyNumber, + ttl: Option, +} + +impl TryFrom for Increment { + type Error = PyErr; + + fn try_from(increment: PyIncrement) -> PyResult { + Ok(Self { + counter: increment.counter.into(), + amount: increment.amount.0, + ttl: ttl(increment.ttl)?, + }) + } +} + +fn increments(increments: Vec) -> PyResult> { + increments.into_iter().map(Increment::try_from).collect() +} + +fn numbers(read: Vec>) -> Vec> { + read.into_iter().map(|count| count.map(PyNumber)).collect() +} + +fn totals(totals: Option>) -> Option> { + totals.map(|totals| totals.into_iter().map(PyNumber).collect()) +} + /// One router's global TPM/RPM counters. The `_sync` methods block with the GIL released, for the /// Rust router's sync calls, which have no event loop to await on. #[pyclass(frozen, name = "NativeRouterUsage")] pub(crate) struct NativeRouterUsage(Arc); +/// A usage read declared on a batch, with the refreshes it reserved, given back at most once. +#[pyclass(frozen, name = "NativeUsageRead")] +pub(crate) struct NativeUsageRead { + #[pyo3(get)] + read: Py, + store: Arc, + reservation: Mutex>, +} + +#[pymethods] +impl NativeUsageRead { + fn release(&self) { + let reservation = self + .reservation + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + if let Some(reservation) = reservation { + self.store.release(reservation); + } + } +} + #[pymethods] impl NativeRouterUsage { #[new] @@ -166,6 +233,127 @@ impl NativeRouterUsage { }) } + /// `get_many` declared on the batch `send` picks, with the refreshes it reserved. + fn usage( + &self, + py: Python<'_>, + counters: Vec, + send: &NativeSend, + ) -> PyResult { + let declared = self + .0 + .declare_get_many(send.send(), &self::counters(counters)) + .map_err(kind_error)?; + Ok(NativeUsageRead { + read: Py::new( + py, + pending("async_batch_get_cache", kind_error, declared.read, numbers), + )?, + store: Arc::clone(&self.0), + reservation: Mutex::new(Some(declared.reservation)), + }) + } + + fn increment_pipeline<'py>( + &self, + py: Python<'py>, + increments: Vec, + ) -> PyResult> { + let store = self.0.clone(); + let increments = self::increments(increments)?; + redis_async(py, "async_increment_pipeline", kind_error, async move { + Ok(totals(store.increment_pipeline(&increments).await)) + }) + } + + /// Counts in memory now and declares the increments on the request's post-call batch. + fn increment_post_call( + &self, + request: &NativeRequestRedis, + increments: Vec, + ) -> PyResult<()> { + self.0 + .increment_post_call(&request.0, &self::increments(increments)?); + Ok(()) + } + + fn increment_remote<'py>( + &self, + py: Python<'py>, + increments: Vec, + ) -> PyResult> { + let store = self.0.clone(); + let increments = self::increments(increments)?; + redis_async(py, "async_increment_pipeline", kind_error, async move { + store.increment_remote(&increments).await.map(totals) + }) + } + + fn increment_local(&self, increment: PyIncrement) -> PyResult { + self.0 + .increment_local(&increment.try_into()?) + .map(PyNumber) + .map_err(kind_error) + } + + fn get_local(&self, counter: PyCounter) -> PyResult> { + self.0 + .get_local(&counter.into()) + .map(|count| count.map(PyNumber)) + .map_err(kind_error) + } + + fn get_local_many(&self, counters: Vec) -> PyResult>> { + self.0 + .get_local_many(&self::counters(counters)) + .map(numbers) + .map_err(kind_error) + } + + fn set_local(&self, counter: PyCounter, value: PyNumber) -> PyResult<()> { + self.0 + .set_local(&counter.into(), value.0) + .map_err(kind_error) + } + + fn current<'py>( + &self, + py: Python<'py>, + counter: PyCounter, + limit: PyNumber, + ) -> PyResult> { + let store = self.0.clone(); + let counter = Counter::from(counter); + redis_async(py, "async_get_cache", kind_error, async move { + Ok(store.current(&counter, limit.0).await?.map(PyNumber)) + }) + } + + fn shared_or_local<'py>( + &self, + py: Python<'py>, + counter: PyCounter, + ) -> PyResult> { + let store = self.0.clone(); + let counter = Counter::from(counter); + redis_async(py, "async_get_cache", kind_error, async move { + Ok(store.shared_or_local(&counter).await?.map(PyNumber)) + }) + } + + fn current_sync( + &self, + py: Python<'_>, + counter: PyCounter, + limit: PyNumber, + ) -> PyResult> { + let store = self.0.clone(); + let counter = Counter::from(counter); + redis_sync(py, "get_cache", kind_error, async move { + Ok(store.current(&counter, limit.0).await?.map(PyNumber)) + }) + } + fn flush_local_counters(&self) -> PyResult<()> { self.0.flush_local_counters().map_err(kind_error) } diff --git a/litellm/rust/crates/python-bridge/src/router/host.rs b/litellm/rust/crates/python-bridge/src/router/host.rs index 04d19e70edc..08169091e1b 100644 --- a/litellm/rust/crates/python-bridge/src/router/host.rs +++ b/litellm/rust/crates/python-bridge/src/router/host.rs @@ -6,11 +6,13 @@ //! current thread, so a host future finds its exchange without the router passing it through. //! //! The routing cache is a host operation too: [`StepCache`] parks every [`RouterCache`] op as a -//! `cache` step, and Python runs it against the router's own `DualCache` (memory plus Redis), so -//! Python and Rust read and write the same cooldowns, fail counters and usage. +//! `cache` step, and Python answers it from the router's stores, so Python and Rust read and write +//! the same cooldowns, fail counters and usage. A step names the state it touches, never a Redis +//! key: the keys are the stores'. use std::{ cell::RefCell, + collections::HashMap, future::{Future, poll_fn}, pin::Pin, sync::{Arc, Mutex, MutexGuard, PoisonError}, @@ -21,9 +23,10 @@ use std::{ use async_trait::async_trait; use litellm_router_core::{ Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, CallbackFiltered, Deployment, - Error, Host, Output, Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache, - RouterEvent, RouterRequest, RouterResponse, RoutingContext, SecretRef, SelectStage, - ThreadRandomness, build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo, + Error, HealthState, Host, Outcomes, Output, Payload, PreRoute, PreRouted, Provided, Randomness, + RouterCache, RouterEvent, RouterRequest, RouterResponse, RoutingContext, SecretRef, + SelectStage, ThreadRandomness, UsageCounts, UsageWindow, build::BuiltDeployment, + health::CooldownValue, host::ModelInfoLookup, model_info::MergedModelInfo, precall::PreCallFacts, }; use pyo3::{PyResult, Python, types::PyAnyMethods}; @@ -102,55 +105,123 @@ pub(crate) enum Stage { ProviderRequest, } -/// The `op` of a `cache` step: one per [`RouterCache`] method. +/// One [`RouterCache`] op, as a `cache` step carries it to Python. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub(crate) enum CacheCall { + /// Answered with one stored cooldown or `None` per id. + Cooldowns { ids: Vec }, + /// Answered with `None`. + PutCooldown { + id: String, + cooldown: CooldownValue, + ttl: f64, + }, + /// Answered with the count after this failure. + IncrementAllowedFails { + id: String, + suffix: Option, + ttl: f64, + }, + /// Answered with `None`. + AddUsage { + window: UsageWindow, + #[serde(serialize_with = "serialize_amount")] + tokens: f64, + #[serde(serialize_with = "serialize_amount")] + requests: f64, + }, + /// Answered with `None`. + AddRequest { window: UsageWindow }, + /// Answered with `[tokens, requests]`, each a number or `None`. + Usage { window: UsageWindow }, + /// Answered with each checked deployment's `{is_healthy, timestamp}` by id, or `None`. + HealthStates, + /// Answered with this worker's `[successes, fails]` of the current minute. + Outcomes { id: String }, + /// Answered with `None`. + CountSuccess { id: String }, + /// Answered with `None`. + CountFailure { id: String }, +} + +/// The `op` of a `cache` step. #[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] pub(crate) enum CacheOp { - /// `async_get_cache(key)`: answered with the stored JSON value, or `None`. - #[strum(serialize = "get")] - Get, - /// `async_set_cache(key, value, ttl)`: answered with `None`. - #[strum(serialize = "set")] - Set, - /// `async_increment(key, value, ttl)`: answered with the new number. The TTL applies when - /// the key is created. - #[strum(serialize = "increment")] - Increment, - /// `async_increment_usage(keys, value, ttl)`: `value` is one amount per key, added as - /// `_increment_deployment_usage` adds them. Answered with `None`. - #[strum(serialize = "increment_usage")] - IncrementUsage, - /// `async_batch_get_cache(keys)`: answered with one JSON value or `None` per key, in order. - #[strum(serialize = "batch_get")] - BatchGet, - /// `async_delete_cache(key)`: answered with `None`. - #[strum(serialize = "delete")] - Delete, + #[strum(serialize = "cooldowns")] + Cooldowns, + #[strum(serialize = "put_cooldown")] + PutCooldown, + #[strum(serialize = "increment_allowed_fails")] + IncrementAllowedFails, + #[strum(serialize = "add_usage")] + AddUsage, + #[strum(serialize = "add_request")] + AddRequest, + #[strum(serialize = "usage")] + Usage, + #[strum(serialize = "health_states")] + HealthStates, + #[strum(serialize = "outcomes")] + Outcomes, + #[strum(serialize = "count_success")] + CountSuccess, + #[strum(serialize = "count_failure")] + CountFailure, +} + +impl CacheCall { + pub(crate) fn op(&self) -> CacheOp { + match self { + Self::Cooldowns { .. } => CacheOp::Cooldowns, + Self::PutCooldown { .. } => CacheOp::PutCooldown, + Self::IncrementAllowedFails { .. } => CacheOp::IncrementAllowedFails, + Self::AddUsage { .. } => CacheOp::AddUsage, + Self::AddRequest { .. } => CacheOp::AddRequest, + Self::Usage { .. } => CacheOp::Usage, + Self::HealthStates => CacheOp::HealthStates, + Self::Outcomes { .. } => CacheOp::Outcomes, + Self::CountSuccess { .. } => CacheOp::CountSuccess, + Self::CountFailure { .. } => CacheOp::CountFailure, + } + } + + /// How many entries a `Values` answer holds. + fn values_len(&self) -> Option { + match self { + Self::Cooldowns { ids } => Some(ids.len()), + Self::Usage { .. } | Self::Outcomes { .. } => Some(2), + _ => None, + } + } } /// Python's outcome of a `cache` step. #[derive(Clone, Debug, PartialEq)] pub(crate) enum CacheOutcome { - /// `get`: the stored value (`None` when missing). Value(Option), - /// `batch_get`: one entry per key. Values(Vec>), - /// `increment`: the counter after the increment. Number(f64), - /// `set`, `delete` and `increment_usage`. Stored, } impl CacheOutcome { - /// The op this outcome answers (`Stored` answers both `set` and `delete`). fn answers(&self, op: CacheOp) -> bool { matches!( (self, op), - (Self::Value(_), CacheOp::Get) - | (Self::Values(_), CacheOp::BatchGet) - | (Self::Number(_), CacheOp::Increment) + (Self::Value(_), CacheOp::HealthStates) + | ( + Self::Values(_), + CacheOp::Cooldowns | CacheOp::Usage | CacheOp::Outcomes + ) + | (Self::Number(_), CacheOp::IncrementAllowedFails) | ( Self::Stored, - CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage + CacheOp::PutCooldown + | CacheOp::AddUsage + | CacheOp::AddRequest + | CacheOp::CountSuccess + | CacheOp::CountFailure ) ) } @@ -219,18 +290,7 @@ pub(crate) enum HostCall { fallback_depth: u32, retry: u32, }, - /// One [`RouterCache`] op against the host's cache. `key` is set for every op but - /// `batch_get`, which sets `keys`; `value` is set for `set` (the JSON to store) and - /// `increment` (the amount); `ttl` is in seconds. - Cache { - op: CacheOp, - key: Option, - keys: Option>, - value: Option, - ttl: Option, - /// The op reads or writes this worker's in-memory tier only (`local_only=True`). - local: bool, - }, + Cache(CacheCall), /// The deployment build's `os.environ/` references: answered with one value per ref. Secrets { index: usize, @@ -263,7 +323,7 @@ impl HostCall { Self::RefuseFallbacks { .. } => StepKind::RefuseFallbacks, Self::Select { .. } => StepKind::Select, Self::PreRoute { .. } => StepKind::PreRoute, - Self::Cache { .. } => StepKind::Cache, + Self::Cache(_) => StepKind::Cache, Self::Secrets { .. } => StepKind::Secrets, Self::Build { kind, .. } => *kind, Self::ModelInfo { .. } => StepKind::ModelInfo, @@ -380,15 +440,17 @@ impl HostCallResponse { got: values.len(), }); } - if let (Self::Cache(outcome), HostCall::Cache { op, keys, .. }) = (self, call) { - if !outcome.answers(*op) { - return Err(DriveError::WrongCacheOutcome { op: *op }); + if let (Self::Cache(outcome), HostCall::Cache(cache)) = (self, call) { + let op = cache.op(); + if !outcome.answers(op) { + return Err(DriveError::WrongCacheOutcome { op }); } - if let (CacheOutcome::Values(values), Some(keys)) = (outcome, keys) - && values.len() != keys.len() + if let (CacheOutcome::Values(values), Some(expected)) = (outcome, cache.values_len()) + && values.len() != expected { return Err(DriveError::BatchLength { - expected: keys.len(), + op, + expected, got: values.len(), }); } @@ -418,8 +480,10 @@ pub(crate) enum Finish { }, /// `report_failure`: the cooldown applied, if the deployment was cooled down. Reported { cooldown: Option }, - /// `report_success` / `count_success`: the success counter's key, if one was counted. + /// `report_success`: the tokens counter's key, if usage was added. Counted { key: Option }, + /// `count_success`: whether a success was counted. + SuccessCounted { counted: bool }, /// `build`: the deployments it installed, in order. Built(Vec), /// `router_model_info`: the merged model info (`None` when the lookup answered `None`). @@ -454,8 +518,12 @@ pub(crate) enum DriveError { UnknownCandidate { id: String }, #[error("the outcome does not answer a `{op}` cache step")] WrongCacheOutcome { op: CacheOp }, - #[error("a `batch_get` cache step for {expected} keys was answered with {got} values")] - BatchLength { expected: usize, got: usize }, + #[error("a `{op}` cache step for {expected} entries was answered with {got} values")] + BatchLength { + op: CacheOp, + expected: usize, + got: usize, + }, #[error("a `secrets` step for {expected} references was answered with {got} values")] SecretsLength { expected: usize, got: usize }, #[error("the router call is pending on something other than a step")] @@ -541,12 +609,8 @@ async fn call_host(call: HostCall) -> HostCallResponse { const CHECKED: &str = "the driver checks each response against its host call"; -fn ttl_seconds(ttl: Option) -> Option { - ttl.map(|ttl| ttl.as_secs_f64()) -} - /// An increment amount as Python's counters store it: an int when it is whole. -fn amount_value(amount: f64) -> Value { +pub(crate) fn amount_value(amount: f64) -> Value { if amount.fract() == 0.0 && amount.abs() < i64::MAX as f64 { Value::from(amount as i64) } else { @@ -554,131 +618,151 @@ fn amount_value(amount: f64) -> Value { } } -/// The host's cache: every op is a `cache` step Python runs against the router's `DualCache`. -pub(crate) struct StepCache; - -async fn cache_step( - op: CacheOp, - key: Option<&str>, - keys: Option<&[String]>, - value: Option, - ttl: Option, -) -> CacheOutcome { - call_cache(op, key, keys, value, ttl, false).await +fn serialize_amount(amount: &f64, serializer: S) -> Result { + serde::Serialize::serialize(&amount_value(*amount), serializer) } -async fn call_cache( - op: CacheOp, - key: Option<&str>, - keys: Option<&[String]>, - value: Option, - ttl: Option, - local: bool, -) -> CacheOutcome { - let call = HostCall::Cache { - op, - key: key.map(str::to_owned), - keys: keys.map(<[String]>::to_vec), - value, - ttl: ttl_seconds(ttl), - local, - }; - let HostCallResponse::Cache(outcome) = call_host(call).await else { +/// The host's cache: every op is a `cache` step Python answers from the router's stores. +pub(crate) struct StepCache; + +async fn cache_step(call: CacheCall) -> CacheOutcome { + let HostCallResponse::Cache(outcome) = call_host(HostCall::Cache(call)).await else { unreachable!("{CHECKED}"); }; outcome } +async fn stored(call: CacheCall) -> Result<(), litellm_cache::Error> { + let CacheOutcome::Stored = cache_step(call).await else { + unreachable!("{CHECKED}"); + }; + Ok(()) +} + +async fn counted(call: CacheCall) -> Result { + let CacheOutcome::Number(next) = cache_step(call).await else { + unreachable!("{CHECKED}"); + }; + Ok(next) +} + +async fn value(call: CacheCall) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = cache_step(call).await else { + unreachable!("{CHECKED}"); + }; + Ok(value) +} + +async fn values(call: CacheCall) -> Result>, litellm_cache::Error> { + let CacheOutcome::Values(values) = cache_step(call).await else { + unreachable!("{CHECKED}"); + }; + Ok(values) +} + #[async_trait] impl RouterCache for StepCache { - async fn async_get_cache(&self, key: &str) -> Result, litellm_cache::Error> { - let CacheOutcome::Value(value) = - cache_step(CacheOp::Get, Some(key), None, None, None).await - else { - unreachable!("{CHECKED}"); - }; - Ok(value) + async fn cooldowns( + &self, + ids: &[String], + ) -> Result>, litellm_cache::Error> { + let stored = values(CacheCall::Cooldowns { ids: ids.to_vec() }).await?; + // A cooldown that does not read as one is no cooldown. + Ok(stored + .into_iter() + .map(|cooldown| serde_json::from_value(cooldown?).ok()) + .collect()) } - async fn async_set_cache( + async fn put_cooldown( &self, - key: &str, - value: Value, - ttl: Option, + id: &str, + cooldown: &CooldownValue, + ttl: Duration, ) -> Result<(), litellm_cache::Error> { - cache_step(CacheOp::Set, Some(key), None, Some(value), ttl).await; - Ok(()) + stored(CacheCall::PutCooldown { + id: id.to_owned(), + cooldown: cooldown.clone(), + ttl: ttl.as_secs_f64(), + }) + .await } - async fn async_increment( + async fn increment_allowed_fails( &self, - key: &str, - amount: f64, - ttl: Option, + id: &str, + suffix: Option<&str>, + ttl: Duration, ) -> Result { - let value = Some(amount_value(amount)); - let CacheOutcome::Number(next) = - cache_step(CacheOp::Increment, Some(key), None, value, ttl).await - else { - unreachable!("{CHECKED}"); - }; - Ok(next) + counted(CacheCall::IncrementAllowedFails { + id: id.to_owned(), + suffix: suffix.map(str::to_owned), + ttl: ttl.as_secs_f64(), + }) + .await } - async fn async_batch_get_cache( + async fn add_usage( &self, - keys: &[String], - ) -> Result>, litellm_cache::Error> { - let CacheOutcome::Values(values) = - cache_step(CacheOp::BatchGet, None, Some(keys), None, None).await - else { - unreachable!("{CHECKED}"); - }; - Ok(values) - } - - async fn async_delete_cache(&self, key: &str) -> Result<(), litellm_cache::Error> { - cache_step(CacheOp::Delete, Some(key), None, None, None).await; - Ok(()) - } - - async fn async_increment_usage( - &self, - increments: &[(String, f64)], - ttl: Option, + window: &UsageWindow, + tokens: f64, + requests: f64, ) -> Result<(), litellm_cache::Error> { - let keys: Vec = increments.iter().map(|(key, _)| key.clone()).collect(); - let amounts = increments - .iter() - .map(|(_, amount)| amount_value(*amount)) - .collect(); - let value = Some(Value::Array(amounts)); - cache_step(CacheOp::IncrementUsage, None, Some(&keys), value, ttl).await; - Ok(()) + stored(CacheCall::AddUsage { + window: window.clone(), + tokens, + requests, + }) + .await } - async fn async_get_local(&self, key: &str) -> Result, litellm_cache::Error> { - let CacheOutcome::Value(value) = - call_cache(CacheOp::Get, Some(key), None, None, None, true).await - else { - unreachable!("{CHECKED}"); - }; - Ok(value) + async fn add_request(&self, window: &UsageWindow) -> Result<(), litellm_cache::Error> { + stored(CacheCall::AddRequest { + window: window.clone(), + }) + .await } - async fn async_increment_local( - &self, - key: &str, - amount: f64, - ttl: Option, - ) -> Result { - let value = Some(amount_value(amount)); - let CacheOutcome::Number(next) = - call_cache(CacheOp::Increment, Some(key), None, value, ttl, true).await - else { - unreachable!("{CHECKED}"); + async fn usage(&self, window: &UsageWindow) -> Result { + let read = values(CacheCall::Usage { + window: window.clone(), + }) + .await?; + let count = |index: usize| read.get(index).cloned().flatten().and_then(|v| v.as_f64()); + Ok(UsageCounts { + tokens: count(0), + requests: count(1), + }) + } + + async fn health_states(&self) -> Result, litellm_cache::Error> { + let states = value(CacheCall::HealthStates).await?; + Ok(states + .and_then(|states| serde_json::from_value(states).ok()) + .unwrap_or_default()) + } + + async fn outcomes(&self, id: &str) -> Result { + let read = values(CacheCall::Outcomes { id: id.to_owned() }).await?; + let count = |index: usize| { + read.get(index) + .cloned() + .flatten() + .and_then(|count| count.as_f64()) + .map_or(0, |count| count as u64) }; - Ok(next) + Ok(Outcomes { + successes: count(0), + fails: count(1), + }) + } + + async fn count_success(&self, id: &str) -> Result<(), litellm_cache::Error> { + stored(CacheCall::CountSuccess { id: id.to_owned() }).await + } + + async fn count_failure(&self, id: &str) -> Result<(), litellm_cache::Error> { + stored(CacheCall::CountFailure { id: id.to_owned() }).await } } @@ -1151,13 +1235,12 @@ impl Driver { #[cfg(test)] mod tests { - use std::{ - collections::HashMap, - time::{SystemTime, UNIX_EPOCH}, - }; + use std::time::{SystemTime, UNIX_EPOCH}; + use futures_util::FutureExt; use litellm_router_core::{ - FailureKind, Op, Router, RouterConfig, RouterDynamicConfig, SeededRandomness, + FailureKind, InMemoryRouterCache, Op, Router, RouterConfig, RouterDynamicConfig, + SeededRandomness, }; use rstest::{fixture, rstest}; use serde_json::{Value, json}; @@ -1195,61 +1278,81 @@ mod tests { HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::RateLimit, Some(429)))) } - /// A dict-backed stand-in for Python's `DualCache` that answers `cache` steps and keeps - /// every one it answered. + /// Router-core's in-memory cache standing in for Python's stores: it answers `cache` steps + /// and keeps every one it answered. #[derive(Default)] struct Store { - entries: HashMap, + cache: InMemoryRouterCache, calls: Vec, } impl Store { fn respond(&mut self, call: &HostCall) -> HostCallResponse { - let HostCall::Cache { - op, - key, - keys, - value, - .. - } = call - else { + let HostCall::Cache(cache_call) = call else { panic!("not a cache step: {call:?}"); }; self.calls.push(call.clone()); - let key = key.clone().unwrap_or_default(); - HostCallResponse::Cache(match op { - CacheOp::Get => CacheOutcome::Value(self.entries.get(&key).cloned()), - CacheOp::Set => { - self.entries.insert(key, value.clone().unwrap()); - CacheOutcome::Stored - } - CacheOp::Increment => { - let current = self.entries.get(&key).and_then(Value::as_f64); - let next = - current.unwrap_or(0.0) + value.as_ref().and_then(Value::as_f64).unwrap(); - self.entries.insert(key, amount_value(next)); - CacheOutcome::Number(next) - } - CacheOp::IncrementUsage => { - let amounts = value.as_ref().and_then(Value::as_array).unwrap(); - for (key, amount) in keys.iter().flatten().zip(amounts) { - let current = self.entries.get(key).and_then(Value::as_f64); - let next = current.unwrap_or(0.0) + amount.as_f64().unwrap(); - self.entries.insert(key.clone(), amount_value(next)); + let cache = &self.cache; + let seconds = Duration::from_secs_f64; + let outcome = async { + Ok::<_, litellm_cache::Error>(match cache_call { + CacheCall::Cooldowns { ids } => CacheOutcome::Values( + cache + .cooldowns(ids) + .await? + .iter() + .map(|cooldown| cooldown.as_ref().map(|cooldown| json!(cooldown))) + .collect(), + ), + CacheCall::PutCooldown { id, cooldown, ttl } => { + cache.put_cooldown(id, cooldown, seconds(*ttl)).await?; + CacheOutcome::Stored } - CacheOutcome::Stored - } - CacheOp::BatchGet => CacheOutcome::Values( - keys.iter() - .flatten() - .map(|key| self.entries.get(key).cloned()) - .collect(), - ), - CacheOp::Delete => { - self.entries.remove(&key); - CacheOutcome::Stored - } - }) + CacheCall::IncrementAllowedFails { id, suffix, ttl } => CacheOutcome::Number( + cache + .increment_allowed_fails(id, suffix.as_deref(), seconds(*ttl)) + .await?, + ), + CacheCall::AddUsage { + window, + tokens, + requests, + } => { + cache.add_usage(window, *tokens, *requests).await?; + CacheOutcome::Stored + } + CacheCall::AddRequest { window } => { + cache.add_request(window).await?; + CacheOutcome::Stored + } + CacheCall::Usage { window } => { + let counts = cache.usage(window).await?; + CacheOutcome::Values( + [counts.tokens, counts.requests] + .map(|count| count.map(Value::from)) + .to_vec(), + ) + } + CacheCall::HealthStates => CacheOutcome::Value(None), + CacheCall::Outcomes { id } => { + let outcomes = cache.outcomes(id).await?; + CacheOutcome::Values( + [outcomes.successes, outcomes.fails] + .map(|count| Some(Value::from(count))) + .to_vec(), + ) + } + CacheCall::CountSuccess { id } => { + cache.count_success(id).await?; + CacheOutcome::Stored + } + CacheCall::CountFailure { id } => { + cache.count_failure(id).await?; + CacheOutcome::Stored + } + }) + }; + HostCallResponse::Cache(outcome.now_or_never().unwrap().unwrap()) } /// Resume with `response`, answering every cache step on the way, up to the next other @@ -1305,18 +1408,10 @@ mod tests { fn cooldown_reads_are_cache_steps(pair: Router) { let mut driver = start(&pair, "primary"); let step = driver.resume(None).unwrap(); - let Step::HostCall(HostCall::Cache { - op: CacheOp::BatchGet, - key: None, - keys: Some(keys), - value: None, - ttl: None, - local: false, - }) = step - else { + let Step::HostCall(HostCall::Cache(CacheCall::Cooldowns { ids })) = step else { panic!("the call should read cooldowns first, got {step:?}"); }; - assert_eq!(keys, ["deployment:a1:cooldown", "deployment:a2:cooldown"]); + assert_eq!(ids, ["a1", "a2"]); } #[rstest] @@ -1365,10 +1460,21 @@ mod tests { .duration_since(UNIX_EPOCH) .unwrap() .as_secs_f64(); - store.entries.insert( - "deployment:a1:cooldown".into(), - json!({"exception_received": "slow down", "status_code": "429", "timestamp": now, "cooldown_time": 60}), - ); + store + .cache + .put_cooldown( + "a1", + &CooldownValue { + exception_received: "slow down".to_owned(), + status_code: "429".to_owned(), + timestamp: now, + cooldown_time: 60.0, + }, + Duration::from_secs(60), + ) + .now_or_never() + .unwrap() + .unwrap(); for _ in 0..8 { let mut call = start(&pair_without_retries, "primary"); let Step::HostCall(HostCall::Attempt { deployment_id, .. }) = @@ -1409,7 +1515,7 @@ mod tests { .calls .iter() .filter(|call| { - matches!(call, HostCall::Cache { op, .. } if !matches!(op, CacheOp::Get | CacheOp::BatchGet)) + matches!(call, HostCall::Cache(cache) if !matches!(cache, CacheCall::Cooldowns { .. } | CacheCall::Usage { .. } | CacheCall::HealthStates | CacheCall::Outcomes { .. })) }) .collect(); assert!(!writes.is_empty(), "the core wrote no outcome"); @@ -1504,13 +1610,7 @@ mod tests { let mut store = Store::default(); let mut driver = start(&pair, "primary"); let step = driver.resume(None).unwrap(); - let Step::HostCall( - call @ HostCall::Cache { - op: CacheOp::BatchGet, - .. - }, - ) = &step - else { + let Step::HostCall(call @ HostCall::Cache(CacheCall::Cooldowns { .. })) = &step else { panic!("expected the cooldown read, got {step:?}"); }; let error = driver.resume(Some(response)).unwrap_err(); @@ -1638,10 +1738,7 @@ mod tests { assert_eq!(model_group, "primary"); assert!(matches!( store.calls.as_slice(), - [HostCall::Cache { - op: CacheOp::BatchGet, - .. - }] + [HostCall::Cache(CacheCall::Cooldowns { .. })] )); } diff --git a/litellm/rust/crates/python-bridge/src/router/mod.rs b/litellm/rust/crates/python-bridge/src/router/mod.rs index 6b58d63a714..535717d8ed8 100644 --- a/litellm/rust/crates/python-bridge/src/router/mod.rs +++ b/litellm/rust/crates/python-bridge/src/router/mod.rs @@ -10,7 +10,7 @@ mod stream; use std::{ sync::{Arc, Mutex, PoisonError}, - time::{Duration, SystemTime}, + time::Duration, }; use litellm_error::Report; @@ -19,9 +19,8 @@ use litellm_router_core::lookup::{ListScope, Lookup, Row}; use litellm_router_core::{ AttemptError, Blocked, BuildPath, Built, Deployment, DeploymentSpec, Error, FailureKind, FailureReport, Op, OutcomeCallbacks, PreRouted, Provided, RetryCause, Router, RouterEvent, - SecretAt, SuccessReport, + SecretAt, SuccessReport, UsageWindow, build::Install, - health::{UsageMetric, current_minute, usage_key}, registry::{DeploymentFilter, ResolveKind, ResolveScope}, strategy::routing_context, }; @@ -32,6 +31,8 @@ use pyo3::{ }; use serde_json::Value; +use litellm_redis_store::router_usage::{Counter, Kind, Scope}; + use self::host::{ CacheOp, CacheOutcome, DriveError, Driver, Finish, HostCall, HostCallResponse, PythonRandomness, Stage, Step, StepHost, StepKind, @@ -384,15 +385,6 @@ impl RouterCoreHandle { .collect() } - /// The routing-cache key deployment `id` serving litellm `model` counts `metric` (`tpm`, - /// `rpm`, `itpm` or `otpm`) under in `minute` (`HH-MM`). - fn usage_key(&self, id: &str, model: &str, metric: &str, minute: &str) -> PyResult { - let metric: UsageMetric = metric - .parse() - .map_err(|_| PyValueError::new_err(format!("`{metric}` is not a usage metric")))?; - Ok(usage_key(id, model, metric, minute)) - } - /// Keep the providers `other`'s build resolved for the deployments this router serves too. fn adopt_providers(&self, other: &Self) { self.router.adopt_providers(&other.router); @@ -457,7 +449,8 @@ impl RouterCoreHandle { } /// Record a success a logging callback saw, driven like `report_failure`; it ends with - /// `done` (`key`: the tokens counter's key when usage was added, else None). + /// `done` (`key`: the tokens counter's key when usage was added, else None, as + /// `deployment_callback_on_success` returns it). /// `usage_model` is the model its usage is keyed by, and `tracks_usage` the host's answer to /// whether it counts any (see `Output::tracks_usage`). `callbacks`: see `report_failure`. #[pyo3(signature = (id, total_tokens, requests, usage_model=None, tracks_usage=None, callbacks=None))] @@ -481,12 +474,14 @@ impl RouterCoreHandle { router .report_success(&id, total_tokens, requests, report) .await - .map(|key| Finish::Counted { key }) + .map(|window| Finish::Counted { + key: window.map(tokens_key), + }) })) } - /// Count a success against `id`, configured or not; ends with `done` (`key`). `callbacks`: - /// see `report_failure`. + /// Count a success against `id`, configured or not; ends with `done` (`counted`: whether it + /// counted one). `callbacks`: see `report_failure`. #[pyo3(signature = (id, callbacks=None))] fn count_success( &self, @@ -500,30 +495,10 @@ impl RouterCoreHandle { router .count_success(&id, callbacks) .await - .map(|key| Finish::Counted { key }) + .map(|counted| Finish::SuccessCounted { counted }) })) } - /// The usage keys of this minute for every deployment of `model_group`: - /// `[(id, tpm_key, rpm_key)]`, or None when the group has no deployment. - fn usage_keys(&self, model_group: &str) -> Option> { - let minute = current_minute(SystemTime::now()); - let group = self.router.registry().group(model_group)?; - Some( - group - .iter() - .map(|deployment| { - let model = &deployment.params.model; - ( - deployment.id.clone(), - usage_key(&deployment.id, model, UsageMetric::Tpm, &minute), - usage_key(&deployment.id, model, UsageMetric::Rpm, &minute), - ) - }) - .collect(), - ) - } - /// Every configured deployment, as `model_list` entries with their ids, in config order. fn deployments<'py>(&self, py: Python<'py>) -> PyResult> { let registry = self.router.registry(); @@ -1241,7 +1216,7 @@ fn host_call_response( result => HostCallResponse::Attempt(result), }) } - HostCall::Cache { op, .. } => cache_outcome(*op, outcome).map(HostCallResponse::Cache), + HostCall::Cache(call) => cache_outcome(call.op(), outcome).map(HostCallResponse::Cache), HostCall::Secrets { .. } => Ok(HostCallResponse::Secrets(from_py_argument( &outcome.ok_or_else(missing)?, )?)), @@ -1311,36 +1286,43 @@ fn pre_route_outcome(outcome: Option>) -> PyResult>) -> PyResult { let outcome = outcome.filter(|outcome| !outcome.is_none()); let json = |value: Option| value.filter(|value| !value.is_null()); match op { - CacheOp::Get => outcome + CacheOp::HealthStates => outcome .map(|outcome| from_py_argument::(&outcome)) .transpose() .map(|value| CacheOutcome::Value(json(value))), - CacheOp::BatchGet => { + CacheOp::Cooldowns | CacheOp::Usage | CacheOp::Outcomes => { let outcome = outcome.ok_or_else(|| { - PyValueError::new_err("a `batch_get` cache outcome must be a list, not None") + PyValueError::new_err(format!("a `{op}` cache outcome must be a list, not None")) })?; if !outcome.is_instance_of::() { - return Err(PyValueError::new_err( - "a `batch_get` cache outcome must be a list with one entry per key", - )); + return Err(PyValueError::new_err(format!( + "a `{op}` cache outcome must be a list with one entry per value asked for" + ))); } let values: Vec> = from_py_argument(&outcome)?; Ok(CacheOutcome::Values(values.into_iter().map(json).collect())) } - CacheOp::Increment => outcome + CacheOp::IncrementAllowedFails => outcome .filter(|outcome| !outcome.is_instance_of::()) .and_then(|outcome| outcome.extract::().ok()) .map(CacheOutcome::Number) .ok_or_else(|| { - PyValueError::new_err("an `increment` cache outcome must be the new int or float") + PyValueError::new_err(format!( + "an `{op}` cache outcome must be the new int or float" + )) }), - CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage => match outcome { + CacheOp::PutCooldown + | CacheOp::AddUsage + | CacheOp::AddRequest + | CacheOp::CountSuccess + | CacheOp::CountFailure => match outcome { None => Ok(CacheOutcome::Stored), Some(_) => Err(PyValueError::new_err(format!( "a `{op}` cache step has no outcome, so pass None" @@ -1349,6 +1331,17 @@ fn cache_outcome(op: CacheOp, outcome: Option>) -> PyResult String { + Counter { + scope: Scope::GlobalRouter, + deployment_id: window.deployment_id, + model: Some(window.model), + window: window.minute, + kind: Kind::Tpm, + } + .key() +} + /// `{"ok": True, "total_tokens": int}` (an attempt's may add `"blocked": str`, `"deferred": bool` /// and `"tracks_usage": bool`) or `{"ok": False, "kind": str, "status": int | None, /// "retry_after": float | None, "num_retries": int | None, "detail": str | None, @@ -1808,20 +1801,8 @@ fn host_call_items(py: Python<'_>, dict: &Bound<'_, PyDict>, call: HostCall) -> dict.set_item("fallback_depth", fallback_depth)?; dict.set_item("retry", retry) } - HostCall::Cache { - op, - key, - keys, - value, - ttl, - local, - } => { - dict.set_item("op", <&'static str>::from(op))?; - dict.set_item("key", key)?; - dict.set_item("keys", keys.map(|keys| PyList::new(py, keys)).transpose()?)?; - dict.set_item("value", value.map(|value| to_py(py, &value)).transpose()?)?; - dict.set_item("ttl", ttl)?; - dict.set_item("local", local) + HostCall::Cache(call) => { + dict.update(to_py(py, &call)?.bind(py).cast::()?.as_mapping()) } HostCall::Secrets { index, refs } => { dict.set_item("index", index)?; @@ -1914,6 +1895,7 @@ fn done_items(dict: &Bound<'_, PyDict>, finish: Finish) -> PyResult<()> { dict.set_item("built", PyList::new(py, records)?) } Finish::Counted { key } => dict.set_item("key", key), + Finish::SuccessCounted { counted } => dict.set_item("counted", counted), Finish::RouterModelInfo(merged) => { let py = dict.py(); match merged { @@ -1977,21 +1959,42 @@ import time store = {} asked = [] +def add(key, amount): + store[key] = store.get(key, 0) + amount + return store[key] + +def window(step): + w = step["window"] + return (w["deployment_id"], w["model"], w["minute"]) + def answer_cache(step): asked.append(step) - op, key = step["op"], step["key"] - if op == "get": - return store.get(key) - if op == "batch_get": - return [store.get(k) for k in step["keys"]] - if op == "set": - store[key] = step["value"] + op = step["op"] + if op == "cooldowns": + return [store.get(("cooldown", id)) for id in step["ids"]] + if op == "put_cooldown": + store[("cooldown", step["id"])] = step["cooldown"] return None - if op == "increment": - store[key] = store.get(key, 0) + step["value"] - return store[key] - if op == "delete": - store.pop(key, None) + if op == "increment_allowed_fails": + return add(("allowed_fails", step["id"], step["suffix"]), 1) + if op == "add_usage": + add(("tokens", *window(step)), step["tokens"]) + add(("requests", *window(step)), step["requests"]) + return None + if op == "add_request": + add(("requests", *window(step)), 1) + return None + if op == "usage": + return [store.get(("tokens", *window(step))), store.get(("requests", *window(step)))] + if op == "health_states": + return None + if op == "outcomes": + return [store.get(("successes", step["id"])), store.get(("fails", step["id"]))] + if op == "count_success": + add(("successes", step["id"]), 1) + return None + if op == "count_failure": + add(("fails", step["id"]), 1) return None raise AssertionError(op) @@ -2004,8 +2007,9 @@ def drive(call, outcome): handle = native.RouterCoreHandle(config, {"num_retries": 1}) picked = drive(handle.select("primary", {}), None) assert (picked["kind"], picked["deployment_id"], picked["model_group"]) == ("done", "a1", "primary"), picked -assert [step["op"] for step in asked] == ["batch_get"] -assert list(asked[0]) == ["kind", "op", "key", "keys", "value", "ttl", "local", "events"], list(asked[0]) +assert [step["op"] for step in asked] == ["cooldowns"] +assert list(asked[0]) == ["kind", "op", "ids", "events"], list(asked[0]) +assert asked[0]["ids"] == ["a1"], asked[0] call = handle.start("acompletion", '{"model": "primary"}') version = handle.registry_version() assert handle.apply({"delete": ["gone"]}) == version + 1 @@ -2027,20 +2031,20 @@ assert cooled.resolve("primary")["ids"] == ["a1", "a2"] assert cooled.resolve("nope") is None assert drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})["kind"] == "failed" # The core records the failure itself: the 429 cools the deployment down in the host's store. -assert f"deployment:{tried}:cooldown" in store, store +assert ("cooldown", tried) in store, store other = drive(cooled.select("primary", {}), None) assert other["kind"] == "done" and other["deployment_id"] != tried, other call = handle.select("primary", {}) step = call.next(None) -assert step["op"] == "batch_get" +assert step["op"] == "cooldowns" for bad in ([], [None, None], "nope", None): try: call.next(bad) except ValueError: pass else: - raise AssertionError(f"{bad!r} should not answer a batch_get of one key") + raise AssertionError(f"{bad!r} should not answer a cooldowns read of one id") assert drive(call, [None])["kind"] == "done" "# ), diff --git a/litellm/rust/crates/python-bridge/src/routes/messages/transport.rs b/litellm/rust/crates/python-bridge/src/routes/messages/transport.rs index 12e782f61aa..8baed9b8a21 100644 --- a/litellm/rust/crates/python-bridge/src/routes/messages/transport.rs +++ b/litellm/rust/crates/python-bridge/src/routes/messages/transport.rs @@ -6,9 +6,9 @@ use litellm_host::{ machine::HostServices, protocol::{Protocol, Reply}, }; -use litellm_host_python::{json_to_py, to_py}; +use litellm_host_python::to_py; use litellm_inference_messages::{ - Delivered, Error, Transport, TransportBody, TransportRequest, TransportResponse, + Delivered, Error, Transport, TransportRequest, TransportResponse, }; use pyo3::{ exceptions::{PyBaseException, PyException, PyStopAsyncIteration}, @@ -160,10 +160,6 @@ impl HostedTransport { .handler .as_ref() .ok_or(litellm_host_python::Error::MissingState)?; - let body = match &sent.body { - TransportBody::Json(body) => json_to_py(py, body)?.unbind(), - TransportBody::Signed(bytes) => PyBytes::new(py, bytes).into_any().unbind(), - }; let awaitable = py .import(ROUTE_HOST_MODULE)? .getattr("transport_send")? @@ -172,7 +168,7 @@ impl HostedTransport { request, &sent.url, to_py(py, &sent.headers)?, - body, + PyBytes::new(py, &sent.body), sent.stream, ))?; self.pending = Some(Pending::Send(reply)); diff --git a/litellm/rust/crates/python-compat/src/number.rs b/litellm/rust/crates/python-compat/src/number.rs index 6e2f067ae25..c84e8dd613f 100644 --- a/litellm/rust/crates/python-compat/src/number.rs +++ b/litellm/rust/crates/python-compat/src/number.rs @@ -7,6 +7,36 @@ pub fn int(text: &str) -> Option { parse(text) } +/// `int(text, 16)`, or `None` where Python raises `ValueError`. +pub fn hex_int(text: &str) -> Option { + let text = text.trim(); + let (negative, unsigned) = match text.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, text.strip_prefix('+').unwrap_or(text)), + }; + let digits = match unsigned + .strip_prefix("0x") + .or_else(|| unsigned.strip_prefix("0X")) + { + Some(rest) => rest.strip_prefix('_').unwrap_or(rest), + None => unsigned, + }; + let hex_at = |index: Option| { + index + .and_then(|index| digits.as_bytes().get(index)) + .is_some_and(u8::is_ascii_hexdigit) + }; + let well_formed = digits.char_indices().all(|(index, c)| { + c.is_ascii_hexdigit() + || (c == '_' && hex_at(index.checked_sub(1)) && hex_at(Some(index + 1))) + }); + if digits.is_empty() || !well_formed { + return None; + } + let magnitude = BigInt::parse_bytes(digits.replace('_', "").as_bytes(), 16)?; + Some(if negative { -magnitude } else { magnitude }) +} + /// `float(text)`, or `None` where Python raises `ValueError`. pub fn float(text: &str) -> Option { parse(text) @@ -55,7 +85,7 @@ mod tests { use std::time::Duration; - use super::{SecondsError, float, int, seconds}; + use super::{SecondsError, float, hex_int, int, seconds}; // Expected values are CPython 3.14 int(text) #[rstest] @@ -78,6 +108,33 @@ mod tests { assert_eq!(int(text), expected.map(BigInt::from)); } + // Expected values are CPython 3.14 int(text, 16) + #[rstest] + #[case::lower("ff", Some(255))] + #[case::upper("FF", Some(255))] + #[case::surrounding_whitespace(" ff\n", Some(255))] + #[case::unicode_spaces("\u{a0}ff\u{2003}", Some(255))] + #[case::plus_sign("+ff", Some(255))] + #[case::minus_sign("-ff", Some(-255))] + #[case::prefix("0xff", Some(255))] + #[case::upper_prefix("0XFF", Some(255))] + #[case::underscore_after_prefix("0x_ff", Some(255))] + #[case::underscore_between_digits("f_f", Some(255))] + #[case::binary_looking("0b1", Some(177))] + #[case::double_underscore("f__f", None)] + #[case::leading_underscore("_ff", None)] + #[case::trailing_underscore("ff_", None)] + #[case::bare_prefix("0x", None)] + #[case::prefix_and_underscore("0x_", None)] + #[case::empty("", None)] + #[case::not_hex("g", None)] + #[case::information_separator("\x1cff", None)] + #[case::decimal_point("1.5", None)] + #[case::non_ascii_digit_is_a_known_limit("\u{667}", None)] + fn hex_int_parses_like_python(#[case] text: &str, #[case] expected: Option) { + assert_eq!(hex_int(text), expected.map(BigInt::from)); + } + // Expected values are CPython 3.14 float(text) #[rstest] #[case::underscore_and_newline(" 1_0.5\n", Some(10.5))] diff --git a/litellm/rust/crates/redis-client/src/client.rs b/litellm/rust/crates/redis-client/src/client.rs index d9a8b3af7e8..f0a7929e0c7 100644 --- a/litellm/rust/crates/redis-client/src/client.rs +++ b/litellm/rust/crates/redis-client/src/client.rs @@ -163,6 +163,10 @@ impl RedisClient { &self.0.namespace } + pub fn sibling_key(&self, key: &str, label: &str) -> Option { + self.0.namespace.sibling(key, label) + } + pub fn is_cluster(&self) -> bool { matches!(self.0.connection, Connection::Cluster(_)) } diff --git a/litellm/rust/crates/redis-client/src/namespace.rs b/litellm/rust/crates/redis-client/src/namespace.rs index 4db02b5b7bb..b615c24d9a9 100644 --- a/litellm/rust/crates/redis-client/src/namespace.rs +++ b/litellm/rust/crates/redis-client/src/namespace.rs @@ -19,6 +19,25 @@ impl Namespace { _ => key.to_owned(), } } + + /// A second key in the cluster slot of `key`, so one script may touch both. `None` when no hash + /// tag can name that slot: `key` is hashed whole and holds a `}`. + pub(crate) fn sibling(&self, key: &str, label: &str) -> Option { + let stored = self.apply(key); + let tag = hash_tag(&stored); + (!tag.contains('}')).then(|| format!("{{{tag}}}:{label}:{key}")) + } +} + +/// What Redis Cluster hashes: the text of the first non-empty `{...}`, else the whole key. +fn hash_tag(key: &str) -> &str { + match key + .split_once('{') + .and_then(|(_, rest)| rest.split_once('}')) + { + Some((tag, _)) if !tag.is_empty() => tag, + _ => key, + } } fn is_prefixed(key: &str, namespace: &str) -> bool { @@ -54,4 +73,27 @@ mod tests { ) { assert_eq!(namespace.apply(key), expected); } + + #[rstest] + #[case::whole_key(Namespace::default(), "team_id:t", Some("{team_id:t}:lease:team_id:t"))] + #[case::namespaced_whole_key( + Namespace::new("litellm"), + "team_id:t", + Some("{litellm:team_id:t}:lease:team_id:t") + )] + #[case::tagged_key( + Namespace::new("litellm"), + "{org}:alias", + Some("{org}:lease:{org}:alias") + )] + #[case::unclosed_brace(Namespace::default(), "alias:a{b", Some("{alias:a{b}:lease:alias:a{b"))] + #[case::empty_tag_with_closing_brace(Namespace::default(), "alias:a{}b", None)] + #[case::closing_brace_first(Namespace::default(), "alias:a}b{", None)] + fn a_sibling_key_names_the_slot_of_its_key( + #[case] namespace: Namespace, + #[case] key: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(namespace.sibling(key, "lease").as_deref(), expected); + } } diff --git a/litellm/rust/crates/redis-store/AGENTS.md b/litellm/rust/crates/redis-store/AGENTS.md index 963f7ff168f..2a243ba74e2 100644 --- a/litellm/rust/crates/redis-store/AGENTS.md +++ b/litellm/rust/crates/redis-store/AGENTS.md @@ -1,4 +1,4 @@ -- One folder per Redis family (`router_cooldowns/`, `router_usage/`, `spend_counters/`, `auth_cache/`, `rate_limits/`), each with `mod.rs` (the store and its ops), `keys.rs` (key types and spellings) and `types.rs` (the family's values, with the `FromValue` / `IntoValue` codecs for what it stores in Redis). It is the only place that family's keys, value codec and scripts are written, and it takes domain ids, never keys. Shared pieces live once at the crate root: `Error` and `ErrorKind` (`error.rs`), `Memory` and `Clock` (`memory.rs`), `Pending` (`request.rs`), `Remote` and its attach-once `RemoteSlot`, which knows nothing of memory (`remote.rs`), `DualCache` (`dual_cache/`) and the Python codecs (`codec.rs`, `value.rs`) +- One folder per Redis family (`router_cooldowns/`, `router_usage/`, `spend_counters/`, `auth_cache/`, `rate_limits/`, `deployment_health/`), each with `mod.rs` (the store and its ops), `keys.rs` (key types and spellings) and `types.rs` (the family's values, with the `FromValue` / `IntoValue` codecs for what it stores in Redis). It is the only place that family's keys, value codec and scripts are written, and it takes domain ids, never keys. Shared pieces live once at the crate root: `Error` and `ErrorKind` (`error.rs`), `Memory` and `Clock` (`memory.rs`), `Pending` (`request.rs`), `Remote` and its attach-once `RemoteSlot`, which knows nothing of memory (`remote.rs`), `DualCache` (`dual_cache/`) and the Python codecs (`codec.rs`, `value.rs`) - An op declared on a `Send` returns `Pending`, a lazy future built when the op is declared. An op that only means something on Redis (an `MGET` the caller falls back from, a script) returns `None` without one, and an op that replaced a `DualCache` call answers from memory. A post-call method returns `false` when there was no Redis to declare on, or `None` when it returns a settlement the caller waits on, like `SpendCounterStore::increment_many_post_call` - A store that sends a pipeline of its own uses `RedisBatch::pipelined`, never `Pipeline` and raw `Value`s - Python and Rust pods share these keys during a rolling deploy, so a module writes the bytes `main`'s Python writes and reads everything `main` reads. Python formats go through `litellm-python-compat` (`repr`, `literal_eval`, `json`), never a hand-rolled codec diff --git a/litellm/rust/crates/redis-store/Cargo.toml b/litellm/rust/crates/redis-store/Cargo.toml index 1dd61d76d3c..697c9b3f5f1 100644 --- a/litellm/rust/crates/redis-store/Cargo.toml +++ b/litellm/rust/crates/redis-store/Cargo.toml @@ -11,10 +11,13 @@ litellm-cache.workspace = true litellm-cache-memory.workspace = true litellm-python-compat.workspace = true litellm-redis-client.workspace = true +parking_lot.workspace = true +sha2.workspace = true strum.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } tracing.workspace = true +uuid.workspace = true [dev-dependencies] litellm-circuit-breaker.workspace = true diff --git a/litellm/rust/crates/redis-store/src/auth_cache/mod.rs b/litellm/rust/crates/redis-store/src/auth_cache/mod.rs index b5a83b5aec6..b181cafecc1 100644 --- a/litellm/rust/crates/redis-store/src/auth_cache/mod.rs +++ b/litellm/rust/crates/redis-store/src/auth_cache/mod.rs @@ -1,20 +1,19 @@ mod keys; mod types; -use std::{ - sync::{Arc, Mutex}, - time::Duration, -}; +use std::{sync::Arc, time::Duration}; use litellm_python_compat::{Value, json}; use litellm_redis_client::{Declared, RedisBatch, commands}; +use parking_lot::Mutex; pub use self::keys::is_key_object; pub use self::types::Settings; use crate::{ - Error, Number, Pending, RequestRedis, Send, + Error, Number, Pending, RequestRedis, Send, codec, dual_cache::DualCache, - memory::{Clock, lock, system_clock}, + failure_log::redis_failure, + memory::{Clock, system_clock}, remote::{Remote, RemoteSlot}, request::ready, }; @@ -22,6 +21,18 @@ use crate::{ /// `RedisCache.async_increment_if_present`, byte for byte, sent as `EVAL` like it. const INCREMENT_IF_PRESENT: &str = "if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"; +/// `SET` the entry only while its generation is the one the lease read. +const SET_LEASED: &str = "if (redis.call('GET', KEYS[2]) or '') ~= ARGV[1] then return 0 end redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3]) return 1"; + +/// The `DEL` and a new generation as one step, whatever order a batch sends its ops in. `DEL` +/// leads so a Redis at its memory limit still deletes the entry. +const DELETE_VOIDING_LEASES: &str = "local deleted = redis.call('DEL', KEYS[1]) redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[2]) return deleted"; + +const GENERATION_LABEL: &str = "auth_cache_generation"; + +/// Longer than any load a lease covers. A generation that expired reads like one never written. +const GENERATION_TTL: Duration = Duration::from_secs(600); + /// The proxy's auth object cache with `enable_redis_auth_cache` on: `UserApiKeyCache`'s two /// `DualCache` partitions, so churn in management objects cannot evict user-key objects, in front of /// one Redis shared by every pod. Keys are the logical cache keys `UserApiKeyCache` callers already @@ -54,13 +65,13 @@ impl AuthCache { } pub fn settings(&self) -> Settings { - *lock(&self.settings) + *self.settings.lock() } /// `UserApiKeyCache.update_cache_ttl` and `update_in_memory_max_size` on both partitions, which /// keep what they hold. pub fn reconfigure(&self, settings: Settings) { - *lock(&self.settings) = settings; + *self.settings.lock() = settings; self.key_objects.reconfigure(settings.memory()); self.objects.reconfigure(settings.memory()); } @@ -130,9 +141,85 @@ impl AuthCache { self.partition(key).declare_set(send, key, value, ttl) } - /// `DualCache.async_delete_cache_pre_call`, memory now and the `DEL` on the batch `send` picks. + /// `DualCache.async_delete_cache_pre_call`, memory now and the `DEL` on the batch `send` picks, + /// in one script with the write that voids the key's leases. pub fn declare_delete(&self, send: Send<'_>, key: &str) -> Result, Error> { - self.partition(key).declare_delete(send, key) + let partition = self.partition(key); + let Some((client, generation)) = self + .redis + .client() + .and_then(|client| Some((client, client.sibling_key(key, GENERATION_LABEL)?))) + else { + return partition.declare_delete(send, key); + }; + partition.memory().delete(key)?; + if let Some(misses) = send.misses(client) { + misses.note([key.to_owned()]); + } + let deleted = send.batch(client).add(commands::eval_source::( + DELETE_VOIDING_LEASES, + &[key, &generation], + (new_generation(), GENERATION_TTL.as_secs()), + )); + Ok(Box::pin( + async move { Ok(deleted.resolve().await.map(drop)?) }, + )) + } + + /// The generation of `key` every pod shares, read on a cache miss before the database read + /// that will fill it and handed to [`AuthCache::set_leased`]. Every delete of the key starts a + /// new one. `None` without Redis, before the first delete, and when Redis did not answer. + pub async fn lease(&self, key: &str) -> Option { + let client = self.redis.client()?; + let generation = client.sibling_key(key, GENERATION_LABEL)?; + client + .send(commands::get::(&generation)) + .await + .unwrap_or_else(|error| { + redis_failure!(&error, "an auth cache lease was not read from Redis"); + None + }) + } + + /// The Redis `SET` of `key`, refused when the key was deleted on any pod since `lease` was + /// read: the value may predate the write behind that delete. Memory is the caller's to write. + /// Also refused for a key no generation key can share a slot with, which no delete could void. + /// `true` without Redis, and when Redis failed or `json.dumps` refuses the value, which only + /// log as in [`AuthCache::set`]. + pub async fn set_leased( + &self, + key: &str, + lease: Option<&str>, + value: Value, + ttl: Option, + ) -> bool { + let Some(remote) = self.redis.get() else { + return true; + }; + let ttl = ttl + .or(self.partition(key).default_in_memory_ttl()) + .unwrap_or(remote.default_ttl); + let written = match codec::dumps(value) { + Err(error) => Err(error), + Ok(encoded) => match remote.client.sibling_key(key, GENERATION_LABEL) { + None => Ok(false), + Some(generation) => { + remote + .client + .send(commands::eval_source( + SET_LEASED, + &[key, &generation], + (lease.unwrap_or_default(), encoded, ttl.as_secs()), + )) + .await + } + } + .map_err(Error::from), + }; + written.unwrap_or_else(|error| { + redis_failure!(&error, "LiteLLM Cache: exception in async add_cache"); + true + }) } /// `UserApiKeyCache.async_set_cache_pipeline`: the key-object entries in one pipeline, then @@ -196,16 +283,45 @@ impl AuthCache { })) } - /// `UserApiKeyCache.async_delete_cache_keys`: the key-object partition and the rest as two - /// concurrent `DualCache.async_delete_cache_keys`. Both run even when one fails. + /// `UserApiKeyCache.async_delete_cache_keys`. Memory is dropped before the first wait on + /// Redis. A key deletes in one script with the write that voids its leases, all of them in one + /// pipeline. Keys no generation can share a slot with go as `DualCache.async_delete_cache_keys` + /// per partition. Every part runs even when another fails. pub async fn delete(&self, keys: &[String]) -> Result<(), Error> { - let (key_objects, others): (Vec, Vec) = - keys.iter().cloned().partition(|key| is_key_object(key)); - let (key_objects, others) = tokio::join!( + keys.iter().try_for_each(|key| self.delete_local(key))?; + let Some(client) = self.redis.client() else { + return Ok(()); + }; + let pipeline = RedisBatch::pipelined(client.clone()); + let (leased, plain): (Vec<_>, Vec<_>) = keys + .iter() + .map(|key| (key, client.sibling_key(key, GENERATION_LABEL))) + .partition(|(_, generation)| generation.is_some()); + let deleted: Vec<_> = leased + .into_iter() + .filter_map(|(key, generation)| Some((key, generation?))) + .map(|(key, generation)| { + pipeline.add(commands::eval_source::( + DELETE_VOIDING_LEASES, + &[key, &generation], + (new_generation(), GENERATION_TTL.as_secs()), + )) + }) + .collect(); + let (key_objects, others): (Vec, Vec) = plain + .into_iter() + .map(|(key, _)| key.clone()) + .partition(|key| is_key_object(key)); + let (deleted, key_objects, others) = tokio::join!( + Declared::all(deleted), self.key_objects.delete_keys(&key_objects), self.objects.delete_keys(&others) ); - key_objects.and(others) + deleted + .into_iter() + .try_for_each(|reply| Ok(reply.map(drop)?)) + .and(key_objects) + .and(others) } /// `RedisCache.async_batch_get_cache` for keys the caller found missing in memory, one `MGET` on @@ -256,3 +372,7 @@ impl AuthCache { } } } + +fn new_generation() -> String { + uuid::Uuid::new_v4().to_string() +} diff --git a/litellm/rust/crates/redis-store/src/deployment_health/keys.rs b/litellm/rust/crates/redis-store/src/deployment_health/keys.rs new file mode 100644 index 00000000000..871e49b3957 --- /dev/null +++ b/litellm/rust/crates/redis-store/src/deployment_health/keys.rs @@ -0,0 +1,2 @@ +/// `DeploymentHealthCache.CACHE_KEY`: one entry holds every deployment's state. +pub(super) const STATES: &str = "litellm:health_check:deployment_health_state"; diff --git a/litellm/rust/crates/redis-store/src/deployment_health/mod.rs b/litellm/rust/crates/redis-store/src/deployment_health/mod.rs new file mode 100644 index 00000000000..9c4530e27ed --- /dev/null +++ b/litellm/rust/crates/redis-store/src/deployment_health/mod.rs @@ -0,0 +1,78 @@ +mod keys; +mod types; + +use std::{sync::Arc, time::Duration}; + +use litellm_redis_client::commands; + +pub use self::types::{HealthState, HealthStates}; +use crate::{ + Error, codec, + dual_cache::{DualCache, MemorySettings}, + memory::{Clock, system_clock}, + remote::{Remote, RemoteSlot}, +}; + +/// The background health checks' per-deployment states, kept by `DeploymentHealthCache` in the +/// router's `DualCache`: memory per worker, Redis shared by every pod. The writer's merge and the +/// readers' staleness rules stay with the host. +pub struct DeploymentHealth { + states: DualCache, +} + +impl DeploymentHealth { + pub fn new(remote: Option, default_in_memory_ttl: Option) -> Self { + Self::with_clock(remote, default_in_memory_ttl, system_clock()) + } + + pub fn with_clock( + remote: Option, + default_in_memory_ttl: Option, + now: Clock, + ) -> Self { + Self { + states: DualCache::new( + Arc::new(RemoteSlot::new(remote)), + MemorySettings { + entries: None, + ttl: default_in_memory_ttl, + }, + now, + ), + } + } + + /// `DualCache.attach_redis_cache`: memory stays, and `false` means a Redis was already attached. + pub fn attach(&self, remote: Remote) -> bool { + self.states.redis().attach(remote) + } + + pub fn has_redis(&self) -> bool { + self.states.redis().get().is_some() + } + + /// `DualCache.get_cache`: memory, else Redis written back to memory. + pub async fn get(&self) -> Result, Error> { + self.states.get(None, keys::STATES, None).await + } + + /// `redis_cache.get_cache`, what every pod merged so far. No Redis, a failed read and an open + /// breaker all read as `None`. + pub async fn get_shared(&self) -> Option { + let client = self.states.redis().client()?; + client + .send(commands::get::>(keys::STATES)) + .await + .map_err(Error::from) + .and_then(|raw| Ok(raw.as_deref().map(codec::decode).transpose()?)) + .unwrap_or_else(|error| { + crate::failure_log::redis_failure!(&error, "LiteLLM Cache: exception in get_cache"); + None + }) + } + + /// `DualCache.set_cache`, whose sync `RedisCache.set_cache` writes `str(value)`. + pub async fn set(&self, states: HealthStates, ttl: Duration) -> Result<(), Error> { + self.states.set_str(keys::STATES, states, Some(ttl)).await + } +} diff --git a/litellm/rust/crates/redis-store/src/deployment_health/types.rs b/litellm/rust/crates/redis-store/src/deployment_health/types.rs new file mode 100644 index 00000000000..977a6a39a08 --- /dev/null +++ b/litellm/rust/crates/redis-store/src/deployment_health/types.rs @@ -0,0 +1,80 @@ +use litellm_python_compat::{Value, truthy::truthy}; + +use crate::{DecodeError, FromValue, IntoValue, Number}; + +/// One background health check's result, the `DeploymentHealthStateValue` dict the proxy writes. +#[derive(Clone, Debug, PartialEq)] +pub struct HealthState { + pub is_healthy: bool, + pub timestamp: Number, + pub reason: String, +} + +/// Every deployment's last health check by id, in the order the writer merged them. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct HealthStates(pub Vec<(String, HealthState)>); + +impl HealthState { + /// What main's readers take from a record: extra fields are ignored, any falsy flag is + /// unhealthy and a missing one healthy, and a timestamp that is no number reads as `0` (stale). + fn read(fields: &[(Value, Value)]) -> Self { + let field = |name: &str| { + fields + .iter() + .find(|(key, _)| matches!(key, Value::Str(key) if key == name)) + .map(|(_, value)| value) + }; + Self { + is_healthy: field("is_healthy").is_none_or(truthy), + timestamp: field("timestamp") + .and_then(Number::from_value) + .unwrap_or(Number::Float(0.0)), + reason: match field("reason") { + Some(Value::Str(reason)) => reason.clone(), + _ => String::new(), + }, + } + } +} + +impl IntoValue for HealthState { + fn into_value(self) -> Value { + let field = |name: &str, value: Value| (Value::Str(name.to_owned()), value); + Value::Dict(vec![ + field("is_healthy", Value::Bool(self.is_healthy)), + field("timestamp", self.timestamp.to_value()), + field("reason", Value::Str(self.reason)), + ]) + } +} + +/// A record that is no dict, under a key that is no string, is dropped alone. +impl FromValue for HealthStates { + fn from_value(value: &Value) -> Result { + let Value::Dict(records) = value else { + return Err(DecodeError::Shape("deployment health states are a dict")); + }; + Ok(Self( + records + .iter() + .filter_map(|record| match record { + (Value::Str(id), Value::Dict(fields)) => { + Some((id.clone(), HealthState::read(fields))) + } + _ => None, + }) + .collect(), + )) + } +} + +impl IntoValue for HealthStates { + fn into_value(self) -> Value { + Value::Dict( + self.0 + .into_iter() + .map(|(id, state)| (Value::Str(id), state.into_value())) + .collect(), + ) + } +} diff --git a/litellm/rust/crates/redis-store/src/dual_cache/batch_read.rs b/litellm/rust/crates/redis-store/src/dual_cache/batch_read.rs index 71dfda49233..8196679b518 100644 --- a/litellm/rust/crates/redis-store/src/dual_cache/batch_read.rs +++ b/litellm/rust/crates/redis-store/src/dual_cache/batch_read.rs @@ -1,13 +1,13 @@ use std::{ collections::{HashMap, VecDeque}, - sync::Mutex, time::Duration, }; -use litellm_redis_client::{Declared, Error as RedisError, commands}; +use litellm_redis_client::{Declared, Error as RedisError, RedisBatch, commands}; +use parking_lot::Mutex; use super::{DualCache, MemorySettings}; -use crate::{Error, FromValue, Send, codec, memory::lock, request::Misses}; +use crate::{Error, FromValue, Send, codec, request::Misses}; /// A batch-reading family's `DualCache` settings, as the host reads them from `litellm` globals. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -66,9 +66,11 @@ impl DualCache = reservation.keys.iter().map(String::as_str).collect(); - let replies = send.batch(client).add(commands::mget::>(&keys)); + let batch = send.batch(client); + let replies = batch.add(commands::mget::>(&keys)); Fetch { reservation, + batch, replies, misses: send.misses(client), } @@ -81,24 +83,37 @@ impl DualCache Result>, Error> { + if let Some(batch) = declared.batch() { + batch.flush().await; + } + self.settle_batch_get(declared).await + } + + /// The second half of `batch_get` once the read's batch answers, without flushing it. Memory + /// is read again here, so a value written to memory since the declaration wins over what Redis + /// answered. The keys Redis lacks are noted as the request's misses. An undecodable value + /// serves memory for the whole read, where `main`'s `RedisCache` returns the raw bytes and + /// `DualCache` backfills memory with them. + pub(crate) async fn settle_batch_get( + &self, + declared: DeclaredRead, ) -> Result>, Error> { let DeclaredRead { keys, fetch } = declared; let Some(Fetch { reservation, + batch: _, replies, misses, }) = fetch else { return self.read_memory(&keys); }; - let fetched = replies.resolve().await; + let fetched = replies.settled().await; let local = self.read_memory(&keys)?; let fetched = match fetched { Ok(fetched) => fetched, @@ -156,7 +171,7 @@ impl DualCache]) -> Reservation { let now = self.now(); let refresh_after = self.reads.refresh_after; - let mut recent = lock(&self.reads.recent); + let mut recent = self.reads.recent.lock(); let due = keys .iter() .zip(local) @@ -174,8 +189,8 @@ impl DualCache recent.insert(key, read), @@ -191,13 +206,27 @@ pub(crate) struct DeclaredRead { fetch: Option, } +impl DeclaredRead { + /// The batch carrying the read, `None` when memory answers it alone. + pub(crate) fn batch(&self) -> Option { + self.fetch.as_ref().map(|fetch| fetch.batch.clone()) + } + + /// The refreshes the read reserved, `None` when memory answers it alone. + pub(crate) fn reservation(&self) -> Option { + self.fetch.as_ref().map(|fetch| fetch.reservation.clone()) + } +} + struct Fetch { reservation: Reservation, + batch: RedisBatch, replies: Declared>>>, misses: Option, } -struct Reservation { +#[derive(Clone)] +pub(crate) struct Reservation { keys: Vec, previous: Vec>, } diff --git a/litellm/rust/crates/redis-store/src/dual_cache/increment.rs b/litellm/rust/crates/redis-store/src/dual_cache/increment.rs index 705bc71b2c1..d565cdb91e0 100644 --- a/litellm/rust/crates/redis-store/src/dual_cache/increment.rs +++ b/litellm/rust/crates/redis-store/src/dual_cache/increment.rs @@ -1,7 +1,7 @@ use std::time::Duration; use futures_util::future::join_all; -use litellm_redis_client::{Declared, Expiry, RedisBatch, commands}; +use litellm_redis_client::{Declared, Error as RedisError, Expiry, RedisBatch, commands}; use super::DualCache; use crate::{Error, Number, RequestRedis, Send, failure_log::redis_failure, memory::Counter}; @@ -63,10 +63,26 @@ impl DualCache { .collect::, _>>() .map_err(|error| redis_failure!(&Error::from(error), PIPELINE_FAILED)) .ok()?; - let Some(client) = self.redis.client() else { - return Some(local); - }; - let pipeline = RedisBatch::pipelined(client.clone()); + match self.increment_pipeline_remote(increments).await { + None => Some(local), + Some(Ok(totals)) => Some(totals), + Some(Err(error)) => { + redis_failure!(&error, PIPELINE_FAILED); + Some(local) + } + } + } + + /// `RedisCache.async_increment_pipeline`, Redis alone: one pipeline of `INCRBYFLOAT` with + /// `EXPIRE` per increment. `None` without Redis or with nothing to increment. + pub(crate) async fn increment_pipeline_remote( + &self, + increments: &[PipelineIncrement], + ) -> Option, RedisError>> { + if increments.is_empty() { + return None; + } + let pipeline = RedisBatch::pipelined(self.redis.client()?.clone()); let totals = increments .iter() .map(|increment| { @@ -77,17 +93,13 @@ impl DualCache { )) }) .collect::>(); - match Declared::all(totals) - .await - .into_iter() - .collect::, _>>() - { - Ok(totals) => Some(totals.into_iter().map(Number::Float).collect()), - Err(error) => { - redis_failure!(&error, PIPELINE_FAILED); - Some(local) - } - } + Some( + Declared::all(totals) + .await + .into_iter() + .map(|total| total.map(Number::Float)) + .collect(), + ) } /// `DualCache.async_increment_cache_pipeline_post_call`: memory now, stopping at the first diff --git a/litellm/rust/crates/redis-store/src/dual_cache/mod.rs b/litellm/rust/crates/redis-store/src/dual_cache/mod.rs index b2c277a118e..36b090bc152 100644 --- a/litellm/rust/crates/redis-store/src/dual_cache/mod.rs +++ b/litellm/rust/crates/redis-store/src/dual_cache/mod.rs @@ -3,15 +3,17 @@ mod increment; mod read; mod write; -use std::{ - sync::{Arc, Mutex}, - time::Duration, -}; +use std::{sync::Arc, time::Duration}; + +use parking_lot::Mutex; pub use self::batch_read::TierPolicy; -pub(crate) use self::{batch_read::BatchReads, increment::PipelineIncrement}; +pub(crate) use self::{ + batch_read::{BatchReads, Reservation}, + increment::PipelineIncrement, +}; use crate::{ - memory::{Clock, Memory, lock}, + memory::{Clock, Memory}, remote::RemoteSlot, }; @@ -83,13 +85,13 @@ impl DualCache { } pub(crate) fn default_in_memory_ttl(&self) -> Option { - *lock(&self.default_in_memory_ttl) + *self.default_in_memory_ttl.lock() } /// `DualCache.update_cache_ttl` and `update_in_memory_max_size` together: both apply in place, /// and the entries memory already holds stay. pub(crate) fn reconfigure(&self, memory: MemorySettings) { - *lock(&self.default_in_memory_ttl) = memory.ttl; + *self.default_in_memory_ttl.lock() = memory.ttl; self.memory.update_max_size(memory.entries); } } @@ -97,10 +99,7 @@ impl DualCache { #[cfg(test)] mod tests { use std::{ - sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, + sync::{Arc, atomic::Ordering}, time::Duration, }; @@ -108,7 +107,9 @@ mod tests { use tokio::runtime::Handle; use super::{DualCache, MemorySettings, PipelineIncrement}; - use crate::{Clock, Number, RequestRedis, memory::Counter, remote::RemoteSlot}; + use crate::{ + Clock, Number, RequestRedis, memory::Counter, remote::RemoteSlot, test_support::clock, + }; #[derive(Clone, Debug, PartialEq)] enum Entry { @@ -131,13 +132,6 @@ mod tests { } } - fn clock() -> (Clock, Arc) { - let seconds = Arc::new(AtomicU64::new(1_000)); - let read = seconds.clone(); - let clock: Clock = Arc::new(move || Duration::from_secs(read.load(Ordering::SeqCst))); - (clock, seconds) - } - fn without_redis( default_in_memory_ttl: Option, now: Clock, @@ -252,6 +246,23 @@ mod tests { assert_eq!(cache.memory().get("short").unwrap(), None); } + #[rstest] + #[tokio::test] + async fn reconfigure_applies_the_new_capacity_to_the_next_write() { + let (now, _) = clock(); + let cache = without_redis(None, now); + cache.set("first", Number::Int(1), None).await.unwrap(); + + cache.reconfigure(MemorySettings { + entries: Some(1), + ttl: None, + }); + cache.set("second", Number::Int(2), None).await.unwrap(); + + assert_eq!(cache.memory().len().unwrap(), 1); + assert_eq!(cache.memory().get("second").unwrap(), Some(Number::Int(2))); + } + // async_increment_cache_pipeline_post_call: InMemoryCache.async_increment_pipeline raises at the // first entry it cannot add to, so later increments never reach memory, and DualCache only logs #[tokio::test] diff --git a/litellm/rust/crates/redis-store/src/lib.rs b/litellm/rust/crates/redis-store/src/lib.rs index 69600aac2a6..46c72203599 100644 --- a/litellm/rust/crates/redis-store/src/lib.rs +++ b/litellm/rust/crates/redis-store/src/lib.rs @@ -1,5 +1,6 @@ pub mod auth_cache; mod codec; +pub mod deployment_health; mod dual_cache; mod error; mod failure_log; @@ -9,9 +10,12 @@ pub mod rate_limits; mod remote; mod request; pub mod router_cooldowns; +pub mod router_session_pins; pub mod router_usage; pub mod sensitive_route_pins; pub mod spend_counters; +#[cfg(test)] +mod test_support; mod value; pub use codec::DecodeError; @@ -20,5 +24,5 @@ pub use error::{Error, ErrorKind}; pub use memory::{Clock, system_clock}; pub use number::Number; pub use remote::Remote; -pub use request::{Pending, RequestRedis, Send}; +pub use request::{DeadlineTag, Pending, RequestRedis, Send}; pub use value::{FromValue, IntoValue}; diff --git a/litellm/rust/crates/redis-store/src/memory.rs b/litellm/rust/crates/redis-store/src/memory.rs index f89d5926760..b67b6d7fe59 100644 --- a/litellm/rust/crates/redis-store/src/memory.rs +++ b/litellm/rust/crates/redis-store/src/memory.rs @@ -1,10 +1,10 @@ use std::{ - sync::{Arc, Mutex, MutexGuard, PoisonError}, + sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; use litellm_cache::Error; -use litellm_cache_memory::InMemoryCache; +use litellm_cache_memory::{Expiry, InMemoryCache}; use crate::Number; @@ -18,11 +18,9 @@ pub fn system_clock() -> Clock { }) } -/// A store's memory tier: Python's `InMemoryCache`, with its 200 entries and 600 s TTL unless the -/// store names others, plus one lock for every read-modify-write on it. +/// A store's memory tier: 200 entries and a 600 s TTL unless the store names others. pub struct Memory { cache: InMemoryCache, - updates: Mutex<()>, } impl Memory { @@ -30,7 +28,6 @@ impl Memory { let now = now.clone(); Self { cache: InMemoryCache::with_clock(entries, None, move || now()), - updates: Mutex::new(()), } } @@ -38,8 +35,30 @@ impl Memory { self.cache.get_cache(key) } + /// A key memory already holds keeps its deadline, as a counter keeps its window. pub fn set(&self, key: &str, value: V, ttl: Option) -> Result<(), Error> { - self.cache.set_cache(key, value, ttl).map(drop) + self.cache + .set_cache(key, value, Expiry::KeepExisting(ttl)) + .map(drop) + } + + pub fn refresh(&self, key: &str, value: V, ttl: Duration) -> Result<(), Error> { + self.cache + .set_cache(key, value, Expiry::Refresh(Some(ttl))) + .map(drop) + } + + /// Writes `value` when memory holds nothing for `key` or `replaces` what it holds. + pub fn set_if( + &self, + key: &str, + value: V, + ttl: Option, + replaces: impl FnOnce(&V) -> bool, + ) -> Result<(), Error> { + self.cache.update(key, Expiry::KeepExisting(ttl), |held| { + Ok((held.is_none_or(replaces).then_some(value), ())) + }) } pub fn delete(&self, key: &str) -> Result<(), Error> { @@ -67,26 +86,21 @@ impl Memory { self.cache.update_max_size_in_memory(entries); } - /// Read-modify-write of one entry, atomic against every other update of this tier. + /// Read-modify-write of one entry, atomic against every other read and write of this tier. pub fn update( &self, key: &str, ttl: Option, update: impl FnOnce(Option) -> Result, ) -> Result { - let _guard = self.guard(); - let value = update(self.get(key)?)?; - self.set(key, value.clone(), ttl)?; - Ok(value) - } - - /// Held across a read and a conditional write that `update` cannot express. - pub fn guard(&self) -> MutexGuard<'_, ()> { - lock(&self.updates) + self.cache.update(key, Expiry::KeepExisting(ttl), |held| { + let value = update(held.cloned())?; + Ok((Some(value.clone()), value)) + }) } } -/// A value `InMemoryCache.increment_cache` can add to. +/// A value `Memory::increment` can add to. pub(crate) trait Counter: Clone + From { fn as_number(&self) -> Option; } @@ -98,25 +112,130 @@ impl Counter for Number { } impl Memory { - /// `InMemoryCache.increment_cache`: `(current or 0) + amount`, refusing an entry that is not a - /// number and leaving it in place. + /// `(current or 0) + amount`, refusing an entry that is not a number and leaving it in place. pub(crate) fn increment( &self, key: &str, amount: Number, ttl: Option, ) -> Result { - let _guard = self.guard(); - let current = match self.get(key)? { - None => Number::Int(0), - Some(value) => value.as_number().ok_or(Error::InvalidEntry(None))?, - }; - let total = current + amount; - self.set(key, V::from(total), ttl)?; - Ok(total) + self.cache.update(key, Expiry::KeepExisting(ttl), |held| { + let current = match held { + None => Number::Int(0), + Some(value) => value.as_number().ok_or(Error::InvalidEntry(None))?, + }; + let total = current + amount; + Ok((Some(V::from(total)), total)) + }) } } -pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { - mutex.lock().unwrap_or_else(PoisonError::into_inner) +#[cfg(test)] +mod tests { + use std::{sync::atomic::Ordering, time::Duration}; + + use rstest::rstest; + + use super::Memory; + use crate::{ + Number, + test_support::{START, clock}, + }; + + type Write = fn(&Memory); + + fn secs(seconds: u64) -> Duration { + Duration::from_secs(seconds) + } + + fn ttl(seconds: u64) -> Option { + Some(secs(seconds)) + } + + fn after(seconds: u64) -> Option { + ttl(START + seconds) + } + + fn set(memory: &Memory) { + memory.set("key", Number::Int(2), ttl(50)).unwrap(); + } + + fn set_if(memory: &Memory) { + memory + .set_if("key", Number::Int(2), ttl(50), |_| true) + .unwrap(); + } + + fn update(memory: &Memory) { + memory + .update("key", ttl(50), |_| Ok(Number::Int(2))) + .unwrap(); + } + + fn increment(memory: &Memory) { + memory.increment("key", Number::Int(2), ttl(50)).unwrap(); + } + + fn refresh(memory: &Memory) { + memory.refresh("key", Number::Int(2), secs(50)).unwrap(); + } + + #[rstest] + #[case::set(set, 10)] + #[case::set_if(set_if, 10)] + #[case::update(update, 10)] + #[case::increment(increment, 10)] + #[case::refresh(refresh, 55)] + fn a_write_to_a_held_key_keeps_its_deadline_unless_it_refreshes( + #[case] write: Write, + #[case] expires_after: u64, + ) { + let (now, seconds) = clock(); + let memory = Memory::new(None, &now); + memory.set("key", Number::Int(0), ttl(10)).unwrap(); + seconds.fetch_add(5, Ordering::SeqCst); + + write(&memory); + + assert_eq!(memory.get("key").unwrap(), Some(Number::Int(2))); + assert_eq!(memory.expires_at("key").unwrap(), after(expires_after)); + } + + #[rstest] + #[case::set(set)] + #[case::set_if(set_if)] + #[case::update(update)] + #[case::increment(increment)] + #[case::refresh(refresh)] + fn a_write_to_a_new_key_takes_its_ttl(#[case] write: Write) { + let (now, _) = clock(); + let memory = Memory::new(None, &now); + + write(&memory); + + assert_eq!(memory.get("key").unwrap(), Some(Number::Int(2))); + assert_eq!(memory.expires_at("key").unwrap(), after(50)); + } + + #[rstest] + #[case::nothing_held(None, false, 2)] + #[case::a_value_it_replaces(Some(Number::Int(1)), true, 2)] + #[case::a_value_it_does_not_replace(Some(Number::Int(1)), false, 1)] + fn set_if_writes_over( + #[case] held: Option, + #[case] replaces: bool, + #[case] expected: i64, + ) { + let (now, _) = clock(); + let memory = Memory::new(None, &now); + if let Some(held) = held { + memory.set("key", held, None).unwrap(); + } + + memory + .set_if("key", Number::Int(2), None, |_| replaces) + .unwrap(); + + assert_eq!(memory.get("key").unwrap(), Some(Number::Int(expected))); + } } diff --git a/litellm/rust/crates/redis-store/src/number.rs b/litellm/rust/crates/redis-store/src/number.rs index 7f3e9f4d10b..ae65461e542 100644 --- a/litellm/rust/crates/redis-store/src/number.rs +++ b/litellm/rust/crates/redis-store/src/number.rs @@ -70,13 +70,16 @@ impl Number { } } -/// Python's `+`: two ints stay an int, anything with a float is a float. +/// Python's `+`: two ints stay an int, anything with a float is a float. An int sum past `i64` +/// becomes the nearest float, which still compares above any limit the way Python's int does. impl Add for Number { type Output = Self; fn add(self, other: Self) -> Self { match (self, other) { - (Self::Int(left), Self::Int(right)) => Self::Int(left + right), + (Self::Int(left), Self::Int(right)) => left + .checked_add(right) + .map_or_else(|| Self::Float(left as f64 + right as f64), Self::Int), (left, right) => Self::Float(left.as_f64() + right.as_f64()), } } diff --git a/litellm/rust/crates/redis-store/src/request.rs b/litellm/rust/crates/redis-store/src/request.rs index 79497a5d503..fbe0dc63b86 100644 --- a/litellm/rust/crates/redis-store/src/request.rs +++ b/litellm/rust/crates/redis-store/src/request.rs @@ -1,17 +1,11 @@ -use std::{ - collections::HashSet, - future::Future, - sync::{Arc, Mutex}, - time::Duration, -}; +use std::{collections::HashSet, future::Future, sync::Arc, time::Duration}; use futures_util::future::{BoxFuture, join_all}; use litellm_redis_client::{CallTag, Declared, RedisBatch, RedisClient, current_tag, tagged}; +use parking_lot::Mutex; use tokio::{runtime::Handle, sync::oneshot, task::JoinHandle}; use tracing::{Dispatch, Instrument, instrument::WithSubscriber}; -use crate::memory::lock; - /// One request's Redis batches: a pre-call batch for the reads it needs before answering and a /// post-call batch for the writes nobody waits on, one of each per Redis client. The host that owns /// the request builds one and passes it down; it knows nothing else about the request. @@ -22,7 +16,7 @@ struct Shared { runtime: Handle, logs: Dispatch, post_call_deadline: Duration, - deadline_tag: Option, + deadline_tag: DeadlineTag, batches: Mutex>, follow_ups: Mutex>>, deadline: Mutex>>, @@ -43,15 +37,15 @@ pub(crate) struct Misses(Arc>>); impl Misses { pub(crate) fn contains(&self, key: &str) -> bool { - lock(&self.0).contains(key) + self.0.lock().contains(key) } pub(crate) fn note(&self, keys: impl IntoIterator) { - lock(&self.0).extend(keys); + self.0.lock().extend(keys); } pub(crate) fn forget(&self, key: &str) { - lock(&self.0).remove(key); + self.0.lock().remove(key); } } @@ -60,15 +54,16 @@ impl RequestRedis { /// Python one that has no Tokio context. Work spawned for the request logs to the subscriber /// current here, since a declaring thread may have none. pub fn new(runtime: Handle, post_call_deadline: Duration) -> Self { - Self::with_deadline_tag(runtime, post_call_deadline, None) + Self::with_deadline_tag(runtime, post_call_deadline, Arc::new(|| None)) } - /// As `new`, with the deadline's flush of the post-call batch run inside `tagged(deadline_tag)`, - /// the scope an explicit `flush_post` runs in. + /// As `new`, with the deadline's flush of the post-call batch run inside the tag `deadline_tag` + /// gives for the declaration that armed it, as an explicit `flush_post` runs inside its + /// caller's. pub fn with_deadline_tag( runtime: Handle, post_call_deadline: Duration, - deadline_tag: Option, + deadline_tag: DeadlineTag, ) -> Self { Self(Arc::new(Shared { runtime, @@ -104,7 +99,17 @@ impl RequestRedis { .0 .runtime .spawn(self.logged(within(current_tag(), task))); - lock(&self.0.follow_ups).push(follow_up); + self.0.follow_ups.lock().push(follow_up); + } + + /// Runs `task` on the request's runtime without `flush_post` waiting for it, for work that + /// settles with the pre-call batch whoever awaits it. + pub(crate) fn detach(&self, task: impl Future + std::marker::Send + 'static) { + drop( + self.0 + .runtime + .spawn(self.logged(within(current_tag(), task))), + ); } fn logged(&self, task: F) -> impl Future + use { @@ -118,8 +123,8 @@ impl RequestRedis { /// Sends every post-call op declared so far and waits for the follow-ups spawned on them; the /// shutdown drain calls it too. pub async fn flush_post(&self) { - lock(&self.0.deadline).take(); - let follow_ups = std::mem::take(&mut *lock(&self.0.follow_ups)); + self.0.deadline.lock().take(); + let follow_ups = std::mem::take(&mut *self.0.follow_ups.lock()); join_all(self.batches().iter().map(|batches| batches.post.flush())).await; join_all(follow_ups) .await @@ -129,11 +134,11 @@ impl RequestRedis { } fn batches(&self) -> Vec { - lock(&self.0.batches).clone() + self.0.batches.lock().clone() } fn batches_for(&self, client: &RedisClient) -> ClientBatches { - let mut batches = lock(&self.0.batches); + let mut batches = self.0.batches.lock(); if let Some(existing) = batches .iter() .find(|batches| batches.client.same_client(client)) @@ -151,14 +156,19 @@ impl RequestRedis { } fn arm_deadline(&self) { - let mut deadline = lock(&self.0.deadline); + if self.0.deadline.lock().is_some() { + return; + } + // Asked before taking the lock: the host may answer under its interpreter lock, which + // another declaring thread can hold while it waits for this one. + let tag = (self.0.deadline_tag)(); + let mut deadline = self.0.deadline.lock(); if deadline.is_some() { return; } let (disarm, disarmed) = oneshot::channel(); *deadline = Some(disarm); let request = self.clone(); - let tag = self.0.deadline_tag.clone(); self.0.runtime.spawn(self.logged(async move { tokio::select! { () = tokio::time::sleep(request.0.post_call_deadline) => within(tag, request.flush_post()).await, @@ -168,6 +178,10 @@ impl RequestRedis { } } +/// Names the scope a deadline flush runs in. It is asked when a post-call declaration arms the +/// deadline, on the declaring thread, so it can describe that declaration's caller. +pub type DeadlineTag = Arc Option + std::marker::Send + Sync>; + async fn within(tag: Option, task: F) -> F::Output { match tag { Some(tag) => tagged(tag, task).await, @@ -196,6 +210,14 @@ impl Send<'_> { } } + /// The request the op rides, `None` for one on its own. + pub(crate) fn request(&self) -> Option<&RequestRedis> { + match self { + Self::Now => None, + Self::PreCall(request) | Self::PostCall(request) => Some(request), + } + } + /// The request's misses on `client`; `None` for an op outside a request. pub(crate) fn misses(&self, client: &RedisClient) -> Option { match self { diff --git a/litellm/rust/crates/redis-store/src/router_cooldowns/mod.rs b/litellm/rust/crates/redis-store/src/router_cooldowns/mod.rs index 69493cbf7ce..ca958512c1a 100644 --- a/litellm/rust/crates/redis-store/src/router_cooldowns/mod.rs +++ b/litellm/rust/crates/redis-store/src/router_cooldowns/mod.rs @@ -191,14 +191,10 @@ impl RouterCooldowns { memory.delete(&key)?; return Ok(false); } - let remaining = Duration::from_secs_f64(remaining); - if memory.expires_at(&key)? > Some(now + remaining + CORRECTION_SLACK) { - memory.delete(&key)?; - memory.set( - &key, - cooldown.clone(), - Some(remaining.min(MAX_CORRECTED_TTL)), - )?; + let remaining = Duration::try_from_secs_f64(remaining).unwrap_or(Duration::MAX); + let ends = now.saturating_add(remaining); + if memory.expires_at(&key)? > Some(ends.saturating_add(CORRECTION_SLACK)) { + memory.refresh(&key, cooldown.clone(), remaining.min(MAX_CORRECTED_TTL))?; } Ok(true) } diff --git a/litellm/rust/crates/redis-store/src/router_session_pins/claim.rs b/litellm/rust/crates/redis-store/src/router_session_pins/claim.rs new file mode 100644 index 00000000000..7b95af3c990 --- /dev/null +++ b/litellm/rust/crates/redis-store/src/router_session_pins/claim.rs @@ -0,0 +1,192 @@ +use litellm_python_compat::{Value, repr::to_str}; + +use super::types::Eligible; + +/// A claim settled against this router's memory alone: who holds the key, and what memory keeps +/// for it. A holder nobody may replace is answered without being written again. +#[derive(Debug, PartialEq)] +pub(super) struct LocalClaim { + pub(super) winner: Value, + pub(super) stored: Option, +} + +/// The claim script's rule for a router without Redis, or one whose Redis claim failed: the first +/// writer keeps the key unless `eligible` no longer lists it. +pub(super) fn local(existing: Option, pin: &Value, eligible: &Eligible) -> LocalClaim { + match (existing, &eligible.values) { + (Some(existing), None) => LocalClaim { + stored: legacy_matches(&existing, pin).then(|| pin.clone()), + winner: existing, + }, + (Some(existing), Some(values)) + if still_eligible(&existing, values, eligible.preserve_legacy_model_ids) => + { + LocalClaim { + stored: Some(existing.clone()), + winner: existing, + } + } + _ => LocalClaim { + stored: Some(pin.clone()), + winner: pin.clone(), + }, + } +} + +fn still_eligible(existing: &Value, eligible: &[Value], preserve_legacy_model_ids: bool) -> bool { + eligible.iter().any(|value| same_pin(existing, value)) + || (preserve_legacy_model_ids + && matches!(existing, Value::Str(_)) + && eligible.iter().any(|value| { + field(value, &Value::Str("model_id".to_owned())).is_some() + && legacy_matches(existing, value) + })) +} + +/// A stored mapping carries every field of `pin` as its text, or the bare string an older writer +/// left is the one value `pin` holds. +fn legacy_matches(stored: &Value, pin: &Value) -> bool { + let Value::Dict(fields) = pin else { + return false; + }; + match stored { + Value::Dict(_) => fields.iter().all(|(key, value)| { + field(stored, key) + .is_some_and(|held| *held != Value::None && Value::Str(to_str(held)) == *value) + }), + Value::Str(_) => matches!(fields.as_slice(), [(_, value)] if value == stored), + _ => false, + } +} + +/// Python's `==` between two pins: mappings compare by their fields, in whatever order. +fn same_pin(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Dict(fields), Value::Dict(other)) => { + fields.len() == other.len() + && fields + .iter() + .all(|(key, value)| field(right, key) == Some(value)) + } + _ => left == right, + } +} + +fn field<'a>(mapping: &'a Value, key: &Value) -> Option<&'a Value> { + let Value::Dict(fields) = mapping else { + return None; + }; + fields + .iter() + .find(|(held, _)| held == key) + .map(|(_, value)| value) +} + +#[cfg(test)] +mod tests { + use litellm_python_compat::json; + use rstest::rstest; + + use super::*; + + fn value(text: &str) -> Value { + json::loads(text).unwrap() + } + + fn eligible(values: &[&str], preserve_legacy_model_ids: bool) -> Eligible { + Eligible { + values: Some(values.iter().copied().map(value).collect()), + preserve_legacy_model_ids, + } + } + + #[rstest] + #[case::nobody_holds_the_key( + None, + r#"{"model_id": "b"}"#, + r#"{"model_id": "b"}"#, + Some(r#"{"model_id": "b"}"#) + )] + #[case::the_first_writer_keeps_it( + Some(r#"{"model_id": "a"}"#), + r#"{"model_id": "b"}"#, + r#"{"model_id": "a"}"#, + None + )] + #[case::the_holder_claiming_again_is_refreshed( + Some(r#"{"model_id": "a"}"#), + r#"{"model_id": "a"}"#, + r#"{"model_id": "a"}"#, + Some(r#"{"model_id": "a"}"#) + )] + #[case::a_holder_with_a_number_matches_its_text( + Some(r#"{"model_id": 7}"#), + r#"{"model_id": "7"}"#, + r#"{"model_id": 7}"#, + Some(r#"{"model_id": "7"}"#) + )] + #[case::a_holder_with_a_null_field_is_not_refreshed( + Some(r#"{"model_id": null}"#), + r#"{"model_id": "None"}"#, + r#"{"model_id": null}"#, + None + )] + #[case::a_bare_id_an_older_writer_left_is_refreshed_as_the_pin( + Some(r#""a""#), + r#"{"model_id": "a"}"#, + r#""a""#, + Some(r#"{"model_id": "a"}"#) + )] + #[case::a_bare_id_does_not_match_a_pin_of_two_fields( + Some(r#""a""#), + r#"{"model_id": "a", "tier": "a"}"#, + r#""a""#, + None + )] + fn a_claim_naming_no_eligible_values_keeps_the_first_writer( + #[case] existing: Option<&str>, + #[case] pin: &str, + #[case] winner: &str, + #[case] stored: Option<&str>, + ) { + let settled = local(existing.map(value), &value(pin), &Eligible::default()); + + assert_eq!( + settled, + LocalClaim { + winner: value(winner), + stored: stored.map(value), + } + ); + } + + #[rstest] + #[case::nobody_holds_the_key(None, &[r#"{"model_id": "b"}"#], false, r#"{"model_id": "b"}"#)] + #[case::an_eligible_holder_keeps_it(Some(r#"{"model_id": "a"}"#), &[r#"{"model_id": "a"}"#, r#"{"model_id": "b"}"#], false, r#"{"model_id": "a"}"#)] + #[case::fields_match_in_any_order(Some(r#"{"model": "m", "tier": "t"}"#), &[r#"{"tier": "t", "model": "m"}"#], false, r#"{"model": "m", "tier": "t"}"#)] + #[case::a_holder_no_longer_eligible_is_replaced(Some(r#"{"model_id": "z"}"#), &[r#"{"model_id": "b"}"#], false, r#"{"model_id": "b"}"#)] + #[case::a_holder_with_an_extra_field_is_replaced(Some(r#"{"model_id": "b", "tier": "t"}"#), &[r#"{"model_id": "b"}"#], false, r#"{"model_id": "b"}"#)] + #[case::a_bare_id_is_kept_when_legacy_ids_are_preserved(Some(r#""a""#), &[r#"{"model_id": "a"}"#], true, r#""a""#)] + #[case::a_bare_id_is_replaced_when_they_are_not(Some(r#""a""#), &[r#"{"model_id": "a"}"#], false, r#"{"model_id": "b"}"#)] + #[case::a_bare_id_only_matches_a_model_id(Some(r#""a""#), &[r#"{"tier": "a"}"#], true, r#"{"model_id": "b"}"#)] + fn a_claim_naming_eligible_values_replaces_only_a_holder_they_do_not_list( + #[case] existing: Option<&str>, + #[case] listed: &[&str], + #[case] preserve_legacy_model_ids: bool, + #[case] winner: &str, + ) { + let settled = local( + existing.map(value), + &value(r#"{"model_id": "b"}"#), + &eligible(listed, preserve_legacy_model_ids), + ); + + assert_eq!( + settled, + LocalClaim { + winner: value(winner), + stored: Some(value(winner)), + } + ); + } +} diff --git a/litellm/rust/crates/redis-store/src/router_session_pins/keys.rs b/litellm/rust/crates/redis-store/src/router_session_pins/keys.rs new file mode 100644 index 00000000000..6e7db8811ce --- /dev/null +++ b/litellm/rust/crates/redis-store/src/router_session_pins/keys.rs @@ -0,0 +1,181 @@ +use litellm_python_compat::{Value, json, number::hex_int}; +use sha2::{Digest, Sha256}; + +use crate::Error; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum PinKey { + /// Deployment affinity by caller. + Deployment { model_group: String, caller: String }, + /// Deployment affinity by session, scoped to the caller when there is one. + DeploymentSession { + model_group: String, + caller: Option, + session_id: String, + }, + /// The complexity router's session pin, whose caller is already the API key hash. + TierSession { + router: String, + caller: Option, + session_id: String, + }, + /// The complexity router's model within a tier. `tiers` hashes the router's tier config, and + /// `replay_fallback` is the model a replayed turn was pinned to. + TierModel { + router: String, + tiers: String, + caller: Option, + session_id: String, + tier: String, + replay_fallback: Option, + }, +} + +impl PinKey { + pub fn spelling(&self) -> Result { + Ok(match self { + Self::Deployment { + model_group, + caller, + } => format!("deployment_affinity:v1:{model_group}:{}", hashed(caller)), + Self::DeploymentSession { + model_group, + caller, + session_id, + } => { + let scope = caller + .as_deref() + .map_or_else(|| "unscoped".to_owned(), hashed); + format!("deployment_affinity:v1:session:{model_group}:{scope}:{session_id}") + } + Self::TierSession { + router, + caller, + session_id, + } => { + let scope = caller.as_deref().filter(|caller| !caller.is_empty()); + format!( + "complexity_router_session_affinity:v1:{router}:{}:{session_id}", + scope.unwrap_or("unscoped") + ) + } + Self::TierModel { + router, + tiers, + caller, + session_id, + tier, + replay_fallback, + } => { + let text = |value: &str| Value::Str(value.to_owned()); + let identity = [ + text(router), + text(tiers), + caller.as_deref().map_or(Value::None, text), + text(session_id), + text(tier), + ] + .into_iter() + .chain( + replay_fallback + .as_deref() + .map(|model| Value::List(vec![text("replay_fallback"), text(model)])), + ) + .collect(); + format!( + "complexity_router_tier_model_affinity:v1:{}", + sha256_hex(&json::dumps(&Value::List(identity))?) + ) + } + }) + } +} + +/// `_hash_user_key`: a caller that already reads as a SHA-256 hex digest to `int(text, 16)` keeps +/// its text lowercased, anything else is hashed. +fn hashed(caller: &str) -> String { + match caller.chars().count() == 64 && hex_int(caller).is_some() { + true => caller.to_lowercase(), + false => sha256_hex(caller), + } +} + +fn sha256_hex(text: &str) -> String { + format!("{:x}", Sha256::digest(text.as_bytes())) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::PinKey; + + fn text(value: &str) -> String { + value.to_owned() + } + + // Expected values are main's DeploymentAffinityCheck.get_affinity_cache_key and + // get_session_affinity_cache_key, and ComplexityRouter's session and tier model keys + #[rstest] + #[case::caller_hashed( + PinKey::Deployment { model_group: text("gpt-4o"), caller: text("sk-caller") }, + "deployment_affinity:v1:gpt-4o:8b6712790a2089c67aa97a2d80022df18cc65c7814350e33baebe79aab508891" + )] + #[case::caller_already_a_digest_is_lowercased( + PinKey::Deployment { model_group: text("gpt-4o"), caller: "A".repeat(64) }, + "deployment_affinity:v1:gpt-4o:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + )] + #[case::caller_int_reads_as_hex_is_kept( + PinKey::Deployment { model_group: text("gpt-4o"), caller: format!("0x{}", "f".repeat(62)) }, + "deployment_affinity:v1:gpt-4o:0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + )] + #[case::user_id_caller( + PinKey::Deployment { model_group: text("gpt-4o"), caller: text("user_id:u-1") }, + "deployment_affinity:v1:gpt-4o:a8ae9346b4f3e606434b5bf77a2ba2ea81313df4283d69d85bcef9e93a78a4f8" + )] + #[case::session_without_caller( + PinKey::DeploymentSession { model_group: text("gpt-4o"), caller: None, session_id: text("sess-1") }, + "deployment_affinity:v1:session:gpt-4o:unscoped:sess-1" + )] + #[case::session_with_caller( + PinKey::DeploymentSession { model_group: text("gpt-4o"), caller: Some(text("sk-caller")), session_id: text("sess-1") }, + "deployment_affinity:v1:session:gpt-4o:8b6712790a2089c67aa97a2d80022df18cc65c7814350e33baebe79aab508891:sess-1" + )] + #[case::session_with_empty_caller_is_hashed( + PinKey::DeploymentSession { model_group: text("gpt-4o"), caller: Some(String::new()), session_id: text("sess-1") }, + "deployment_affinity:v1:session:gpt-4o:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855:sess-1" + )] + #[case::tier_session( + PinKey::TierSession { router: text("smart"), caller: Some(text("key-hash")), session_id: text("sess-1") }, + "complexity_router_session_affinity:v1:smart:key-hash:sess-1" + )] + #[case::tier_session_with_empty_caller_is_unscoped( + PinKey::TierSession { router: text("smart"), caller: Some(String::new()), session_id: text("sess-1") }, + "complexity_router_session_affinity:v1:smart:unscoped:sess-1" + )] + #[case::tier_model( + PinKey::TierModel { + router: text("smart"), + tiers: text("cfg"), + caller: Some(text("sk-caller")), + session_id: text("sess-1"), + tier: text("SIMPLE"), + replay_fallback: None, + }, + "complexity_router_tier_model_affinity:v1:d93f3c0fe46c4ef59f1e45d1b2aa37455b2663b05deb14d4afb3339bc2094f51" + )] + #[case::tier_model_replay_without_caller( + PinKey::TierModel { + router: text("smart"), + tiers: text("cfg"), + caller: None, + session_id: text("séss"), + tier: text("COMPLEX"), + replay_fallback: Some(text("m-a")), + }, + "complexity_router_tier_model_affinity:v1:c83437032b0b0d7d8229f7aaf444096a83510ee630e5a510e4770b0f6f03a8d3" + )] + fn a_pin_key_is_spelled_as_main_spells_it(#[case] key: PinKey, #[case] expected: &str) { + assert_eq!(key.spelling().unwrap(), expected); + } +} diff --git a/litellm/rust/crates/redis-store/src/router_session_pins/lua.rs b/litellm/rust/crates/redis-store/src/router_session_pins/lua.rs new file mode 100644 index 00000000000..bb84cee3381 --- /dev/null +++ b/litellm/rust/crates/redis-store/src/router_session_pins/lua.rs @@ -0,0 +1,51 @@ +/// `_READ_PIN_SCRIPT`, byte for byte, so pods on either stack share one script SHA. +pub(super) const READ_PIN: &str = r#" +local current = redis.call('GET', KEYS[1]) +if current ~= false then + redis.call('EXPIRE', KEYS[1], ARGV[1]) +end +return current +"#; + +/// `_CLAIM_PIN_SCRIPT`, byte for byte. +pub(super) const CLAIM_PIN: &str = r#" +local current = redis.call('GET', KEYS[1]) +if current == false then + redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) + return ARGV[1] +end +if ARGV[3] then + local decoded, stored = pcall(cjson.decode, current) + if ARGV[4] then + local legacy = current + if decoded and type(stored) == 'string' then legacy = stored end + for _, eligible in ipairs(cjson.decode(ARGV[3])) do + if eligible['model_id'] == legacy then + redis.call('EXPIRE', KEYS[1], ARGV[2]) + return current + end + end + end + if decoded and type(stored) == 'table' then + for _, eligible in ipairs(cjson.decode(ARGV[3])) do + local matches = true + for key, value in pairs(eligible) do + if stored[key] ~= value then matches = false; break end + end + for key, _ in pairs(stored) do + if eligible[key] == nil then matches = false; break end + end + if matches then + redis.call('EXPIRE', KEYS[1], ARGV[2]) + return current + end + end + end + redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) + return ARGV[1] +end +if current == ARGV[1] then + redis.call('EXPIRE', KEYS[1], ARGV[2]) +end +return current +"#; diff --git a/litellm/rust/crates/redis-store/src/router_session_pins/mod.rs b/litellm/rust/crates/redis-store/src/router_session_pins/mod.rs new file mode 100644 index 00000000000..d3f3fa8263d --- /dev/null +++ b/litellm/rust/crates/redis-store/src/router_session_pins/mod.rs @@ -0,0 +1,231 @@ +mod claim; +mod keys; +mod lua; +mod types; + +use std::{ + sync::{Arc, LazyLock}, + time::Duration, +}; + +use litellm_python_compat::{Value, json}; +use litellm_redis_client::{Script, commands}; + +use crate::{ + DecodeError, Error, Pending, Send, codec, + dual_cache::{DualCache, MemorySettings}, + memory::{Clock, system_clock}, + remote::{Remote, RemoteSlot}, + request::ready, +}; + +static READ_PIN: LazyLock